From 5623e0ea38563155d6b9d103417ab4c50f820e7c Mon Sep 17 00:00:00 2001 From: Anton Dosov Date: Tue, 20 Sep 2022 16:28:34 +0200 Subject: [PATCH 01/76] [ILM] Hide readonly action if downsample enabled (#140804) --- .../features/downsample.helpers.ts | 52 +++++++++++++ .../edit_policy/features/downsample.test.ts | 73 +++++++++++++++++++ .../features/searchable_snapshots.helpers.ts | 4 + .../features/searchable_snapshots.test.ts | 4 + .../policy_serialization.test.ts | 65 ++++++++++++++--- .../phases/cold_phase/cold_phase.tsx | 16 +++- .../components/phases/hot_phase/hot_phase.tsx | 4 +- .../phases/warm_phase/warm_phase.tsx | 12 ++- .../sections/edit_policy/constants.ts | 4 + .../form/configuration_context.tsx | 20 ++++- 10 files changed, 235 insertions(+), 19 deletions(-) create mode 100644 x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/downsample.helpers.ts create mode 100644 x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/downsample.test.ts diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/downsample.helpers.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/downsample.helpers.ts new file mode 100644 index 0000000000000..be96aaabe4a71 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/downsample.helpers.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HttpSetup } from '@kbn/core/public'; +import { + createDownsampleActions, + createReadonlyActions, + createRolloverActions, + createSavePolicyAction, + createTogglePhaseAction, +} from '../../helpers'; +import { initTestBed } from '../init_test_bed'; +import { AppServicesContext } from '../../../../public/types'; + +type SetupReturn = ReturnType; + +export type DownsampleTestBed = SetupReturn extends Promise ? U : SetupReturn; + +export const setupDownsampleTestBed = async ( + httpSetup: HttpSetup, + args?: { + appServicesContext?: Partial; + } +) => { + const testBed = await initTestBed(httpSetup, args); + + return { + ...testBed, + actions: { + togglePhase: createTogglePhaseAction(testBed), + savePolicy: createSavePolicyAction(testBed), + ...createRolloverActions(testBed), + hot: { + ...createReadonlyActions(testBed, 'hot'), + ...createDownsampleActions(testBed, 'hot'), + }, + warm: { + ...createReadonlyActions(testBed, 'warm'), + ...createDownsampleActions(testBed, 'warm'), + }, + cold: { + ...createReadonlyActions(testBed, 'cold'), + ...createDownsampleActions(testBed, 'cold'), + }, + frozen: {}, + }, + }; +}; diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/downsample.test.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/downsample.test.ts new file mode 100644 index 0000000000000..70aae4b44b987 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/downsample.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act } from 'react-dom/test-utils'; +import { setupEnvironment } from '../../helpers'; +import { DownsampleTestBed, setupDownsampleTestBed } from './downsample.helpers'; + +describe(' downsample', () => { + let testBed: DownsampleTestBed; + const { httpSetup, httpRequestsMockHelpers } = setupEnvironment(); + + beforeEach(async () => { + httpRequestsMockHelpers.setDefaultResponses(); + + await act(async () => { + testBed = await setupDownsampleTestBed(httpSetup); + }); + + const { component } = testBed; + component.update(); + + const { actions } = testBed; + await actions.rollover.toggleDefault(); + await actions.togglePhase('warm'); + await actions.togglePhase('cold'); + }); + + test('enabling downsample in hot should hide readonly in hot, warm and cold', async () => { + const { actions } = testBed; + + expect(actions.hot.readonlyExists()).toBeTruthy(); + expect(actions.warm.readonlyExists()).toBeTruthy(); + expect(actions.cold.readonlyExists()).toBeTruthy(); + + await actions.hot.downsample.toggle(); + + expect(actions.hot.readonlyExists()).toBeFalsy(); + expect(actions.warm.readonlyExists()).toBeFalsy(); + expect(actions.cold.readonlyExists()).toBeFalsy(); + }); + + test('enabling downsample in warm should hide readonly in warm and cold', async () => { + const { actions } = testBed; + + expect(actions.hot.readonlyExists()).toBeTruthy(); + expect(actions.warm.readonlyExists()).toBeTruthy(); + expect(actions.cold.readonlyExists()).toBeTruthy(); + + await actions.warm.downsample.toggle(); + + expect(actions.hot.readonlyExists()).toBeTruthy(); + expect(actions.warm.readonlyExists()).toBeFalsy(); + expect(actions.cold.readonlyExists()).toBeFalsy(); + }); + + test('enabling downsample in cold should hide readonly in cold', async () => { + const { actions } = testBed; + + expect(actions.hot.readonlyExists()).toBeTruthy(); + expect(actions.warm.readonlyExists()).toBeTruthy(); + expect(actions.cold.readonlyExists()).toBeTruthy(); + + await actions.cold.downsample.toggle(); + + expect(actions.hot.readonlyExists()).toBeTruthy(); + expect(actions.warm.readonlyExists()).toBeTruthy(); + expect(actions.cold.readonlyExists()).toBeFalsy(); + }); +}); diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/searchable_snapshots.helpers.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/searchable_snapshots.helpers.ts index f4b41cedd70f1..e70f721a48075 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/searchable_snapshots.helpers.ts +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/searchable_snapshots.helpers.ts @@ -7,6 +7,7 @@ import { HttpSetup } from '@kbn/core/public'; import { + createDownsampleActions, createForceMergeActions, createMinAgeActions, createReadonlyActions, @@ -41,16 +42,19 @@ export const setupSearchableSnapshotsTestBed = async ( ...createSearchableSnapshotActions(testBed, 'hot'), ...createForceMergeActions(testBed, 'hot'), ...createShrinkActions(testBed, 'hot'), + ...createDownsampleActions(testBed, 'hot'), }, warm: { ...createForceMergeActions(testBed, 'warm'), ...createShrinkActions(testBed, 'warm'), ...createReadonlyActions(testBed, 'warm'), + ...createDownsampleActions(testBed, 'warm'), }, cold: { ...createMinAgeActions(testBed, 'cold'), ...createSearchableSnapshotActions(testBed, 'cold'), ...createReadonlyActions(testBed, 'cold'), + ...createDownsampleActions(testBed, 'cold'), }, frozen: { ...createMinAgeActions(testBed, 'frozen'), diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/searchable_snapshots.test.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/searchable_snapshots.test.ts index 09bb8b85082b6..03b7670c1eac5 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/searchable_snapshots.test.ts +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/features/searchable_snapshots.test.ts @@ -40,17 +40,21 @@ describe(' searchable snapshots', () => { expect(actions.warm.forceMergeExists()).toBeTruthy(); expect(actions.warm.shrinkExists()).toBeTruthy(); expect(actions.warm.readonlyExists()).toBeTruthy(); + expect(actions.warm.downsample.exists()).toBeTruthy(); expect(actions.cold.searchableSnapshotsExists()).toBeTruthy(); expect(actions.cold.readonlyExists()).toBeTruthy(); + expect(actions.cold.downsample.exists()).toBeTruthy(); await actions.hot.setSearchableSnapshot('my-repo'); expect(actions.warm.forceMergeExists()).toBeFalsy(); expect(actions.warm.shrinkExists()).toBeFalsy(); expect(actions.warm.readonlyExists()).toBeFalsy(); + expect(actions.warm.downsample.exists()).toBeFalsy(); // searchable snapshot in cold is still visible expect(actions.cold.searchableSnapshotsExists()).toBeTruthy(); expect(actions.cold.readonlyExists()).toBeFalsy(); + expect(actions.cold.downsample.exists()).toBeFalsy(); }); test('disabling rollover toggle, but enabling default rollover', async () => { diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/serialization/policy_serialization.test.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/serialization/policy_serialization.test.ts index 321a7efbfc5e3..983cfbadfa017 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/serialization/policy_serialization.test.ts +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/serialization/policy_serialization.test.ts @@ -201,8 +201,6 @@ describe(' serialization', () => { await actions.hot.setShrinkCount('2'); await actions.hot.toggleReadonly(); await actions.hot.setIndexPriority('123'); - await actions.hot.downsample.toggle(); - await actions.hot.downsample.setDownsampleInterval('2', 'h'); await actions.savePolicy(); @@ -233,7 +231,6 @@ describe(' serialization', () => { priority: 123, }, readonly: {}, - downsample: { fixed_interval: '2h' }, }, }, }, @@ -257,6 +254,24 @@ describe(' serialization', () => { expect(parsedReqBody.phases.hot.actions.searchable_snapshot.snapshot_repository).toBe('abc'); }); + // Setting downsample disables setting readonly so we test this separately + test('setting downsample', async () => { + const { actions } = testBed; + + await actions.rollover.toggleDefault(); + await actions.hot.downsample.toggle(); + await actions.hot.downsample.setDownsampleInterval('2', 'h'); + + await actions.savePolicy(); + + const lastReq: HttpFetchOptionsWithPath[] = httpSetup.post.mock.calls.pop() || []; + const [requestUrl, requestBody] = lastReq; + const parsedReqBody = JSON.parse((requestBody as Record).body); + + expect(requestUrl).toBe(`${API_BASE_PATH}/policies`); + expect(parsedReqBody.phases.hot.actions.downsample).toEqual({ fixed_interval: '2h' }); + }); + test('disabling rollover', async () => { const { actions } = testBed; @@ -326,8 +341,6 @@ describe(' serialization', () => { await actions.warm.setBestCompression(true); await actions.warm.toggleReadonly(); await actions.warm.setIndexPriority('123'); - await actions.warm.downsample.toggle(); - await actions.warm.downsample.setDownsampleInterval('20', 'm'); await actions.savePolicy(); expect(httpSetup.post).toHaveBeenLastCalledWith( @@ -365,7 +378,6 @@ describe(' serialization', () => { number_of_replicas: 123, }, readonly: {}, - downsample: { fixed_interval: '20m' }, }, }, }, @@ -374,6 +386,25 @@ describe(' serialization', () => { ); }); + // Setting downsample disables setting readonly so we test this separately + test('setting downsample', async () => { + const { actions } = testBed; + + await actions.togglePhase('warm'); + await actions.warm.setMinAgeValue('11'); + await actions.warm.downsample.toggle(); + await actions.warm.downsample.setDownsampleInterval('20', 'm'); + + await actions.savePolicy(); + + const lastReq: HttpFetchOptionsWithPath[] = httpSetup.post.mock.calls.pop() || []; + const [requestUrl, requestBody] = lastReq; + const parsedReqBody = JSON.parse((requestBody as Record).body); + + expect(requestUrl).toBe(`${API_BASE_PATH}/policies`); + expect(parsedReqBody.phases.warm.actions.downsample).toEqual({ fixed_interval: '20m' }); + }); + describe('policy with include and exclude', () => { beforeEach(async () => { httpRequestsMockHelpers.setLoadPolicies([POLICY_WITH_INCLUDE_EXCLUDE]); @@ -469,8 +500,6 @@ describe(' serialization', () => { await actions.cold.setReplicas('123'); await actions.cold.toggleReadonly(); await actions.cold.setIndexPriority('123'); - await actions.cold.downsample.toggle(); - await actions.cold.downsample.setDownsampleInterval('5'); await actions.savePolicy(); @@ -502,7 +531,6 @@ describe(' serialization', () => { number_of_replicas: 123, }, readonly: {}, - downsample: { fixed_interval: '5d' }, }, }, }, @@ -511,6 +539,25 @@ describe(' serialization', () => { ); }); + // Setting downsample disables setting readonly so we test this separately + test('setting downsample', async () => { + const { actions } = testBed; + + await actions.togglePhase('cold'); + await actions.cold.setMinAgeValue('11'); + await actions.cold.downsample.toggle(); + await actions.cold.downsample.setDownsampleInterval('2'); + + await actions.savePolicy(); + + const lastReq: HttpFetchOptionsWithPath[] = httpSetup.post.mock.calls.pop() || []; + const [requestUrl, requestBody] = lastReq; + const parsedReqBody = JSON.parse((requestBody as Record).body); + + expect(requestUrl).toBe(`${API_BASE_PATH}/policies`); + expect(parsedReqBody.phases.cold.actions.downsample).toEqual({ fixed_interval: '2d' }); + }); + // Setting searchable snapshot field disables setting replicas so we test this separately test('setting searchable snapshot', async () => { const { actions } = testBed; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase/cold_phase.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase/cold_phase.tsx index e3214eab3ec47..dd983f336a224 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase/cold_phase.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase/cold_phase.tsx @@ -30,17 +30,25 @@ const i18nTexts = { }; export const ColdPhase: FunctionComponent = () => { - const { isUsingSearchableSnapshotInHotPhase } = useConfiguration(); + const { + isUsingSearchableSnapshotInHotPhase, + isUsingDownsampleInHotPhase, + isUsingDownsampleInWarmPhase, + isUsingDownsampleInColdPhase, + } = useConfiguration(); return ( }> - {/* Readonly section */} - {!isUsingSearchableSnapshotInHotPhase && } - {!isUsingSearchableSnapshotInHotPhase && } + {/* Readonly section */} + {!isUsingSearchableSnapshotInHotPhase && + !isUsingDownsampleInHotPhase && + !isUsingDownsampleInWarmPhase && + !isUsingDownsampleInColdPhase && } + {/* Data tier allocation section */} { const [formData] = useFormData({ watch: [isUsingDefaultRolloverPath, ...rolloverFieldPaths], }); - const { isUsingRollover } = useConfiguration(); + const { isUsingRollover, isUsingDownsampleInHotPhase } = useConfiguration(); const isUsingDefaultRollover: boolean = get(formData, isUsingDefaultRolloverPath); const showEmptyRolloverFieldsError = useRolloverValueRequiredValidation(); @@ -176,8 +176,8 @@ export const HotPhase: FunctionComponent = () => { {} {license.canUseSearchableSnapshot() && } - + {!isUsingDownsampleInHotPhase && } )} diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/warm_phase.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/warm_phase.tsx index fba9556b5f4ea..af69a388cd747 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/warm_phase.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/warm_phase.tsx @@ -31,7 +31,11 @@ const i18nTexts = { }; export const WarmPhase: FunctionComponent = () => { - const { isUsingSearchableSnapshotInHotPhase } = useConfiguration(); + const { + isUsingSearchableSnapshotInHotPhase, + isUsingDownsampleInHotPhase, + isUsingDownsampleInWarmPhase, + } = useConfiguration(); return ( @@ -41,10 +45,12 @@ export const WarmPhase: FunctionComponent = () => { {!isUsingSearchableSnapshotInHotPhase && } - {!isUsingSearchableSnapshotInHotPhase && } - {!isUsingSearchableSnapshotInHotPhase && } + {!isUsingSearchableSnapshotInHotPhase && + !isUsingDownsampleInHotPhase && + !isUsingDownsampleInWarmPhase && } + {/* Data tier allocation section */} + `_meta.${phase}.downsample.enabled`; + /** * These strings describe the path to their respective values in the serialized * ILM form. diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/configuration_context.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/configuration_context.tsx index 5d506d6235f3f..63dfaeed57e9f 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/configuration_context.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/configuration_context.tsx @@ -10,7 +10,11 @@ import React, { FunctionComponent, createContext, useContext } from 'react'; import { useFormData } from '../../../../shared_imports'; -import { isUsingDefaultRolloverPath, isUsingCustomRolloverPath } from '../constants'; +import { + isUsingDefaultRolloverPath, + isUsingCustomRolloverPath, + isUsingDownsamplePath, +} from '../constants'; export interface Configuration { /** @@ -26,6 +30,14 @@ export interface Configuration { */ isUsingSearchableSnapshotInHotPhase: boolean; isUsingSearchableSnapshotInColdPhase: boolean; + + /** + * When downsample enabled it implicitly makes index readonly, + * We should hide readonly action if downsample is enabled + */ + isUsingDownsampleInHotPhase: boolean; + isUsingDownsampleInWarmPhase: boolean; + isUsingDownsampleInColdPhase: boolean; } const ConfigurationContext = createContext(null as any); @@ -43,6 +55,9 @@ export const ConfigurationProvider: FunctionComponent = ({ children }) => { pathToColdPhaseSearchableSnapshot, isUsingCustomRolloverPath, isUsingDefaultRolloverPath, + isUsingDownsamplePath('hot'), + isUsingDownsamplePath('warm'), + isUsingDownsamplePath('cold'), ], }); const isUsingDefaultRollover = get(formData, isUsingDefaultRolloverPath); @@ -53,6 +68,9 @@ export const ConfigurationProvider: FunctionComponent = ({ children }) => { isUsingRollover: isUsingDefaultRollover === false ? isUsingCustomRollover : true, isUsingSearchableSnapshotInHotPhase: get(formData, pathToHotPhaseSearchableSnapshot) != null, isUsingSearchableSnapshotInColdPhase: get(formData, pathToColdPhaseSearchableSnapshot) != null, + isUsingDownsampleInHotPhase: !!get(formData, isUsingDownsamplePath('hot')), + isUsingDownsampleInWarmPhase: !!get(formData, isUsingDownsamplePath('warm')), + isUsingDownsampleInColdPhase: !!get(formData, isUsingDownsamplePath('cold')), }; return {children}; From 01b604eeb66e7ce34eae1d0b3bec625a53f7eea9 Mon Sep 17 00:00:00 2001 From: Kevin Qualters <56408403+kqualters-elastic@users.noreply.github.com> Date: Tue, 20 Sep 2022 10:29:45 -0400 Subject: [PATCH 02/76] [Security Solution] Make timerange an optional request param, fire unbounded request when 0 results (#140831) * Make timerange an optional request param, fire unbounded request when 0 results * WIP working hook, request running twice sometimes * Add cypress test, update reducer/selector tests, fix types * Remove unneeded ternary --- .../common/endpoint/schema/resolver.ts | 10 ++-- .../e2e/detection_alerts/resolver.cy.ts | 45 ++++++++++++++ .../cypress/screens/alerts.ts | 4 ++ .../security_solution/cypress/tasks/alerts.ts | 5 ++ .../public/resolver/store/data/action.ts | 8 ++- .../resolver/store/data/reducer.test.ts | 34 ++++++++++- .../public/resolver/store/data/reducer.ts | 2 + .../resolver/store/data/selectors.test.ts | 4 +- .../public/resolver/store/data/selectors.ts | 4 ++ .../store/middleware/resolver_tree_fetcher.ts | 58 ++++++++++++++++--- .../public/resolver/store/selectors.ts | 2 + .../public/resolver/types.ts | 8 ++- .../view/resolver_without_providers.tsx | 3 +- .../resolver/view/use_autotune_timerange.ts | 43 ++++++++++++++ .../server/endpoint/routes/resolver.ts | 1 - .../routes/resolver/tree/queries/base.ts | 54 +++++++++++++++++ .../resolver/tree/queries/descendants.ts | 50 ++++------------ .../routes/resolver/tree/queries/lifecycle.ts | 41 ++++--------- .../routes/resolver/tree/queries/stats.ts | 47 +++------------ .../routes/resolver/tree/utils/fetch.ts | 2 +- .../test/security_solution_cypress/config.ts | 1 + 21 files changed, 294 insertions(+), 132 deletions(-) create mode 100644 x-pack/plugins/security_solution/cypress/e2e/detection_alerts/resolver.cy.ts create mode 100644 x-pack/plugins/security_solution/public/resolver/view/use_autotune_timerange.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/base.ts diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts b/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts index c397d70c6b019..15c89c8cd9c28 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/resolver.ts @@ -24,10 +24,12 @@ export const validateTree = { descendants: schema.number({ defaultValue: 1000, min: 0, max: 10000 }), // if the ancestry array isn't specified allowing 200 might be too high ancestors: schema.number({ defaultValue: 200, min: 0, max: 10000 }), - timeRange: schema.object({ - from: schema.string(), - to: schema.string(), - }), + timeRange: schema.maybe( + schema.object({ + from: schema.string(), + to: schema.string(), + }) + ), schema: schema.object({ // the ancestry field is optional ancestry: schema.maybe(schema.string({ minLength: 1 })), diff --git a/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/resolver.cy.ts b/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/resolver.cy.ts new file mode 100644 index 0000000000000..aa2263b9b518c --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/e2e/detection_alerts/resolver.cy.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ANALYZER_NODE } from '../../screens/alerts'; + +import { openAnalyzerForFirstAlertInTimeline } from '../../tasks/alerts'; +import { createCustomRuleEnabled } from '../../tasks/api_calls/rules'; +import { getNewRule } from '../../objects/rule'; +import { cleanKibana } from '../../tasks/common'; +import { setStartDate } from '../../tasks/date_picker'; +import { TOASTER } from '../../screens/alerts_detection_rules'; +import { waitForAlertsToPopulate } from '../../tasks/create_new_rule'; +import { login, visit } from '../../tasks/login'; +import { ALERTS_URL } from '../../urls/navigation'; + +describe('Analyze events view for alerts', () => { + before(() => { + cleanKibana(); + login(); + createCustomRuleEnabled(getNewRule()); + }); + beforeEach(() => { + visit(ALERTS_URL); + waitForAlertsToPopulate(); + }); + + it('should render analyzer when button is clicked', () => { + openAnalyzerForFirstAlertInTimeline(); + cy.get(ANALYZER_NODE).first().should('be.visible'); + }); + + it(`should render an analyzer view and display + a toast indicating the date range of found events when a time range has 0 events in it`, () => { + const dateContainingZeroEvents = 'Jul 27, 2022 @ 00:00:00.000'; + setStartDate(dateContainingZeroEvents); + waitForAlertsToPopulate(); + openAnalyzerForFirstAlertInTimeline(); + cy.get(TOASTER).should('be.visible'); + cy.get(ANALYZER_NODE).first().should('be.visible'); + }); +}); diff --git a/x-pack/plugins/security_solution/cypress/screens/alerts.ts b/x-pack/plugins/security_solution/cypress/screens/alerts.ts index 28ccdef683971..db8596074ca99 100644 --- a/x-pack/plugins/security_solution/cypress/screens/alerts.ts +++ b/x-pack/plugins/security_solution/cypress/screens/alerts.ts @@ -81,6 +81,10 @@ export const SELECT_TABLE = '[data-test-subj="table"]'; export const SEND_ALERT_TO_TIMELINE_BTN = '[data-test-subj="send-alert-to-timeline-button"]'; +export const OPEN_ANALYZER_BTN = '[data-test-subj="view-in-analyzer"]'; + +export const ANALYZER_NODE = '[data-test-subj="resolver:node"'; + export const SEVERITY = '[data-test-subj^=formatted-field][data-test-subj$=severity]'; export const SOURCE_IP = '[data-test-subj^=formatted-field][data-test-subj$=source\\.ip]'; diff --git a/x-pack/plugins/security_solution/cypress/tasks/alerts.ts b/x-pack/plugins/security_solution/cypress/tasks/alerts.ts index 69a0159f7f44d..b9191075a214f 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/alerts.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/alerts.ts @@ -25,6 +25,7 @@ import { TAKE_ACTION_POPOVER_BTN, TIMELINE_CONTEXT_MENU_BTN, CLOSE_FLYOUT, + OPEN_ANALYZER_BTN, } from '../screens/alerts'; import { REFRESH_BUTTON } from '../screens/security_header'; import { @@ -158,6 +159,10 @@ export const investigateFirstAlertInTimeline = () => { cy.get(SEND_ALERT_TO_TIMELINE_BTN).first().click({ force: true }); }; +export const openAnalyzerForFirstAlertInTimeline = () => { + cy.get(OPEN_ANALYZER_BTN).first().click({ force: true }); +}; + export const addAlertPropertyToTimeline = (propertySelector: string, rowIndex: number) => { cy.get(propertySelector).eq(rowIndex).trigger('mouseover'); cy.get(ALERT_TABLE_CELL_ACTIONS_ADD_TO_TIMELINE).first().click({ force: true }); diff --git a/x-pack/plugins/security_solution/public/resolver/store/data/action.ts b/x-pack/plugins/security_solution/public/resolver/store/data/action.ts index c93ba64ba801e..253135516530d 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/data/action.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/data/action.ts @@ -11,7 +11,7 @@ import type { SafeResolverEvent, ResolverSchema, } from '../../../../common/endpoint/types'; -import type { TreeFetcherParameters, PanelViewAndParameters } from '../../types'; +import type { TreeFetcherParameters, PanelViewAndParameters, TimeFilters } from '../../types'; interface ServerReturnedResolverData { readonly type: 'serverReturnedResolverData'; @@ -32,6 +32,12 @@ interface ServerReturnedResolverData { * The database parameters that was used to fetch the resolver tree */ parameters: TreeFetcherParameters; + + /** + * If the user supplied date range results in 0 process events, + * an unbounded request is made, and the time range of the result set displayed to the user through this value. + */ + detectedBounds?: TimeFilters; }; } diff --git a/x-pack/plugins/security_solution/public/resolver/store/data/reducer.test.ts b/x-pack/plugins/security_solution/public/resolver/store/data/reducer.test.ts index 52cb97dbc24f5..f24925441b8fb 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/data/reducer.test.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/data/reducer.test.ts @@ -10,7 +10,7 @@ import { createStore } from 'redux'; import { RelatedEventCategory } from '../../../../common/endpoint/generate_data'; import { dataReducer } from './reducer'; import * as selectors from './selectors'; -import type { DataState, GeneratedTreeMetadata } from '../../types'; +import type { DataState, GeneratedTreeMetadata, TimeFilters } from '../../types'; import type { DataAction } from './action'; import { generateTreeWithDAL } from '../../data_access_layer/mocks/generator_tree'; import { endpointSourceSchema, winlogSourceSchema } from '../../mocks/tree_schema'; @@ -24,11 +24,19 @@ type SourceAndSchemaFunction = () => { schema: ResolverSchema; dataSource: strin */ describe('Resolver Data Middleware', () => { let store: Store; - let dispatchTree: (tree: NewResolverTree, sourceAndSchema: SourceAndSchemaFunction) => void; + let dispatchTree: ( + tree: NewResolverTree, + sourceAndSchema: SourceAndSchemaFunction, + detectedBounds?: TimeFilters + ) => void; beforeEach(() => { store = createStore(dataReducer, undefined); - dispatchTree = (tree: NewResolverTree, sourceAndSchema: SourceAndSchemaFunction) => { + dispatchTree = ( + tree: NewResolverTree, + sourceAndSchema: SourceAndSchemaFunction, + detectedBounds?: TimeFilters + ) => { const { schema, dataSource } = sourceAndSchema(); const action: DataAction = { type: 'serverReturnedResolverData', @@ -41,6 +49,7 @@ describe('Resolver Data Middleware', () => { indices: [], filters: {}, }, + detectedBounds, }, }; store.dispatch(action); @@ -76,6 +85,25 @@ describe('Resolver Data Middleware', () => { expect(selectors.hasMoreGenerations(store.getState())).toBeFalsy(); }); }); + describe('when a tree with detected bounds is loaded', () => { + it('should set the detected bounds when in the payload', () => { + dispatchTree(generatedTreeMetadata.formattedTree, endpointSourceSchema, { + from: 'Sep 19, 2022 @ 20:49:13.452', + to: 'Sep 19, 2022 @ 20:49:13.452', + }); + expect(selectors.detectedBounds(store.getState())).toBeTruthy(); + }); + + it('should clear the previous detected bounds when a new response without detected bounds is recevied', () => { + dispatchTree(generatedTreeMetadata.formattedTree, endpointSourceSchema, { + from: 'Sep 19, 2022 @ 20:49:13.452', + to: 'Sep 19, 2022 @ 20:49:13.452', + }); + expect(selectors.detectedBounds(store.getState())).toBeTruthy(); + dispatchTree(generatedTreeMetadata.formattedTree, endpointSourceSchema); + expect(selectors.detectedBounds(store.getState())).toBeFalsy(); + }); + }); }); describe('when the generated tree has dimensions larger than the limits sent to the server', () => { diff --git a/x-pack/plugins/security_solution/public/resolver/store/data/reducer.ts b/x-pack/plugins/security_solution/public/resolver/store/data/reducer.ts index 64fe9080a3d5d..f2abb722b09bb 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/data/reducer.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/data/reducer.ts @@ -20,6 +20,7 @@ const initialState: DataState = { }, resolverComponentInstanceID: undefined, indices: [], + detectedBounds: undefined, }; /* eslint-disable complexity */ export const dataReducer: Reducer = (state = initialState, action) => { @@ -101,6 +102,7 @@ export const dataReducer: Reducer = (state = initialS // This cannot model multiple in-flight requests pendingRequestParameters: undefined, }, + detectedBounds: action.payload.detectedBounds, }; return nextState; } else if (action.type === 'serverFailedToReturnResolverData') { diff --git a/x-pack/plugins/security_solution/public/resolver/store/data/selectors.test.ts b/x-pack/plugins/security_solution/public/resolver/store/data/selectors.test.ts index bbec11486bcce..511cebc2c6448 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/data/selectors.test.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/data/selectors.test.ts @@ -6,7 +6,7 @@ */ import * as selectors from './selectors'; -import type { DataState, TimeRange } from '../../types'; +import type { DataState } from '../../types'; import type { ResolverAction } from '../actions'; import { dataReducer } from './reducer'; import { createStore } from 'redux'; @@ -425,7 +425,7 @@ describe('data state', () => { expect(selectors.timeRangeFilters(state())?.to).toBe(new Date(maxDate).toISOString()); }); describe('when resolver receives time range filters', () => { - const timeRangeFilters: TimeRange = { + const timeRangeFilters = { to: 'to', from: 'from', }; diff --git a/x-pack/plugins/security_solution/public/resolver/store/data/selectors.ts b/x-pack/plugins/security_solution/public/resolver/store/data/selectors.ts index 72afd77482c8f..0d43f22747f06 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/data/selectors.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/data/selectors.ts @@ -46,6 +46,10 @@ export function isTreeLoading(state: DataState): boolean { return state.tree?.pendingRequestParameters !== undefined; } +export function detectedBounds(state: DataState) { + return state.detectedBounds; +} + /** * If a request was made and it threw an error or returned a failure response code. */ diff --git a/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts b/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts index c2403920bac55..e4da1af5f4d79 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/middleware/resolver_tree_fetcher.ts @@ -14,6 +14,7 @@ import type { } from '../../../../common/endpoint/types'; import type { ResolverState, DataAccessLayer } from '../../types'; import * as selectors from '../selectors'; +import { firstNonNullValue } from '../../../../common/endpoint/models/ecs_safety_helpers'; import type { ResolverAction } from '../actions'; import { ancestorsRequestAmount, descendantsRequestAmount } from '../../models/resolver_tree'; @@ -83,15 +84,56 @@ export function ResolverTreeFetcher( nodes: result, }; - api.dispatch({ - type: 'serverReturnedResolverData', - payload: { - result: resolverTree, - dataSource, + if (resolverTree.nodes.length === 0) { + const unboundedTree = await dataAccessLayer.resolverTree({ + dataId: entityIDToFetch, schema: dataSourceSchema, - parameters: databaseParameters, - }, - }); + indices: databaseParameters.indices, + ancestors: ancestorsRequestAmount(dataSourceSchema), + descendants: descendantsRequestAmount(), + }); + if (unboundedTree.length > 0) { + const timestamps = unboundedTree.map((event) => + firstNonNullValue(event.data['@timestamp']) + ); + const oldestTimestamp = timestamps[0]; + const newestTimestamp = timestamps.slice(-1); + api.dispatch({ + type: 'serverReturnedResolverData', + payload: { + result: { ...resolverTree, nodes: unboundedTree }, + dataSource, + schema: dataSourceSchema, + parameters: databaseParameters, + detectedBounds: { + from: String(oldestTimestamp), + to: String(newestTimestamp), + }, + }, + }); + // 0 results with unbounded query, fail as before + } else { + api.dispatch({ + type: 'serverReturnedResolverData', + payload: { + result: resolverTree, + dataSource, + schema: dataSourceSchema, + parameters: databaseParameters, + }, + }); + } + } else { + api.dispatch({ + type: 'serverReturnedResolverData', + payload: { + result: resolverTree, + dataSource, + schema: dataSourceSchema, + parameters: databaseParameters, + }, + }); + } } catch (error) { // https://developer.mozilla.org/en-US/docs/Web/API/DOMException#exception-AbortError if (error instanceof DOMException && error.name === 'AbortError') { diff --git a/x-pack/plugins/security_solution/public/resolver/store/selectors.ts b/x-pack/plugins/security_solution/public/resolver/store/selectors.ts index 1940e680e1b56..8d19d49745230 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/selectors.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/selectors.ts @@ -30,6 +30,8 @@ export const projectionMatrix = composeSelectors( export const translation = composeSelectors(cameraStateSelector, cameraSelectors.translation); +export const detectedBounds = composeSelectors(dataStateSelector, dataSelectors.detectedBounds); + /** * A matrix that when applied to a Vector2 converts it from screen coordinates to world coordinates. * See https://en.wikipedia.org/wiki/Orthographic_projection diff --git a/x-pack/plugins/security_solution/public/resolver/types.ts b/x-pack/plugins/security_solution/public/resolver/types.ts index d0f63ff522438..00ecd995176eb 100644 --- a/x-pack/plugins/security_solution/public/resolver/types.ts +++ b/x-pack/plugins/security_solution/public/resolver/types.ts @@ -307,6 +307,8 @@ export interface DataState { data: SafeResolverEvent | null; }; + readonly detectedBounds?: TimeFilters; + readonly tree?: { /** * The parameters passed from the resolver properties @@ -670,8 +672,8 @@ export interface IsometricTaxiLayout { * Defines the type for bounding a search by a time box. */ export interface TimeRange { - from: string; - to: string; + from: string | number; + to: string | number; } /** @@ -762,7 +764,7 @@ export interface DataAccessLayer { }: { dataId: string; schema: ResolverSchema; - timeRange: TimeRange; + timeRange?: TimeRange; indices: string[]; ancestors: number; descendants: number; diff --git a/x-pack/plugins/security_solution/public/resolver/view/resolver_without_providers.tsx b/x-pack/plugins/security_solution/public/resolver/view/resolver_without_providers.tsx index 91c2527265361..f10864cf228f7 100644 --- a/x-pack/plugins/security_solution/public/resolver/view/resolver_without_providers.tsx +++ b/x-pack/plugins/security_solution/public/resolver/view/resolver_without_providers.tsx @@ -27,6 +27,7 @@ import { PanelRouter } from './panels'; import { useColors } from './use_colors'; import { useSyncSelectedNode } from './use_sync_selected_node'; import { ResolverNoProcessEvents } from './resolver_no_process_events'; +import { useAutotuneTimerange } from './use_autotune_timerange'; /** * The highest level connected Resolver component. Needs a `Provider` in its ancestry to work. @@ -58,7 +59,7 @@ export const ResolverWithoutProviders = React.memo( shouldUpdate, filters, }); - + useAutotuneTimerange(); /** * This will keep the selectedNode in the view in sync with the nodeID specified in the url */ diff --git a/x-pack/plugins/security_solution/public/resolver/view/use_autotune_timerange.ts b/x-pack/plugins/security_solution/public/resolver/view/use_autotune_timerange.ts new file mode 100644 index 0000000000000..7ecaf5dfbcbdb --- /dev/null +++ b/x-pack/plugins/security_solution/public/resolver/view/use_autotune_timerange.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useMemo, useEffect } from 'react'; +import { i18n } from '@kbn/i18n'; +import { useSelector } from 'react-redux'; +import * as selectors from '../store/selectors'; +import { useAppToasts } from '../../common/hooks/use_app_toasts'; +import { useFormattedDate } from './panels/use_formatted_date'; +import type { ResolverState } from '../types'; + +export function useAutotuneTimerange() { + const { addSuccess } = useAppToasts(); + const { from: detectedFrom, to: detectedTo } = useSelector((state: ResolverState) => { + const detectedBounds = selectors.detectedBounds(state); + return { + from: detectedBounds?.from ? detectedBounds.from : undefined, + to: detectedBounds?.to ? detectedBounds.to : undefined, + }; + }); + const detectedFormattedFrom = useFormattedDate(detectedFrom); + const detectedFormattedTo = useFormattedDate(detectedTo); + + const successMessage = useMemo(() => { + return i18n.translate('xpack.securitySolution.resolver.unboundedRequest.toast', { + defaultMessage: `No process events were found with your selected time range, however they were + found using a start date of {from} and an end date of {to}. Select a different time range in + the date picker to use a different range.`, + values: { + from: detectedFormattedFrom, + to: detectedFormattedTo, + }, + }); + }, [detectedFormattedFrom, detectedFormattedTo]); + useEffect(() => { + if (detectedFrom || detectedTo) { + addSuccess(successMessage); + } + }, [addSuccess, successMessage, detectedFrom, detectedTo]); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts index 44f897d8883cc..ac9fb318d4c96 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver.ts @@ -13,7 +13,6 @@ import { validateEntities, validateTree, } from '../../../common/endpoint/schema/resolver'; - import { handleTree } from './resolver/tree/handler'; import { handleEntities } from './resolver/entity/handler'; import { handleEvents } from './resolver/events'; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/base.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/base.ts new file mode 100644 index 0000000000000..6637e7931b056 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/base.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { JsonValue } from '@kbn/utility-types'; +import type { ResolverSchema } from '../../../../../../common/endpoint/types'; +import type { TimeRange } from '../utils'; +import { resolverFields } from '../utils'; + +export interface ResolverQueryParams { + readonly schema: ResolverSchema; + readonly indexPatterns: string | string[]; + readonly timeRange: TimeRange | undefined; + readonly isInternalRequest: boolean; + readonly resolverFields?: JsonValue[]; + getRangeFilter?: () => Array<{ + range: { '@timestamp': { gte: string; lte: string; format: string } }; + }>; +} + +export class BaseResolverQuery implements ResolverQueryParams { + readonly schema: ResolverSchema; + readonly indexPatterns: string | string[]; + readonly timeRange: TimeRange | undefined; + readonly isInternalRequest: boolean; + readonly resolverFields?: JsonValue[]; + + constructor({ schema, indexPatterns, timeRange, isInternalRequest }: ResolverQueryParams) { + this.resolverFields = resolverFields(schema); + this.schema = schema; + this.indexPatterns = indexPatterns; + this.timeRange = timeRange; + this.isInternalRequest = isInternalRequest; + } + + getRangeFilter() { + return this.timeRange + ? [ + { + range: { + '@timestamp': { + gte: this.timeRange.from, + lte: this.timeRange.to, + format: 'strict_date_optional_time', + }, + }, + }, + ] + : []; + } +} diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts index 650549b9a9c49..daa3a513821d1 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/descendants.ts @@ -8,32 +8,20 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { IScopedClusterClient } from '@kbn/core/server'; import type { JsonObject, JsonValue } from '@kbn/utility-types'; -import type { FieldsObject, ResolverSchema } from '../../../../../../common/endpoint/types'; -import type { NodeID, TimeRange } from '../utils'; -import { resolverFields, validIDs } from '../utils'; -interface DescendantsParams { - schema: ResolverSchema; - indexPatterns: string | string[]; - timeRange: TimeRange; - isInternalRequest: boolean; -} +import type { FieldsObject } from '../../../../../../common/endpoint/types'; +import type { NodeID } from '../utils'; +import { validIDs } from '../utils'; +import type { ResolverQueryParams } from './base'; +import { BaseResolverQuery } from './base'; /** * Builds a query for retrieving descendants of a node. */ -export class DescendantsQuery { - private readonly schema: ResolverSchema; - private readonly indexPatterns: string | string[]; - private readonly timeRange: TimeRange; - private readonly isInternalRequest: boolean; - private readonly resolverFields: JsonValue[]; +export class DescendantsQuery extends BaseResolverQuery { + declare readonly resolverFields: JsonValue[]; - constructor({ schema, indexPatterns, timeRange, isInternalRequest }: DescendantsParams) { - this.resolverFields = resolverFields(schema); - this.schema = schema; - this.indexPatterns = indexPatterns; - this.timeRange = timeRange; - this.isInternalRequest = isInternalRequest; + constructor({ schema, indexPatterns, timeRange, isInternalRequest }: ResolverQueryParams) { + super({ schema, indexPatterns, timeRange, isInternalRequest }); } private query(nodes: NodeID[], size: number): JsonObject { @@ -48,15 +36,7 @@ export class DescendantsQuery { query: { bool: { filter: [ - { - range: { - '@timestamp': { - gte: this.timeRange.from, - lte: this.timeRange.to, - format: 'strict_date_optional_time', - }, - }, - }, + ...this.getRangeFilter(), { terms: { [this.schema.parent]: nodes }, }, @@ -135,15 +115,7 @@ export class DescendantsQuery { query: { bool: { filter: [ - { - range: { - '@timestamp': { - gte: this.timeRange.from, - lte: this.timeRange.to, - format: 'strict_date_optional_time', - }, - }, - }, + ...this.getRangeFilter(), { terms: { [ancestryField]: nodes, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts index 039867c9760f7..26f917a3008d2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/lifecycle.ts @@ -7,32 +7,19 @@ import type { IScopedClusterClient } from '@kbn/core/server'; import type { JsonObject, JsonValue } from '@kbn/utility-types'; -import type { FieldsObject, ResolverSchema } from '../../../../../../common/endpoint/types'; -import type { NodeID, TimeRange } from '../utils'; -import { validIDs, resolverFields } from '../utils'; - -interface LifecycleParams { - schema: ResolverSchema; - indexPatterns: string | string[]; - timeRange: TimeRange; - isInternalRequest: boolean; -} +import type { FieldsObject } from '../../../../../../common/endpoint/types'; +import type { NodeID } from '../utils'; +import { validIDs } from '../utils'; +import type { ResolverQueryParams } from './base'; +import { BaseResolverQuery } from './base'; /** * Builds a query for retrieving descendants of a node. */ -export class LifecycleQuery { - private readonly schema: ResolverSchema; - private readonly indexPatterns: string | string[]; - private readonly timeRange: TimeRange; - private readonly isInternalRequest: boolean; - private readonly resolverFields: JsonValue[]; - constructor({ schema, indexPatterns, timeRange, isInternalRequest }: LifecycleParams) { - this.resolverFields = resolverFields(schema); - this.schema = schema; - this.indexPatterns = indexPatterns; - this.timeRange = timeRange; - this.isInternalRequest = isInternalRequest; +export class LifecycleQuery extends BaseResolverQuery { + declare readonly resolverFields: JsonValue[]; + constructor({ schema, indexPatterns, timeRange, isInternalRequest }: ResolverQueryParams) { + super({ schema, indexPatterns, timeRange, isInternalRequest }); } private query(nodes: NodeID[]): JsonObject { @@ -47,15 +34,7 @@ export class LifecycleQuery { query: { bool: { filter: [ - { - range: { - '@timestamp': { - gte: this.timeRange.from, - lte: this.timeRange.to, - format: 'strict_date_optional_time', - }, - }, - }, + ...this.getRangeFilter(), { terms: { [this.schema.id]: nodes }, }, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts index eb06b7b6f5f5e..38986126f7051 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/queries/stats.ts @@ -8,8 +8,10 @@ import type { IScopedClusterClient } from '@kbn/core/server'; import type { AlertsClient } from '@kbn/rule-registry-plugin/server'; import type { JsonObject } from '@kbn/utility-types'; -import type { EventStats, ResolverSchema } from '../../../../../../common/endpoint/types'; -import type { NodeID, TimeRange } from '../utils'; +import type { EventStats } from '../../../../../../common/endpoint/types'; +import type { NodeID } from '../utils'; +import type { ResolverQueryParams } from './base'; +import { BaseResolverQuery } from './base'; interface AggBucket { key: string; @@ -26,27 +28,12 @@ interface CategoriesAgg extends AggBucket { }; } -interface StatsParams { - schema: ResolverSchema; - indexPatterns: string | string[]; - timeRange: TimeRange; - isInternalRequest: boolean; -} - /** * Builds a query for retrieving descendants of a node. */ -export class StatsQuery { - private readonly schema: ResolverSchema; - private readonly indexPatterns: string | string[]; - private readonly timeRange: TimeRange; - private readonly isInternalRequest: boolean; - - constructor({ schema, indexPatterns, timeRange, isInternalRequest }: StatsParams) { - this.schema = schema; - this.indexPatterns = indexPatterns; - this.timeRange = timeRange; - this.isInternalRequest = isInternalRequest; +export class StatsQuery extends BaseResolverQuery { + constructor({ schema, indexPatterns, timeRange, isInternalRequest }: ResolverQueryParams) { + super({ schema, indexPatterns, timeRange, isInternalRequest }); } private query(nodes: NodeID[]): JsonObject { @@ -55,15 +42,7 @@ export class StatsQuery { query: { bool: { filter: [ - { - range: { - '@timestamp': { - gte: this.timeRange.from, - lte: this.timeRange.to, - format: 'strict_date_optional_time', - }, - }, - }, + ...this.getRangeFilter(), { terms: { [this.schema.id]: nodes }, }, @@ -105,15 +84,7 @@ export class StatsQuery { query: { bool: { filter: [ - { - range: { - '@timestamp': { - gte: this.timeRange.from, - lte: this.timeRange.to, - format: 'strict_date_optional_time', - }, - }, - }, + ...this.getRangeFilter(), { terms: { [this.schema.id]: nodes }, }, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts index 148371cac74ab..85b5bbd8a277d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree/utils/fetch.ts @@ -30,7 +30,7 @@ export interface TreeOptions { descendantLevels: number; descendants: number; ancestors: number; - timeRange: { + timeRange?: { from: string; to: string; }; diff --git a/x-pack/test/security_solution_cypress/config.ts b/x-pack/test/security_solution_cypress/config.ts index 8d638789b61ff..982d52920e2ac 100644 --- a/x-pack/test/security_solution_cypress/config.ts +++ b/x-pack/test/security_solution_cypress/config.ts @@ -36,6 +36,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { ...xpackFunctionalTestsConfig.get('kbnTestServer.serverArgs'), '--csp.strict=false', '--csp.warnLegacyBrowsers=false', + '--usageCollection.uiCounters.enabled=false', // define custom kibana server args here `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`, // retrieve rules from the filesystem but not from fleet for Cypress tests From 0af26e2ab932ff8a8d1be4d1410f47ec16cb70fc Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 20 Sep 2022 10:30:41 -0400 Subject: [PATCH 03/76] [Fleet] Fix history block to support hash param (#141081) --- .../hooks/index.test.tsx | 46 +++++++++++++++++++ .../edit_package_policy_page/hooks/index.tsx | 5 +- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/index.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/index.test.tsx index 86748a8644c79..daf8177b9e73e 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/index.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/index.test.tsx @@ -102,4 +102,50 @@ describe('useHistoryBlock', () => { expect(renderer.startServices.application.navigateToUrl).not.toBeCalled(); }); }); + + describe('with hash params', () => { + it('should not block if not edited', () => { + const renderer = createFleetTestRendererMock(); + + renderer.renderHook(() => useHistoryBlock(false)); + + act(() => renderer.mountHistory.push('/test#/hash')); + + const { location } = renderer.mountHistory; + expect(location.pathname).toBe('/test'); + expect(location.hash).toBe('#/hash'); + expect(renderer.startServices.overlays.openConfirm).not.toBeCalled(); + }); + + it('should block if edited and navigate on confirm', async () => { + const renderer = createFleetTestRendererMock(); + + renderer.startServices.overlays.openConfirm.mockResolvedValue(true); + renderer.renderHook(() => useHistoryBlock(true)); + + act(() => renderer.mountHistory.push('/test#/hash')); + // needed because we have an async useEffect + await act(() => new Promise((resolve) => resolve())); + + expect(renderer.startServices.overlays.openConfirm).toBeCalled(); + expect(renderer.startServices.application.navigateToUrl).toBeCalledWith( + '/mock/test#/hash', + expect.anything() + ); + }); + + it('should block if edited and not navigate on cancel', async () => { + const renderer = createFleetTestRendererMock(); + + renderer.startServices.overlays.openConfirm.mockResolvedValue(false); + renderer.renderHook(() => useHistoryBlock(true)); + + act(() => renderer.mountHistory.push('/test#/hash')); + // needed because we have an async useEffect + await act(() => new Promise((resolve) => resolve())); + + expect(renderer.startServices.overlays.openConfirm).toBeCalled(); + expect(renderer.startServices.application.navigateToUrl).not.toBeCalled(); + }); + }); }); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/index.tsx index 8e04fbddd2ccc..edf04f8733ad8 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/hooks/index.tsx @@ -35,7 +35,10 @@ export function useHistoryBlock(isEdited: boolean) { if (confirmRes) { unblock(); - application.navigateToUrl(state.pathname + state.search, { state: state.state }); + + application.navigateToUrl(state.pathname + state.hash + state.search, { + state: state.state, + }); } } confirmAsync(); From 0f7cfd16f7e68b2ad455347d2a7a60940fc0c8b1 Mon Sep 17 00:00:00 2001 From: Julian Gernun Date: Tue, 20 Sep 2022 16:31:25 +0200 Subject: [PATCH 04/76] 137988 browser fields UI (#140516) * first commit * first commit * get auth index and try field caps * use esClient * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * wait for promise to finish * logs for debugging * format field capabilities * add simplier browserFields mapper * update response and remove width * update response * refactor * types and refactor * update api response * fix column ids * add columns toggle and reset * sort visible columns id on toggle on * merging info * call api * load info on browser field loaded * remove logs * add useColumns hook * remove browser fields dependency * update fn name * update types * update imported type package * update mock object * error message for no o11y alert indices * add endpoint integration test * activate commented tests * add unit test * comment uncommented tests * fix tests * review by Xavier * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * remove unnecessary api calls * update types * update param names + right type * update types * update id and index to be same type as rest * update timelines id and index format * add schema update on load * add functional test * fix tests * reactivate skipped test * update row action types to work with new api * rollback basic fields as array update o11y render cell fn * update cell render fn to handle strings too * update column recovery on update * recover previous deleted column stats * add browser fields error handling * add toast on error and avoid calling field api when in siem * remove spread operator * add toast mock * update render cell cb when value is an object * remove not needed prop * fix browser field types * fix reset fields action * add missing hook dependency * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * fix browser field modal types * update browser field types * update render cell * update export type * fix default columns * remove description column in browser field modal * fix populate default fields on reset * delete description field in triggers_actions_ui * avoid to refetch the data because all the data is already there * remove description tests * insert new column in first pos + minor fixes * update onToggleColumn callback to avoid innecesary calls Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Xavier Mouligneau --- .../public/plugin.tsx | 7 +- .../case_view/case_view_page.test.tsx | 5 + .../render_cell_value/render_cell_value.tsx | 21 +- x-pack/plugins/rule_registry/common/index.ts | 1 + x-pack/plugins/rule_registry/common/types.ts | 21 ++ .../server/alert_data_client/alerts_client.ts | 2 +- .../alert_data_client/browser_fields/index.ts | 4 +- .../get_browser_fields_by_feature_id.ts | 3 +- x-pack/plugins/rule_registry/server/types.ts | 9 - .../indicators_field_browser.tsx | 2 +- .../hooks/use_toolbar_options.tsx | 2 +- .../threat_intelligence/public/types.ts | 2 +- .../alerts_table/alerts_table.test.tsx | 5 + .../sections/alerts_table/alerts_table.tsx | 63 +++-- .../alerts_table/alerts_table_state.test.tsx | 10 + .../alerts_table/alerts_table_state.tsx | 54 ++-- .../bulk_actions/bulk_actions.test.tsx | 5 + .../alerts_table/hooks/translations.ts | 7 + .../alerts_table/hooks/use_columns.ts | 235 ++++++++++++++++++ .../use_fetch_browser_fields_capabilities.tsx | 73 ++++++ .../sections/alerts_table/toolbar/index.tsx | 3 +- .../toolbar/toolbar_visibility.tsx | 53 +++- .../categories_selector.tsx | 2 +- .../field_items/field_items.test.tsx | 8 - .../components/field_items/field_items.tsx | 31 +-- .../components/field_table/field_table.tsx | 3 +- .../sections/field_browser/field_browser.tsx | 3 +- .../field_browser/field_browser_modal.tsx | 3 +- .../sections/field_browser/helpers.test.ts | 2 +- .../sections/field_browser/helpers.ts | 2 +- .../sections/field_browser/mock.ts | 2 +- .../sections/field_browser/types.ts | 20 +- .../triggers_actions_ui/public/types.ts | 6 +- .../alerts_table.ts | 35 +++ .../pages/alerts/table_storage.ts | 3 +- 35 files changed, 571 insertions(+), 136 deletions(-) create mode 100644 x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/use_columns.ts create mode 100644 x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/use_fetch_browser_fields_capabilities.tsx diff --git a/x-pack/examples/triggers_actions_ui_example/public/plugin.tsx b/x-pack/examples/triggers_actions_ui_example/public/plugin.tsx index 57c752f140159..20683fb935126 100644 --- a/x-pack/examples/triggers_actions_ui_example/public/plugin.tsx +++ b/x-pack/examples/triggers_actions_ui_example/public/plugin.tsx @@ -142,7 +142,12 @@ export class TriggersActionsUiExamplePlugin useInternalFlyout, getRenderCellValue: () => (props: any) => { const value = props.data.find((d: any) => d.field === props.columnId)?.value ?? []; - return <>{value.length ? value.join() : '--'}; + + if (Array.isArray(value)) { + return <>{value.length ? value.join() : '--'}; + } + + return <>{value}; }, sort, }; diff --git a/x-pack/plugins/cases/public/components/case_view/case_view_page.test.tsx b/x-pack/plugins/cases/public/components/case_view/case_view_page.test.tsx index 8719ef2662954..a06ae9e772c11 100644 --- a/x-pack/plugins/cases/public/components/case_view/case_view_page.test.tsx +++ b/x-pack/plugins/cases/public/components/case_view/case_view_page.test.tsx @@ -427,6 +427,11 @@ describe('CaseViewPage', () => { }), }, }, + notifications: { + toasts: { + addDanger: () => {}, + }, + }, }, }), })); diff --git a/x-pack/plugins/observability/public/pages/alerts/components/render_cell_value/render_cell_value.tsx b/x-pack/plugins/observability/public/pages/alerts/components/render_cell_value/render_cell_value.tsx index 15de861284f45..ebdd07ae52995 100644 --- a/x-pack/plugins/observability/public/pages/alerts/components/render_cell_value/render_cell_value.tsx +++ b/x-pack/plugins/observability/public/pages/alerts/components/render_cell_value/render_cell_value.tsx @@ -16,6 +16,7 @@ import { TIMESTAMP, } from '@kbn/rule-data-utils'; import type { CellValueElementProps, TimelineNonEcsData } from '@kbn/timelines-plugin/common'; +import { isEmpty } from 'lodash'; import { AlertStatusIndicator } from '../../../../components/shared/alert_status_indicator'; import { TimestampTooltip } from '../../../../components/shared/timestamp_tooltip'; import { asDuration } from '../../../../../common/utils/formatters'; @@ -38,6 +39,20 @@ export const getMappedNonEcsValue = ({ return undefined; }; +const getRenderValue = (mappedNonEcsValue: any) => { + // can be updated when working on https://github.com/elastic/kibana/issues/140819 + const value = Array.isArray(mappedNonEcsValue) ? mappedNonEcsValue.join() : mappedNonEcsValue; + + if (!isEmpty(value)) { + if (typeof value === 'object') { + return JSON.stringify(value); + } + return value; + } + + return '—'; +}; + /** * This implementation of `EuiDataGrid`'s `renderCellValue` * accepts `EuiDataGridCellValueElementProps`, plus `data` @@ -53,10 +68,12 @@ export const getRenderCellValue = ({ }) => { return ({ columnId, data }: CellValueElementProps) => { if (!data) return null; - const value = getMappedNonEcsValue({ + const mappedNonEcsValue = getMappedNonEcsValue({ data, fieldName: columnId, - })?.reduce((x) => x[0]); + }); + + const value = getRenderValue(mappedNonEcsValue); switch (columnId) { case ALERT_STATUS: diff --git a/x-pack/plugins/rule_registry/common/index.ts b/x-pack/plugins/rule_registry/common/index.ts index 1f6053735d9b7..6c12d82cb95eb 100644 --- a/x-pack/plugins/rule_registry/common/index.ts +++ b/x-pack/plugins/rule_registry/common/index.ts @@ -11,3 +11,4 @@ export type { RuleRegistrySearchRequestPagination, } from './search_strategy'; export { BASE_RAC_ALERTS_API_PATH } from './constants'; +export type { BrowserFields, BrowserField } from './types'; diff --git a/x-pack/plugins/rule_registry/common/types.ts b/x-pack/plugins/rule_registry/common/types.ts index d1c5b6706c391..fd238b66c82bb 100644 --- a/x-pack/plugins/rule_registry/common/types.ts +++ b/x-pack/plugins/rule_registry/common/types.ts @@ -9,6 +9,9 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import * as t from 'io-ts'; +import type { IFieldSubType } from '@kbn/es-query'; +import type { RuntimeField } from '@kbn/data-views-plugin/common'; + // note: these schemas are not exhaustive. See the `Sort` type of `@elastic/elasticsearch` if you need to enhance it. const fieldSchema = t.string; export const sortOrderSchema = t.union([t.literal('asc'), t.literal('desc'), t.literal('_doc')]); @@ -302,3 +305,21 @@ export interface ClusterPutComponentTemplateBody { mappings: estypes.MappingTypeMapping; }; } + +export interface BrowserField { + aggregatable: boolean; + category: string; + description?: string | null; + example?: string | number | null; + fields: Readonly>>; + format?: string; + indexes: string[]; + name: string; + searchable: boolean; + type: string; + subType?: IFieldSubType; + readFromDocValues: boolean; + runtimeField?: RuntimeField; +} + +export type BrowserFields = Record>; diff --git a/x-pack/plugins/rule_registry/server/alert_data_client/alerts_client.ts b/x-pack/plugins/rule_registry/server/alert_data_client/alerts_client.ts index 31731cecbeccb..3727675d99852 100644 --- a/x-pack/plugins/rule_registry/server/alert_data_client/alerts_client.ts +++ b/x-pack/plugins/rule_registry/server/alert_data_client/alerts_client.ts @@ -31,6 +31,7 @@ import { import { Logger, ElasticsearchClient, EcsEventOutcome } from '@kbn/core/server'; import { AuditLogger } from '@kbn/security-plugin/server'; import { IndexPatternsFetcher } from '@kbn/data-plugin/server'; +import { BrowserFields } from '../../common'; import { alertAuditEvent, operationAlertAuditActionMap } from './audit_events'; import { ALERT_WORKFLOW_STATUS, @@ -42,7 +43,6 @@ import { ParsedTechnicalFields } from '../../common/parse_technical_fields'; import { Dataset, IRuleDataService } from '../rule_data_plugin_service'; import { getAuthzFilter, getSpacesFilter } from '../lib'; import { fieldDescriptorToBrowserFieldMapper } from './browser_fields'; -import { BrowserFields } from '../types'; // TODO: Fix typings https://github.com/elastic/kibana/issues/101776 type NonNullableProps = Omit & { diff --git a/x-pack/plugins/rule_registry/server/alert_data_client/browser_fields/index.ts b/x-pack/plugins/rule_registry/server/alert_data_client/browser_fields/index.ts index 074c3f60006c8..e0327495dba2d 100644 --- a/x-pack/plugins/rule_registry/server/alert_data_client/browser_fields/index.ts +++ b/x-pack/plugins/rule_registry/server/alert_data_client/browser_fields/index.ts @@ -6,7 +6,7 @@ */ import { FieldDescriptor } from '@kbn/data-views-plugin/server'; -import { BrowserField, BrowserFields } from '../../types'; +import { BrowserFields, BrowserField } from '../../../common'; const getFieldCategory = (fieldCapability: FieldDescriptor) => { const name = fieldCapability.name.split('.'); @@ -21,7 +21,7 @@ const getFieldCategory = (fieldCapability: FieldDescriptor) => { const browserFieldFactory = ( fieldCapability: FieldDescriptor, category: string -): { [fieldName in string]: BrowserField } => { +): Readonly>> => { return { [fieldCapability.name]: { ...fieldCapability, diff --git a/x-pack/plugins/rule_registry/server/routes/get_browser_fields_by_feature_id.ts b/x-pack/plugins/rule_registry/server/routes/get_browser_fields_by_feature_id.ts index 6b2d59c824ab3..44789b06d2123 100644 --- a/x-pack/plugins/rule_registry/server/routes/get_browser_fields_by_feature_id.ts +++ b/x-pack/plugins/rule_registry/server/routes/get_browser_fields_by_feature_id.ts @@ -9,6 +9,7 @@ import { IRouter } from '@kbn/core/server'; import { transformError } from '@kbn/securitysolution-es-utils'; import * as t from 'io-ts'; +import { BrowserFields } from '../../common'; import { RacRequestHandlerContext } from '../types'; import { BASE_RAC_ALERTS_API_PATH } from '../../common/constants'; import { buildRouteValidation } from './utils/route_validation'; @@ -50,7 +51,7 @@ export const getBrowserFieldsByFeatureId = (router: IRouter; - -export type BrowserField = FieldSpec & { - category: string; -}; - -export type BrowserFields = { - [category in string]: { fields: { [fieldName in string]: BrowserField } }; -}; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_browser/indicators_field_browser.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_browser/indicators_field_browser.tsx index 2adcc4ee5b9ee..22cada18b84d8 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_browser/indicators_field_browser.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_browser/indicators_field_browser.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import { BrowserField } from '@kbn/triggers-actions-ui-plugin/public/application/sections/field_browser/types'; +import { BrowserField } from '@kbn/rule-registry-plugin/common'; import { VFC } from 'react'; import { useKibana } from '../../../../hooks/use_kibana'; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/hooks/use_toolbar_options.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/hooks/use_toolbar_options.tsx index 27a57f3e44ca7..a7c4148e88aef 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/hooks/use_toolbar_options.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_table/hooks/use_toolbar_options.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { useMemo } from 'react'; import { EuiDataGridColumn, EuiText } from '@elastic/eui'; -import { BrowserField } from '@kbn/triggers-actions-ui-plugin/public/application/sections/field_browser/types'; +import { BrowserField } from '@kbn/rule-registry-plugin/common'; import { IndicatorsFieldBrowser } from '../../indicators_field_browser'; export const useToolbarOptions = ({ diff --git a/x-pack/plugins/threat_intelligence/public/types.ts b/x-pack/plugins/threat_intelligence/public/types.ts index acca5c6356ca9..976033934aa9a 100644 --- a/x-pack/plugins/threat_intelligence/public/types.ts +++ b/x-pack/plugins/threat_intelligence/public/types.ts @@ -16,8 +16,8 @@ import { import { Storage } from '@kbn/kibana-utils-plugin/public'; import { TimelinesUIStart } from '@kbn/timelines-plugin/public'; import type { TriggersAndActionsUIPublicPluginStart as TriggersActionsStart } from '@kbn/triggers-actions-ui-plugin/public'; -import { BrowserField } from '@kbn/triggers-actions-ui-plugin/public/application/sections/field_browser/types'; import { DataViewBase } from '@kbn/es-query'; +import { BrowserField } from '@kbn/rule-registry-plugin/common'; import { Store } from 'redux'; import { DataProvider } from '@kbn/timelines-plugin/common'; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table.test.tsx index a17efca41fd19..26bc284f679bc 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table.test.tsx @@ -92,6 +92,11 @@ describe('AlertsTable', () => { visibleColumns: columns.map((c) => c.id), 'data-test-subj': 'testTable', updatedAt: Date.now(), + onToggleColumn: () => {}, + onResetColumns: () => {}, + onColumnsChange: () => {}, + onChangeVisibleColumns: () => {}, + browserFields: {}, }; const AlertsTableWithLocale: React.FunctionComponent = (props) => ( diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table.tsx index 302392674ccdd..76804970ebee5 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table.tsx @@ -44,7 +44,6 @@ const AlertsTable: React.FunctionComponent = (props: AlertsTab alerts, alertsCount, isLoading, - onColumnsChange, onPageChange, onSortChange, sort: sortingFields, @@ -66,18 +65,6 @@ const AlertsTable: React.FunctionComponent = (props: AlertsTab useBulkActionsConfig: props.alertsTableConfiguration.useBulkActions, }); - const toolbarVisibility = useCallback(() => { - const { rowSelection } = bulkActionsState; - return getToolbarVisibility({ - bulkActions, - alertsCount, - rowSelection, - alerts: alertsData.alerts, - updatedAt: props.updatedAt, - isLoading, - }); - }, [bulkActionsState, bulkActions, alertsCount, alertsData.alerts, props.updatedAt, isLoading])(); - const { pagination, onChangePageSize, @@ -91,7 +78,14 @@ const AlertsTable: React.FunctionComponent = (props: AlertsTab pageSize: props.pageSize, }); - const [visibleColumns, setVisibleColumns] = useState(props.visibleColumns); + const { + visibleColumns, + onToggleColumn, + onResetColumns, + updatedAt, + browserFields, + onChangeVisibleColumns, + } = props; // TODO when every solution is using this table, we will be able to simplify it by just passing the alert index const handleFlyoutAlert = useCallback( @@ -104,16 +98,32 @@ const AlertsTable: React.FunctionComponent = (props: AlertsTab [alerts, setFlyoutAlertIndex] ); - const onChangeVisibleColumns = useCallback( - (newColumns: string[]) => { - setVisibleColumns(newColumns); - onColumnsChange( - props.columns.sort((a, b) => newColumns.indexOf(a.id) - newColumns.indexOf(b.id)), - newColumns - ); - }, - [onColumnsChange, props.columns] - ); + const toolbarVisibility = useCallback(() => { + const { rowSelection } = bulkActionsState; + return getToolbarVisibility({ + bulkActions, + alertsCount, + rowSelection, + alerts: alertsData.alerts, + updatedAt, + isLoading, + columnIds: visibleColumns, + onToggleColumn, + onResetColumns, + browserFields, + }); + }, [ + bulkActionsState, + bulkActions, + alertsCount, + alertsData.alerts, + updatedAt, + browserFields, + isLoading, + visibleColumns, + onToggleColumn, + onResetColumns, + ])(); const leadingControlColumns = useMemo(() => { const isActionButtonsColumnActive = @@ -203,7 +213,10 @@ const AlertsTable: React.FunctionComponent = (props: AlertsTab columnId: string; }) => { const value = data.find((d) => d.field === columnId)?.value ?? []; - return <>{value.length ? value.join() : '--'}; + if (Array.isArray(value)) { + return <>{value.length ? value.join() : '--'}; + } + return <>{value}; }; const renderCellValue = useCallback( diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table_state.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table_state.test.tsx index da9522d2db52c..d76644a28ddc6 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table_state.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table_state.test.tsx @@ -21,10 +21,12 @@ import { PLUGIN_ID } from '../../../common/constants'; import { TypeRegistry } from '../../type_registry'; import AlertsTableState, { AlertsTableStateProps } from './alerts_table_state'; import { useFetchAlerts } from './hooks/use_fetch_alerts'; +import { useFetchBrowserFieldCapabilities } from './hooks/use_fetch_browser_fields_capabilities'; import { DefaultSort } from './hooks'; import { __IntlProvider as IntlProvider } from '@kbn/i18n-react'; jest.mock('./hooks/use_fetch_alerts'); +jest.mock('./hooks/use_fetch_browser_fields_capabilities'); jest.mock('@kbn/kibana-utils-plugin/public'); jest.mock('@kbn/kibana-react-plugin/public', () => ({ useKibana: () => ({ @@ -55,6 +57,11 @@ jest.mock('@kbn/kibana-react-plugin/public', () => ({ }), }, }, + notifications: { + toasts: { + addDanger: () => {}, + }, + }, }, }), })); @@ -137,6 +144,9 @@ hookUseFetchAlerts.mockImplementation(() => [ }, ]); +const hookUseFetchBrowserFieldCapabilities = useFetchBrowserFieldCapabilities as jest.Mock; +hookUseFetchBrowserFieldCapabilities.mockImplementation(() => [false, {}]); + const AlertsTableWithLocale: React.FunctionComponent = (props) => ( diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table_state.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table_state.tsx index 27aa81d7ba47c..b99568cfb6bce 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table_state.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/alerts_table_state.tsx @@ -35,6 +35,7 @@ import { ALERTS_TABLE_CONF_ERROR_MESSAGE, ALERTS_TABLE_CONF_ERROR_TITLE } from ' import { TypeRegistry } from '../../type_registry'; import { bulkActionsReducer } from './bulk_actions/reducer'; import { useGetUserCasesPermissions } from './hooks/use_get_user_cases_permissions'; +import { useColumns } from './hooks/use_columns'; const DefaultPagination = { pageSize: 10, @@ -59,7 +60,7 @@ export interface AlertsTableStateProps { showExpandToDetails: boolean; } -interface AlertsTableStorage { +export interface AlertsTableStorage { columns: EuiDataGridColumn[]; visibleColumns?: string[]; sort: SortCombinations[]; @@ -93,6 +94,7 @@ const AlertsTableWithBulkActionsContextComponent: React.FunctionComponent<{ ); const AlertsTableWithBulkActionsContext = React.memo(AlertsTableWithBulkActionsContextComponent); +const EMPTY_FIELDS = [{ field: '*', include_unmapped: true }]; const AlertsTableState = ({ alertsTableConfigurationRegistry, @@ -106,6 +108,7 @@ const AlertsTableState = ({ showExpandToDetails, }: AlertsTableStateProps) => { const { cases } = useKibana<{ cases: CaseUi }>().services; + const hasAlertsTableConfiguration = alertsTableConfigurationRegistry?.has(configurationId) ?? false; const alertsTableConfiguration = hasAlertsTableConfiguration @@ -143,7 +146,23 @@ const AlertsTableState = ({ ...DefaultPagination, pageSize: pageSize ?? DefaultPagination.pageSize, }); - const [columns, setColumns] = useState(storageAlertsTable.current.columns); + + const { + columns, + onColumnsChange, + browserFields, + isBrowserFieldDataLoading, + onToggleColumn, + onResetColumns, + visibleColumns, + onChangeVisibleColumns, + } = useColumns({ + featureIds, + storageAlertsTable, + storage, + id, + defaultColumns: (alertsTableConfiguration && alertsTableConfiguration.columns) ?? [], + }); const [ isLoading, @@ -156,7 +175,7 @@ const AlertsTableState = ({ updatedAt, }, ] = useFetchAlerts({ - fields: columns.map((col) => ({ field: col.id, include_unmapped: true })), + fields: EMPTY_FIELDS, featureIds, query, pagination, @@ -194,18 +213,6 @@ const AlertsTableState = ({ }, [id] ); - const onColumnsChange = useCallback( - (newColumns: EuiDataGridColumn[], visibleColumns: string[]) => { - setColumns(newColumns); - storageAlertsTable.current = { - ...storageAlertsTable.current, - columns: newColumns, - visibleColumns, - }; - storage.current.set(id, storageAlertsTable.current); - }, - [id, storage] - ); const useFetchAlertsData = useCallback(() => { return { @@ -215,7 +222,6 @@ const AlertsTableState = ({ isInitializing, isLoading, getInspectQuery, - onColumnsChange, onPageChange, onSortChange, refresh, @@ -228,7 +234,6 @@ const AlertsTableState = ({ getInspectQuery, isInitializing, isLoading, - onColumnsChange, onPageChange, onSortChange, pagination.pageIndex, @@ -252,9 +257,14 @@ const AlertsTableState = ({ showExpandToDetails, trailingControlColumns: [], useFetchAlertsData, - visibleColumns: storageAlertsTable.current.visibleColumns ?? [], + visibleColumns, 'data-test-subj': 'internalAlertsState', updatedAt, + browserFields, + onToggleColumn, + onResetColumns, + onColumnsChange, + onChangeVisibleColumns, }), [ alertsTableConfiguration, @@ -264,7 +274,13 @@ const AlertsTableState = ({ id, showExpandToDetails, useFetchAlertsData, + visibleColumns, updatedAt, + browserFields, + onToggleColumn, + onResetColumns, + onColumnsChange, + onChangeVisibleColumns, ] ); @@ -281,7 +297,7 @@ const AlertsTableState = ({ return hasAlertsTableConfiguration ? ( <> {!isLoading && alertsCount === 0 && } - {isLoading && ( + {(isLoading || isBrowserFieldDataLoading) && ( )} {alertsCount !== 0 && CasesContext && cases && ( diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/bulk_actions/bulk_actions.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/bulk_actions/bulk_actions.test.tsx index 6effedbd32ef4..43743d09d381a 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/bulk_actions/bulk_actions.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/bulk_actions/bulk_actions.test.tsx @@ -96,6 +96,11 @@ describe('AlertsTable.BulkActions', () => { visibleColumns: columns.map((c) => c.id), 'data-test-subj': 'testTable', updatedAt: Date.now(), + onToggleColumn: () => {}, + onResetColumns: () => {}, + onColumnsChange: () => {}, + onChangeVisibleColumns: () => {}, + browserFields: {}, }; const tablePropsWithBulkActions = { diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/translations.ts b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/translations.ts index 88c7eb6ad0a67..529137b1cfeec 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/translations.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/translations.ts @@ -13,3 +13,10 @@ export const ERROR_FETCH_ALERTS = i18n.translate( defaultMessage: `An error has occurred on alerts search`, } ); + +export const ERROR_FETCH_BROWSER_FIELDS = i18n.translate( + 'xpack.triggersActionsUI.components.alertTable.useFetchBrowserFieldsCapabilities.errorMessageText', + { + defaultMessage: 'An error has occurred loading browser fields', + } +); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/use_columns.ts b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/use_columns.ts new file mode 100644 index 0000000000000..a23fc758afeed --- /dev/null +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/use_columns.ts @@ -0,0 +1,235 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiDataGridColumn } from '@elastic/eui'; +import { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; +import { AlertConsumers } from '@kbn/rule-data-utils'; +import { BrowserField, BrowserFields } from '@kbn/rule-registry-plugin/common'; +import { useCallback, useEffect, useState } from 'react'; +import { AlertsTableStorage } from '../alerts_table_state'; +import { useFetchBrowserFieldCapabilities } from './use_fetch_browser_fields_capabilities'; + +interface UseColumnsArgs { + featureIds: AlertConsumers[]; + storageAlertsTable: React.MutableRefObject; + storage: React.MutableRefObject; + id: string; + defaultColumns: EuiDataGridColumn[]; +} + +const fieldTypeToDataGridColumnTypeMapper = (fieldType: string | undefined) => { + if (fieldType === 'date') return 'datetime'; + if (fieldType === 'number') return 'numeric'; + if (fieldType === 'object') return 'json'; + return fieldType; +}; + +/** + * EUI Data Grid expects the columns to have a property 'schema' defined for proper sorting + * this schema as its own types as can be check out in the docs so we add it here manually + * https://eui.elastic.co/#/tabular-content/data-grid-schema-columns + */ +const euiColumnFactory = ( + column: EuiDataGridColumn, + browserFields: BrowserFields +): EuiDataGridColumn => { + const browserFieldsProps = getBrowserFieldProps(column.id, browserFields); + return { + ...column, + schema: fieldTypeToDataGridColumnTypeMapper(browserFieldsProps.type), + }; +}; + +/** + * Searches in browser fields object for a specific field + */ +const getBrowserFieldProps = ( + columnId: string, + browserFields: BrowserFields +): Partial => { + for (const [, categoryDescriptor] of Object.entries(browserFields)) { + if (!categoryDescriptor.fields) { + continue; + } + + for (const [fieldName, fieldDescriptor] of Object.entries(categoryDescriptor.fields)) { + if (fieldName === columnId) { + return fieldDescriptor; + } + } + } + return { type: 'string' }; +}; + +/** + * @param columns Columns to be considered in the alerts table + * @param browserFields constant object with all field capabilities + * @returns columns but with the info needed by the data grid to work as expected, e.g sorting + */ +const populateColumns = ( + columns: EuiDataGridColumn[], + browserFields: BrowserFields +): EuiDataGridColumn[] => { + return columns.map((column: EuiDataGridColumn) => { + return euiColumnFactory(column, browserFields); + }); +}; + +const getColumnByColumnId = (columns: EuiDataGridColumn[], columnId: string) => { + return columns.find(({ id }: { id: string }) => id === columnId); +}; + +const persist = ({ + id, + storageAlertsTable, + columns, + visibleColumns, + storage, +}: { + id: string; + storageAlertsTable: React.MutableRefObject; + storage: React.MutableRefObject; + columns: EuiDataGridColumn[]; + visibleColumns: string[]; +}) => { + storageAlertsTable.current = { + ...storageAlertsTable.current, + columns, + visibleColumns, + }; + storage.current.set(id, storageAlertsTable.current); +}; + +export const useColumns = ({ + featureIds, + storageAlertsTable, + storage, + id, + defaultColumns, +}: UseColumnsArgs) => { + const [isBrowserFieldDataLoading, browserFields] = useFetchBrowserFieldCapabilities({ + featureIds, + }); + const [columns, setColumns] = useState(storageAlertsTable.current.columns); + const [isColumnsPopulated, setColumnsPopulated] = useState(false); + const [visibleColumns, setVisibleColumns] = useState( + storageAlertsTable.current.visibleColumns ?? [] + ); + + useEffect(() => { + if (isBrowserFieldDataLoading !== false || isColumnsPopulated) return; + + const populatedColumns = populateColumns(columns, browserFields); + setColumnsPopulated(true); + setColumns(populatedColumns); + }, [browserFields, columns, isBrowserFieldDataLoading, isColumnsPopulated]); + + const onColumnsChange = useCallback( + (newColumns: EuiDataGridColumn[], newVisibleColumns: string[]) => { + setColumns(newColumns); + persist({ + id, + storage, + storageAlertsTable, + columns: newColumns, + visibleColumns: newVisibleColumns, + }); + }, + [id, storage, storageAlertsTable] + ); + + const onChangeVisibleColumns = useCallback( + (newColumns: string[]) => { + setVisibleColumns(newColumns); + onColumnsChange( + columns.sort((a, b) => newColumns.indexOf(a.id) - newColumns.indexOf(b.id)), + newColumns + ); + }, + [onColumnsChange, columns] + ); + + const onToggleColumn = useCallback( + (columnId: string): void => { + const visibleIndex = visibleColumns.indexOf(columnId); + const defaultIndex = defaultColumns.findIndex( + (column: EuiDataGridColumn) => column.id === columnId + ); + + const isVisible = visibleIndex >= 0; + const isInDefaultConfig = defaultIndex >= 0; + + let newColumnIds: string[] = []; + + // if the column is shown, remove it + if (isVisible) { + newColumnIds = [ + ...visibleColumns.slice(0, visibleIndex), + ...visibleColumns.slice(visibleIndex + 1), + ]; + } + + // if the column isn't shown but it's part of the default config + // insert into the same position as in the default config + if (!isVisible && isInDefaultConfig) { + newColumnIds = [ + ...visibleColumns.slice(0, defaultIndex), + columnId, + ...visibleColumns.slice(defaultIndex), + ]; + } + + // if the column isn't shown and it's not part of the default config + // push it into the second position. Behaviour copied by t_grid, security + // does this to insert right after the timestamp column + if (!isVisible && !isInDefaultConfig) { + newColumnIds = [visibleColumns[0], columnId, ...visibleColumns.slice(1)]; + } + + const newColumns = newColumnIds.map((_columnId) => { + const column = getColumnByColumnId(defaultColumns, _columnId); + return euiColumnFactory(column ? column : { id: _columnId }, browserFields); + }); + + setVisibleColumns(newColumnIds); + setColumns(newColumns); + persist({ + id, + storage, + storageAlertsTable, + columns: newColumns, + visibleColumns: newColumnIds, + }); + }, + [browserFields, defaultColumns, id, storage, storageAlertsTable, visibleColumns] + ); + + const onResetColumns = useCallback(() => { + const newVisibleColumns = defaultColumns.map((column) => column.id); + const populatedDefaultColumns = populateColumns(defaultColumns, browserFields); + setVisibleColumns(newVisibleColumns); + setColumns(populatedDefaultColumns); + persist({ + id, + storage, + storageAlertsTable, + columns: populatedDefaultColumns, + visibleColumns: newVisibleColumns, + }); + }, [browserFields, defaultColumns, id, storage, storageAlertsTable]); + + return { + columns, + isBrowserFieldDataLoading, + browserFields, + visibleColumns, + onColumnsChange, + onToggleColumn, + onResetColumns, + onChangeVisibleColumns, + }; +}; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/use_fetch_browser_fields_capabilities.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/use_fetch_browser_fields_capabilities.tsx new file mode 100644 index 0000000000000..3a07a7f1cf3be --- /dev/null +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/hooks/use_fetch_browser_fields_capabilities.tsx @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ValidFeatureId } from '@kbn/rule-data-utils'; +import type { EcsFieldsResponse } from '@kbn/rule-registry-plugin/common/search_strategy'; +import { BASE_RAC_ALERTS_API_PATH, BrowserFields } from '@kbn/rule-registry-plugin/common'; +import { useCallback, useEffect, useState } from 'react'; +import { useKibana } from '../../../../common/lib/kibana'; +import { ERROR_FETCH_BROWSER_FIELDS } from './translations'; + +export interface FetchAlertsArgs { + featureIds: ValidFeatureId[]; +} + +export interface FetchAlertResp { + alerts: EcsFieldsResponse[]; +} + +export type UseFetchAlerts = ({ featureIds }: FetchAlertsArgs) => [boolean, FetchAlertResp]; + +const INVALID_FEATURE_ID = 'siem'; + +export const useFetchBrowserFieldCapabilities = ({ + featureIds, +}: FetchAlertsArgs): [boolean | undefined, BrowserFields] => { + const { + http, + notifications: { toasts }, + } = useKibana().services; + + const [isLoading, setIsLoading] = useState(undefined); + const [browserFields, setBrowserFields] = useState(() => ({})); + + const getBrowserFieldInfo = useCallback(async () => { + if (!http) return Promise.resolve({}); + + try { + return await http.get(`${BASE_RAC_ALERTS_API_PATH}/browser_fields`, { + query: { featureIds }, + }); + } catch (e) { + toasts.addDanger(ERROR_FETCH_BROWSER_FIELDS); + return {}; + } + }, [featureIds, http, toasts]); + + useEffect(() => { + if (featureIds.includes(INVALID_FEATURE_ID)) { + setIsLoading(false); + } + }, [featureIds]); + + useEffect(() => { + if (isLoading !== undefined) return; + + setIsLoading(true); + + const callApi = async () => { + const browserFieldsInfo = await getBrowserFieldInfo(); + + setBrowserFields(browserFieldsInfo); + setIsLoading(false); + }; + + callApi(); + }, [getBrowserFieldInfo, isLoading]); + + return [isLoading, browserFields]; +}; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/toolbar/index.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/toolbar/index.tsx index e261dc7d09cd9..41190da81cca9 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/toolbar/index.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/toolbar/index.tsx @@ -5,5 +5,4 @@ * 2.0. */ -export { getToolbarVisibility } from './toolbar_visibility'; -export { AlertsCount } from './components/alerts_count/alerts_count'; +export * from './toolbar_visibility'; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/toolbar/toolbar_visibility.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/toolbar/toolbar_visibility.tsx index bc4c4817ec37c..d51b84aa84aeb 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/toolbar/toolbar_visibility.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_table/toolbar/toolbar_visibility.tsx @@ -8,22 +8,47 @@ import { EuiDataGridToolBarVisibilityOptions } from '@elastic/eui'; import { EcsFieldsResponse } from '@kbn/rule-registry-plugin/common/search_strategy'; import React, { lazy, Suspense } from 'react'; +import { BrowserFields } from '@kbn/rule-registry-plugin/common'; import { AlertsCount } from './components/alerts_count/alerts_count'; import { BulkActionsConfig } from '../../../../types'; import { LastUpdatedAt } from './components/last_updated_at'; +import { FieldBrowser } from '../../field_browser'; const BulkActionsToolbar = lazy(() => import('../bulk_actions/components/toolbar')); const getDefaultVisibility = ({ alertsCount, updatedAt, + columnIds, + onToggleColumn, + onResetColumns, + browserFields, }: { alertsCount: number; updatedAt: number; -}) => { + columnIds: string[]; + onToggleColumn: (columnId: string) => void; + onResetColumns: () => void; + browserFields: BrowserFields; +}): EuiDataGridToolBarVisibilityOptions => { + const hasBrowserFields = Object.keys(browserFields).length > 0; const additionalControls = { right: , - left: { append: }, + left: { + append: ( + <> + + {hasBrowserFields ? ( + + ) : undefined} + + ), + }, }; return { @@ -40,6 +65,10 @@ export const getToolbarVisibility = ({ alerts, isLoading, updatedAt, + columnIds, + onToggleColumn, + onResetColumns, + browserFields, }: { bulkActions: BulkActionsConfig[]; alertsCount: number; @@ -47,18 +76,30 @@ export const getToolbarVisibility = ({ alerts: EcsFieldsResponse[]; isLoading: boolean; updatedAt: number; + columnIds: string[]; + onToggleColumn: (columnId: string) => void; + onResetColumns: () => void; + browserFields: any; }): EuiDataGridToolBarVisibilityOptions => { const selectedRowsCount = rowSelection.size; - const defaultVisibility = getDefaultVisibility({ alertsCount, updatedAt }); + const defaultVisibility = getDefaultVisibility({ + alertsCount, + updatedAt, + columnIds, + onToggleColumn, + onResetColumns, + browserFields, + }); + const isBulkActionsActive = + selectedRowsCount === 0 || selectedRowsCount === undefined || bulkActions.length === 0; - if (selectedRowsCount === 0 || selectedRowsCount === undefined || bulkActions.length === 0) - return defaultVisibility; + if (isBulkActionsActive) return defaultVisibility; const options = { showColumnSelector: false, showSortSelector: false, additionalControls: { - ...defaultVisibility.additionalControls, + right: , left: { append: ( <> diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/categories_selector/categories_selector.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/categories_selector/categories_selector.tsx index 42e62e418beec..c87bec9684678 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/categories_selector/categories_selector.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/categories_selector/categories_selector.tsx @@ -17,7 +17,7 @@ import { EuiSelectable, FilterChecked, } from '@elastic/eui'; -import type { BrowserFields } from '../../types'; +import { BrowserFields } from '@kbn/rule-registry-plugin/common'; import * as i18n from '../../translations'; import { getFieldCount, isEscape } from '../../helpers'; import { styles } from './categories_selector.styles'; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_items/field_items.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_items/field_items.test.tsx index d098486c166e2..8be1964647f7f 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_items/field_items.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_items/field_items.test.tsx @@ -127,12 +127,6 @@ describe('field_items', () => { sortable: true, width: '225px', }, - { - field: 'description', - name: 'Description', - sortable: true, - width: '400px', - }, { field: 'category', name: 'Category', @@ -188,12 +182,10 @@ describe('field_items', () => { ); expect(getAllByText('Name').at(0)).toBeInTheDocument(); - expect(getAllByText('Description').at(0)).toBeInTheDocument(); expect(getAllByText('Category').at(0)).toBeInTheDocument(); expect(getByTestId(`field-${timestampFieldId}-checkbox`)).toBeInTheDocument(); expect(getByTestId(`field-${timestampFieldId}-name`)).toBeInTheDocument(); - expect(getByTestId(`field-${timestampFieldId}-description`)).toBeInTheDocument(); expect(getByTestId(`field-${timestampFieldId}-category`)).toBeInTheDocument(); }); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_items/field_items.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_items/field_items.tsx index 3210af448a936..fa6b0d5e7b442 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_items/field_items.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_items/field_items.tsx @@ -11,20 +11,15 @@ import { EuiToolTip, EuiFlexGroup, EuiFlexItem, - EuiScreenReaderOnly, EuiBadge, EuiBasicTableColumn, EuiTableActionsColumnType, } from '@elastic/eui'; import { uniqBy } from 'lodash/fp'; -import { getEmptyValue, getExampleText, getIconFromType } from '../../helpers'; -import type { - BrowserFields, - BrowserFieldItem, - FieldTableColumns, - GetFieldTableColumns, -} from '../../types'; +import { BrowserFields } from '@kbn/rule-registry-plugin/common'; +import { getIconFromType } from '../../helpers'; +import type { BrowserFieldItem, FieldTableColumns, GetFieldTableColumns } from '../../types'; import { FieldName } from '../field_name'; import * as i18n from '../../translations'; import { styles } from './field_items.style'; @@ -93,26 +88,6 @@ const getDefaultFieldTableColumns = (highlight: string): FieldTableColumns => [ sortable: true, width: '225px', }, - { - field: 'description', - name: i18n.DESCRIPTION, - render: (description: string, { name, example }) => ( - - <> - -

{i18n.DESCRIPTION_FOR_FIELD(name)}

- - - - {`${description ?? getEmptyValue()} ${getExampleText(example)}`} - - - - - ), - sortable: true, - width: '400px', - }, { field: 'category', name: i18n.CATEGORY, diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_table/field_table.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_table/field_table.tsx index 638c614a54e88..b4c6ec91b48bb 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_table/field_table.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/components/field_table/field_table.tsx @@ -6,9 +6,10 @@ */ import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { EuiInMemoryTable, Pagination, Direction, useEuiTheme } from '@elastic/eui'; +import { BrowserFields } from '@kbn/rule-registry-plugin/common'; import { getFieldColumns, getFieldItems, isActionsColumn } from '../field_items'; import { CATEGORY_TABLE_CLASS_NAME, TABLE_HEIGHT } from '../../helpers'; -import type { BrowserFields, FieldBrowserProps, GetFieldTableColumns } from '../../types'; +import type { FieldBrowserProps, GetFieldTableColumns } from '../../types'; import { FieldTableHeader } from './field_table_header'; import { styles } from './field_table.styles'; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/field_browser.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/field_browser.tsx index e90d8bdb5e602..219a06713e7c3 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/field_browser.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/field_browser.tsx @@ -8,7 +8,8 @@ import { EuiButtonEmpty, EuiToolTip } from '@elastic/eui'; import { debounce } from 'lodash'; import React, { useEffect, useRef, useState, useCallback, useMemo } from 'react'; -import type { FieldBrowserProps, BrowserFields } from './types'; +import { BrowserFields } from '@kbn/rule-registry-plugin/common'; +import type { FieldBrowserProps } from './types'; import { FieldBrowserModal } from './field_browser_modal'; import { filterBrowserFieldsByFieldName, filterSelectedBrowserFields } from './helpers'; import * as i18n from './translations'; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/field_browser_modal.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/field_browser_modal.tsx index 84e10ad88cc5e..2a6a8634ba666 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/field_browser_modal.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/field_browser_modal.tsx @@ -18,7 +18,8 @@ import { } from '@elastic/eui'; import React, { useCallback } from 'react'; -import type { FieldBrowserProps, BrowserFields } from './types'; +import { BrowserFields } from '@kbn/rule-registry-plugin/common'; +import type { FieldBrowserProps } from './types'; import { Search } from './components/search'; import { CLOSE_BUTTON_CLASS_NAME, FIELD_BROWSER_WIDTH, RESET_FIELDS_CLASS_NAME } from './helpers'; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/helpers.test.ts b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/helpers.test.ts index 3bfa39ae3d748..86a49b0613d5d 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/helpers.test.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/helpers.test.ts @@ -13,7 +13,7 @@ import { filterBrowserFieldsByFieldName, filterSelectedBrowserFields, } from './helpers'; -import type { BrowserFields } from './types'; +import { BrowserFields } from '@kbn/rule-registry-plugin/common'; describe('helpers', () => { describe('categoryHasFields', () => { diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/helpers.ts b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/helpers.ts index 006fb5f8e5014..ebbdeec76a154 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/helpers.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/helpers.ts @@ -5,8 +5,8 @@ * 2.0. */ +import { BrowserField, BrowserFields } from '@kbn/rule-registry-plugin/common'; import { isEmpty } from 'lodash/fp'; -import { BrowserField, BrowserFields } from './types'; export const FIELD_BROWSER_WIDTH = 925; export const TABLE_HEIGHT = 260; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/mock.ts b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/mock.ts index 7515f1ce99ca3..2192c795c7099 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/mock.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/mock.ts @@ -6,7 +6,7 @@ */ import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { BrowserFields } from './types'; +import { BrowserFields } from '@kbn/rule-registry-plugin/common'; const DEFAULT_INDEX_PATTERN = [ 'apm-*-transaction*', diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/types.ts b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/types.ts index de117ccaccb5e..caed72a14b0c2 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/types.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/field_browser/types.ts @@ -6,26 +6,8 @@ */ import type { EuiBasicTableColumn } from '@elastic/eui'; -import type { IFieldSubType } from '@kbn/es-query'; -import type { RuntimeField } from '@kbn/data-views-plugin/common'; +import { BrowserFields } from '@kbn/rule-registry-plugin/common'; -export interface BrowserField { - aggregatable: boolean; - category: string; - description: string | null; - example: string | number | null; - fields: Readonly>>; - format: string; - indexes: string[]; - name: string; - searchable: boolean; - type: string; - subType?: IFieldSubType; - readFromDocValues: boolean; - runtimeField?: RuntimeField; -} - -export type BrowserFields = Readonly>>; /** * An item rendered in the table */ diff --git a/x-pack/plugins/triggers_actions_ui/public/types.ts b/x-pack/plugins/triggers_actions_ui/public/types.ts index b57c1fb706efe..ae827a5b33f76 100644 --- a/x-pack/plugins/triggers_actions_ui/public/types.ts +++ b/x-pack/plugins/triggers_actions_ui/public/types.ts @@ -410,7 +410,6 @@ export interface FetchAlertData { isInitializing: boolean; isLoading: boolean; getInspectQuery: () => { request: {}; response: {} }; - onColumnsChange: (columns: EuiDataGridColumn[], visibleColumns: string[]) => void; onPageChange: (pagination: RuleRegistrySearchRequestPagination) => void; onSortChange: (sort: EuiDataGridSorting['columns']) => void; refresh: () => void; @@ -434,6 +433,11 @@ export interface AlertsTableProps { visibleColumns: string[]; 'data-test-subj': string; updatedAt: number; + browserFields: any; + onToggleColumn: (columnId: string) => void; + onResetColumns: () => void; + onColumnsChange: (columns: EuiDataGridColumn[], visibleColumns: string[]) => void; + onChangeVisibleColumns: (newColumns: string[]) => void; } // TODO We need to create generic type between our plugin, right now we have different one because of the old alerts table diff --git a/x-pack/test/examples/triggers_actions_ui_examples/alerts_table.ts b/x-pack/test/examples/triggers_actions_ui_examples/alerts_table.ts index 35c32061a1ddf..3351dfd0064b5 100644 --- a/x-pack/test/examples/triggers_actions_ui_examples/alerts_table.ts +++ b/x-pack/test/examples/triggers_actions_ui_examples/alerts_table.ts @@ -46,6 +46,32 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { ); }); + it('should let the user choose between fields', async () => { + await waitAndClickByTestId('show-field-browser'); + await waitAndClickByTestId('categories-filter-button'); + await waitAndClickByTestId('categories-selector-option-name-base'); + await find.clickByCssSelector('#_id'); + await waitAndClickByTestId('close'); + + const headers = await find.allByCssSelector('.euiDataGridHeaderCell'); + expect(headers.length).to.be(6); + }); + + it('should take into account the column type when sorting', async () => { + const sortElQuery = + '[data-test-subj="dataGridHeaderCellActionGroup-kibana.alert.duration.us"] > li:nth-child(2)'; + + await waitAndClickByTestId('dataGridHeaderCell-kibana.alert.duration.us'); + + await retry.try(async () => { + const exists = await find.byCssSelector(sortElQuery); + if (!exists) throw new Error('Still loading...'); + }); + + const sortItem = await find.byCssSelector(sortElQuery); + expect(await sortItem.getVisibleText()).to.be('Sort Low-High'); + }); + it('should sort properly', async () => { await find.clickDisplayedByCssSelector( '[data-test-subj="dataGridHeaderCell-event.action"] .euiDataGridHeaderCell__button' @@ -161,4 +187,13 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { return rows; } }); + + const waitAndClickByTestId = async (testId: string) => { + retry.try(async () => { + const exists = await testSubjects.exists(testId); + if (!exists) throw new Error('Still loading...'); + }); + + return find.clickDisplayedByCssSelector(`[data-test-subj="${testId}"]`); + }; } diff --git a/x-pack/test/observability_functional/apps/observability/pages/alerts/table_storage.ts b/x-pack/test/observability_functional/apps/observability/pages/alerts/table_storage.ts index 99acb4fef2ca7..e091f8ddeb7bf 100644 --- a/x-pack/test/observability_functional/apps/observability/pages/alerts/table_storage.ts +++ b/x-pack/test/observability_functional/apps/observability/pages/alerts/table_storage.ts @@ -49,8 +49,7 @@ export default ({ getService, getPageObject }: FtrProviderContext) => { expect(durationColumnExists).to.be(false); }); - // TODO Enable this test after fixing: https://github.com/elastic/kibana/issues/137988 - it.skip('remembers sorting changes', async () => { + it('remembers sorting changes', async () => { const timestampColumnButton = await testSubjects.find( 'dataGridHeaderCellActionButton-@timestamp' ); From d29521e897e6cc9fd2a8c98851aa3d8970373c4d Mon Sep 17 00:00:00 2001 From: Kevin Delemme Date: Tue, 20 Sep 2022 10:34:48 -0400 Subject: [PATCH 05/76] feat(slo): Delete SLO (#140760) --- .../observability/server/assets/constants.ts | 2 + .../observability/server/errors/errors.ts | 17 ++ .../observability/server/errors/handler.ts | 16 ++ .../observability/server/errors/index.ts | 9 + .../server/routes/register_routes.ts | 25 ++- .../observability/server/routes/slo/route.ts | 43 ++++- .../server/services/slo/create_slo.test.ts | 47 ++--- .../server/services/slo/create_slo.ts | 22 ++- .../server/services/slo/delete_slo.test.ts | 51 +++++ .../server/services/slo/delete_slo.ts | 43 +++++ .../server/services/slo/index.ts | 3 +- .../server/services/slo/mocks/index.ts | 11 +- .../services/slo/resource_installer.test.ts | 12 +- .../server/services/slo/resource_installer.ts | 12 +- .../services/slo/slo_repository.test.ts | 65 ++++--- .../server/services/slo/slo_repository.ts | 36 +++- .../apm_transaction_duration.ts | 8 +- .../apm_transaction_error_rate.ts | 8 +- .../services/slo/transform_installer.test.ts | 102 ---------- .../services/slo/transform_installer.ts | 56 ------ .../services/slo/transform_manager.test.ts | 174 ++++++++++++++++++ .../server/services/slo/transform_manager.ts | 83 +++++++++ .../observability/server/types/schema/slo.ts | 6 + 23 files changed, 606 insertions(+), 245 deletions(-) create mode 100644 x-pack/plugins/observability/server/errors/errors.ts create mode 100644 x-pack/plugins/observability/server/errors/handler.ts create mode 100644 x-pack/plugins/observability/server/errors/index.ts create mode 100644 x-pack/plugins/observability/server/services/slo/delete_slo.test.ts create mode 100644 x-pack/plugins/observability/server/services/slo/delete_slo.ts delete mode 100644 x-pack/plugins/observability/server/services/slo/transform_installer.test.ts delete mode 100644 x-pack/plugins/observability/server/services/slo/transform_installer.ts create mode 100644 x-pack/plugins/observability/server/services/slo/transform_manager.test.ts create mode 100644 x-pack/plugins/observability/server/services/slo/transform_manager.ts diff --git a/x-pack/plugins/observability/server/assets/constants.ts b/x-pack/plugins/observability/server/assets/constants.ts index 8afa22d5f695e..ea96f7a3b5763 100644 --- a/x-pack/plugins/observability/server/assets/constants.ts +++ b/x-pack/plugins/observability/server/assets/constants.ts @@ -13,3 +13,5 @@ export const SLO_RESOURCES_VERSION = 1; export const getSLODestinationIndexName = (spaceId: string) => `${SLO_INDEX_TEMPLATE_NAME}-v${SLO_RESOURCES_VERSION}-${spaceId}`; + +export const getSLOTransformId = (sloId: string) => `slo-${sloId}`; diff --git a/x-pack/plugins/observability/server/errors/errors.ts b/x-pack/plugins/observability/server/errors/errors.ts new file mode 100644 index 0000000000000..2e00a8cc22bfe --- /dev/null +++ b/x-pack/plugins/observability/server/errors/errors.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable max-classes-per-file */ + +export class ObservabilityError extends Error { + constructor(message?: string) { + super(message); + this.name = this.constructor.name; + } +} + +export class SLONotFound extends ObservabilityError {} diff --git a/x-pack/plugins/observability/server/errors/handler.ts b/x-pack/plugins/observability/server/errors/handler.ts new file mode 100644 index 0000000000000..9ce2dd8b37965 --- /dev/null +++ b/x-pack/plugins/observability/server/errors/handler.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ObservabilityError, SLONotFound } from './errors'; + +export function getHTTPResponseCode(error: ObservabilityError): number { + if (error instanceof SLONotFound) { + return 404; + } + + return 400; +} diff --git a/x-pack/plugins/observability/server/errors/index.ts b/x-pack/plugins/observability/server/errors/index.ts new file mode 100644 index 0000000000000..e466d5b8ae4a1 --- /dev/null +++ b/x-pack/plugins/observability/server/errors/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './errors'; +export * from './handler'; diff --git a/x-pack/plugins/observability/server/routes/register_routes.ts b/x-pack/plugins/observability/server/routes/register_routes.ts index 654c50042148c..1a1fdf220afb2 100644 --- a/x-pack/plugins/observability/server/routes/register_routes.ts +++ b/x-pack/plugins/observability/server/routes/register_routes.ts @@ -17,6 +17,7 @@ import { RuleDataPluginService } from '@kbn/rule-registry-plugin/server'; import { SpacesServiceStart } from '@kbn/spaces-plugin/server'; import { ObservabilityRequestHandlerContext } from '../types'; import { AbstractObservabilityServerRouteRepository } from './types'; +import { getHTTPResponseCode, ObservabilityError } from '../errors'; export function registerRoutes({ repository, @@ -71,6 +72,24 @@ export function registerRoutes({ return response.ok({ body: data }); } catch (error) { + if (error instanceof ObservabilityError) { + logger.error(error.message); + return response.customError({ + statusCode: getHTTPResponseCode(error), + body: { + message: error.message, + }, + }); + } + + if (Boom.isBoom(error)) { + logger.error(error.output.payload.message); + return response.customError({ + statusCode: error.output.statusCode, + body: { message: error.output.payload.message }, + }); + } + logger.error(error); const opts = { statusCode: 500, @@ -79,16 +98,12 @@ export function registerRoutes({ }, }; - if (Boom.isBoom(error)) { - opts.statusCode = error.output.statusCode; - } - if (error instanceof errors.RequestAbortedError) { opts.statusCode = 499; opts.body.message = 'Client closed request'; } - return response.custom(opts); + return response.customError(opts); } } ); diff --git a/x-pack/plugins/observability/server/routes/slo/route.ts b/x-pack/plugins/observability/server/routes/slo/route.ts index 77b87222a60a8..7dfdbc0f29f3c 100644 --- a/x-pack/plugins/observability/server/routes/slo/route.ts +++ b/x-pack/plugins/observability/server/routes/slo/route.ts @@ -7,8 +7,9 @@ import { CreateSLO, + DeleteSLO, DefaultResourceInstaller, - DefaultTransformInstaller, + DefaultTransformManager, KibanaSavedObjectsSLORepository, } from '../../services/slo'; import { @@ -17,7 +18,7 @@ import { TransformGenerator, } from '../../services/slo/transform_generators'; import { SLITypes } from '../../types/models'; -import { createSLOParamsSchema } from '../../types/schema'; +import { createSLOParamsSchema, deleteSLOParamsSchema } from '../../types/schema'; import { createObservabilityServerRoute } from '../create_observability_server_route'; const transformGenerators: Record = { @@ -36,10 +37,15 @@ const createSLORoute = createObservabilityServerRoute({ const soClient = (await context.core).savedObjects.client; const spaceId = spacesService.getSpaceId(request); - const resourceInstaller = new DefaultResourceInstaller(esClient, logger); + const resourceInstaller = new DefaultResourceInstaller(esClient, logger, spaceId); const repository = new KibanaSavedObjectsSLORepository(soClient); - const transformInstaller = new DefaultTransformInstaller(transformGenerators, esClient, logger); - const createSLO = new CreateSLO(resourceInstaller, repository, transformInstaller, spaceId); + const transformManager = new DefaultTransformManager( + transformGenerators, + esClient, + logger, + spaceId + ); + const createSLO = new CreateSLO(resourceInstaller, repository, transformManager); const response = await createSLO.execute(params.body); @@ -47,4 +53,29 @@ const createSLORoute = createObservabilityServerRoute({ }, }); -export const slosRouteRepository = createSLORoute; +const deleteSLORoute = createObservabilityServerRoute({ + endpoint: 'DELETE /api/observability/slos/{id}', + options: { + tags: [], + }, + params: deleteSLOParamsSchema, + handler: async ({ context, request, params, logger, spacesService }) => { + const esClient = (await context.core).elasticsearch.client.asCurrentUser; + const soClient = (await context.core).savedObjects.client; + const spaceId = spacesService.getSpaceId(request); + + const repository = new KibanaSavedObjectsSLORepository(soClient); + const transformManager = new DefaultTransformManager( + transformGenerators, + esClient, + logger, + spaceId + ); + + const deleteSLO = new DeleteSLO(repository, transformManager, esClient); + + await deleteSLO.execute(params.path.id); + }, +}); + +export const slosRouteRepository = { ...createSLORoute, ...deleteSLORoute }; diff --git a/x-pack/plugins/observability/server/services/slo/create_slo.test.ts b/x-pack/plugins/observability/server/services/slo/create_slo.test.ts index 69be1c1fa865e..b02afd06571c0 100644 --- a/x-pack/plugins/observability/server/services/slo/create_slo.test.ts +++ b/x-pack/plugins/observability/server/services/slo/create_slo.test.ts @@ -10,57 +10,60 @@ import { createAPMTransactionErrorRateIndicator, createSLOParams } from './fixtu import { createResourceInstallerMock, createSLORepositoryMock, - createTransformInstallerMock, + createTransformManagerMock, } from './mocks'; import { ResourceInstaller } from './resource_installer'; import { SLORepository } from './slo_repository'; -import { TransformInstaller } from './transform_installer'; +import { TransformManager } from './transform_manager'; -const SPACE_ID = 'some-space-id'; - -describe('createSLO', () => { +describe('CreateSLO', () => { let mockResourceInstaller: jest.Mocked; let mockRepository: jest.Mocked; - let mockTransformInstaller: jest.Mocked; + let mockTransformManager: jest.Mocked; let createSLO: CreateSLO; beforeEach(() => { mockResourceInstaller = createResourceInstallerMock(); mockRepository = createSLORepositoryMock(); - mockTransformInstaller = createTransformInstallerMock(); - createSLO = new CreateSLO( - mockResourceInstaller, - mockRepository, - mockTransformInstaller, - SPACE_ID - ); + mockTransformManager = createTransformManagerMock(); + createSLO = new CreateSLO(mockResourceInstaller, mockRepository, mockTransformManager); }); describe('happy path', () => { it('calls the expected services', async () => { const sloParams = createSLOParams(createAPMTransactionErrorRateIndicator()); + mockTransformManager.install.mockResolvedValue('slo-transform-id'); + const response = await createSLO.execute(sloParams); - expect(mockResourceInstaller.ensureCommonResourcesInstalled).toHaveBeenCalledWith(SPACE_ID); + expect(mockResourceInstaller.ensureCommonResourcesInstalled).toHaveBeenCalled(); expect(mockRepository.save).toHaveBeenCalledWith( expect.objectContaining({ ...sloParams, id: expect.any(String) }) ); - expect(mockTransformInstaller.installAndStartTransform).toHaveBeenCalledWith( - expect.objectContaining({ ...sloParams, id: expect.any(String) }), - SPACE_ID + expect(mockTransformManager.install).toHaveBeenCalledWith( + expect.objectContaining({ ...sloParams, id: expect.any(String) }) ); + expect(mockTransformManager.start).toHaveBeenCalledWith('slo-transform-id'); expect(response).toEqual(expect.objectContaining({ id: expect.any(String) })); }); }); describe('unhappy path', () => { - it('deletes the SLO saved objects when transform installation fails', async () => { - mockTransformInstaller.installAndStartTransform.mockRejectedValue( - new Error('Transform Error') - ); + it('deletes the SLO when transform installation fails', async () => { + mockTransformManager.install.mockRejectedValue(new Error('Transform install error')); + const sloParams = createSLOParams(createAPMTransactionErrorRateIndicator()); + + await expect(createSLO.execute(sloParams)).rejects.toThrowError('Transform install error'); + expect(mockRepository.deleteById).toBeCalled(); + }); + + it('removes the transform and deletes the SLO when transform start fails', async () => { + mockTransformManager.install.mockResolvedValue('slo-transform-id'); + mockTransformManager.start.mockRejectedValue(new Error('Transform start error')); const sloParams = createSLOParams(createAPMTransactionErrorRateIndicator()); - await expect(createSLO.execute(sloParams)).rejects.toThrowError('Transform Error'); + await expect(createSLO.execute(sloParams)).rejects.toThrowError('Transform start error'); + expect(mockTransformManager.uninstall).toBeCalledWith('slo-transform-id'); expect(mockRepository.deleteById).toBeCalled(); }); }); diff --git a/x-pack/plugins/observability/server/services/slo/create_slo.ts b/x-pack/plugins/observability/server/services/slo/create_slo.ts index 230955c621760..c90bd9c5c2ccc 100644 --- a/x-pack/plugins/observability/server/services/slo/create_slo.ts +++ b/x-pack/plugins/observability/server/services/slo/create_slo.ts @@ -10,31 +10,41 @@ import uuid from 'uuid'; import { SLO } from '../../types/models'; import { ResourceInstaller } from './resource_installer'; import { SLORepository } from './slo_repository'; -import { TransformInstaller } from './transform_installer'; - +import { TransformManager } from './transform_manager'; import { CreateSLOParams, CreateSLOResponse } from '../../types/schema'; export class CreateSLO { constructor( private resourceInstaller: ResourceInstaller, private repository: SLORepository, - private transformInstaller: TransformInstaller, - private spaceId: string + private transformManager: TransformManager ) {} public async execute(sloParams: CreateSLOParams): Promise { const slo = this.toSLO(sloParams); - await this.resourceInstaller.ensureCommonResourcesInstalled(this.spaceId); + await this.resourceInstaller.ensureCommonResourcesInstalled(); await this.repository.save(slo); + let sloTransformId; try { - await this.transformInstaller.installAndStartTransform(slo, this.spaceId); + sloTransformId = await this.transformManager.install(slo); } catch (err) { await this.repository.deleteById(slo.id); throw err; } + try { + await this.transformManager.start(sloTransformId); + } catch (err) { + await Promise.all([ + this.transformManager.uninstall(sloTransformId), + this.repository.deleteById(slo.id), + ]); + + throw err; + } + return this.toResponse(slo); } diff --git a/x-pack/plugins/observability/server/services/slo/delete_slo.test.ts b/x-pack/plugins/observability/server/services/slo/delete_slo.test.ts new file mode 100644 index 0000000000000..996928eae34b9 --- /dev/null +++ b/x-pack/plugins/observability/server/services/slo/delete_slo.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; +import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { getSLOTransformId } from '../../assets/constants'; +import { DeleteSLO } from './delete_slo'; +import { createAPMTransactionErrorRateIndicator, createSLO } from './fixtures/slo'; +import { createSLORepositoryMock, createTransformManagerMock } from './mocks'; +import { SLORepository } from './slo_repository'; +import { TransformManager } from './transform_manager'; + +describe('DeleteSLO', () => { + let mockRepository: jest.Mocked; + let mockTransformManager: jest.Mocked; + let mockEsClient: jest.Mocked; + let deleteSLO: DeleteSLO; + + beforeEach(() => { + mockRepository = createSLORepositoryMock(); + mockTransformManager = createTransformManagerMock(); + mockEsClient = elasticsearchServiceMock.createElasticsearchClient(); + deleteSLO = new DeleteSLO(mockRepository, mockTransformManager, mockEsClient); + }); + + describe('happy path', () => { + it('removes the transform, the roll up data and the SLO from the repository', async () => { + const slo = createSLO(createAPMTransactionErrorRateIndicator()); + mockRepository.findById.mockResolvedValueOnce(slo); + + await deleteSLO.execute(slo.id); + + expect(mockTransformManager.stop).toHaveBeenCalledWith(getSLOTransformId(slo.id)); + expect(mockTransformManager.uninstall).toHaveBeenCalledWith(getSLOTransformId(slo.id)); + expect(mockEsClient.deleteByQuery).toHaveBeenCalledWith( + expect.objectContaining({ + query: { + match: { + 'slo.id': slo.id, + }, + }, + }) + ); + expect(mockRepository.deleteById).toHaveBeenCalledWith(slo.id); + }); + }); +}); diff --git a/x-pack/plugins/observability/server/services/slo/delete_slo.ts b/x-pack/plugins/observability/server/services/slo/delete_slo.ts new file mode 100644 index 0000000000000..5c56f47efd66e --- /dev/null +++ b/x-pack/plugins/observability/server/services/slo/delete_slo.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core/server'; +import { getSLOTransformId, SLO_INDEX_TEMPLATE_NAME } from '../../assets/constants'; +import { SLO } from '../../types/models'; +import { SLORepository } from './slo_repository'; +import { TransformManager } from './transform_manager'; + +export class DeleteSLO { + constructor( + private repository: SLORepository, + private transformManager: TransformManager, + private esClient: ElasticsearchClient + ) {} + + public async execute(sloId: string): Promise { + const slo = await this.repository.findById(sloId); + + const sloTransformId = getSLOTransformId(sloId); + await this.transformManager.stop(sloTransformId); + await this.transformManager.uninstall(sloTransformId); + + await this.deleteRollupData(slo); + await this.repository.deleteById(sloId); + } + + private async deleteRollupData(slo: SLO): Promise { + await this.esClient.deleteByQuery({ + index: slo.settings.destination_index ?? `${SLO_INDEX_TEMPLATE_NAME}*`, + wait_for_completion: false, + query: { + match: { + 'slo.id': slo.id, + }, + }, + }); + } +} diff --git a/x-pack/plugins/observability/server/services/slo/index.ts b/x-pack/plugins/observability/server/services/slo/index.ts index be64e0b0c0f75..895a7f4497ad5 100644 --- a/x-pack/plugins/observability/server/services/slo/index.ts +++ b/x-pack/plugins/observability/server/services/slo/index.ts @@ -7,5 +7,6 @@ export * from './resource_installer'; export * from './slo_repository'; -export * from './transform_installer'; +export * from './transform_manager'; export * from './create_slo'; +export * from './delete_slo'; diff --git a/x-pack/plugins/observability/server/services/slo/mocks/index.ts b/x-pack/plugins/observability/server/services/slo/mocks/index.ts index 0712fd1c9f8fb..3a9a918fc6480 100644 --- a/x-pack/plugins/observability/server/services/slo/mocks/index.ts +++ b/x-pack/plugins/observability/server/services/slo/mocks/index.ts @@ -7,7 +7,7 @@ import { ResourceInstaller } from '../resource_installer'; import { SLORepository } from '../slo_repository'; -import { TransformInstaller } from '../transform_installer'; +import { TransformManager } from '../transform_manager'; const createResourceInstallerMock = (): jest.Mocked => { return { @@ -15,9 +15,12 @@ const createResourceInstallerMock = (): jest.Mocked => { }; }; -const createTransformInstallerMock = (): jest.Mocked => { +const createTransformManagerMock = (): jest.Mocked => { return { - installAndStartTransform: jest.fn(), + install: jest.fn(), + uninstall: jest.fn(), + start: jest.fn(), + stop: jest.fn(), }; }; @@ -29,4 +32,4 @@ const createSLORepositoryMock = (): jest.Mocked => { }; }; -export { createResourceInstallerMock, createTransformInstallerMock, createSLORepositoryMock }; +export { createResourceInstallerMock, createTransformManagerMock, createSLORepositoryMock }; diff --git a/x-pack/plugins/observability/server/services/slo/resource_installer.test.ts b/x-pack/plugins/observability/server/services/slo/resource_installer.test.ts index 69159d184c0f7..40dc0e46022bc 100644 --- a/x-pack/plugins/observability/server/services/slo/resource_installer.test.ts +++ b/x-pack/plugins/observability/server/services/slo/resource_installer.test.ts @@ -22,7 +22,11 @@ describe('resourceInstaller', () => { it('installs the common resources', async () => { const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient(); mockClusterClient.indices.existsIndexTemplate.mockResponseOnce(false); - const installer = new DefaultResourceInstaller(mockClusterClient, loggerMock.create()); + const installer = new DefaultResourceInstaller( + mockClusterClient, + loggerMock.create(), + 'space-id' + ); await installer.ensureCommonResourcesInstalled(); @@ -51,7 +55,11 @@ describe('resourceInstaller', () => { mockClusterClient.ingest.getPipeline.mockResponseOnce({ [SLO_INGEST_PIPELINE_NAME]: { _meta: { version: SLO_RESOURCES_VERSION } }, } as IngestGetPipelineResponse); - const installer = new DefaultResourceInstaller(mockClusterClient, loggerMock.create()); + const installer = new DefaultResourceInstaller( + mockClusterClient, + loggerMock.create(), + 'space-id' + ); await installer.ensureCommonResourcesInstalled(); diff --git a/x-pack/plugins/observability/server/services/slo/resource_installer.ts b/x-pack/plugins/observability/server/services/slo/resource_installer.ts index ad38710417fed..280183d70459a 100644 --- a/x-pack/plugins/observability/server/services/slo/resource_installer.ts +++ b/x-pack/plugins/observability/server/services/slo/resource_installer.ts @@ -25,13 +25,17 @@ import { getSLOIndexTemplate } from '../../assets/index_templates/slo_index_temp import { getSLOPipelineTemplate } from '../../assets/ingest_templates/slo_pipeline_template'; export interface ResourceInstaller { - ensureCommonResourcesInstalled(spaceId: string): Promise; + ensureCommonResourcesInstalled(): Promise; } export class DefaultResourceInstaller implements ResourceInstaller { - constructor(private esClient: ElasticsearchClient, private logger: Logger) {} + constructor( + private esClient: ElasticsearchClient, + private logger: Logger, + private spaceId: string + ) {} - public async ensureCommonResourcesInstalled(spaceId: string = 'default'): Promise { + public async ensureCommonResourcesInstalled(): Promise { const alreadyInstalled = await this.areResourcesAlreadyInstalled(); if (alreadyInstalled) { @@ -61,7 +65,7 @@ export class DefaultResourceInstaller implements ResourceInstaller { await this.createOrUpdateIngestPipelineTemplate( getSLOPipelineTemplate( SLO_INGEST_PIPELINE_NAME, - this.getPipelinePrefix(SLO_RESOURCES_VERSION, spaceId) + this.getPipelinePrefix(SLO_RESOURCES_VERSION, this.spaceId) ) ); } catch (err) { diff --git a/x-pack/plugins/observability/server/services/slo/slo_repository.test.ts b/x-pack/plugins/observability/server/services/slo/slo_repository.test.ts index 89c59bb2be7ce..8c8e38285f1f2 100644 --- a/x-pack/plugins/observability/server/services/slo/slo_repository.test.ts +++ b/x-pack/plugins/observability/server/services/slo/slo_repository.test.ts @@ -6,24 +6,16 @@ */ import { SavedObject } from '@kbn/core-saved-objects-common'; -import { SavedObjectsClientContract } from '@kbn/core/server'; +import { SavedObjectsClientContract, SavedObjectsErrorHelpers } from '@kbn/core/server'; import { savedObjectsClientMock } from '@kbn/core/server/mocks'; import { SLO, StoredSLO } from '../../types/models'; import { SO_SLO_TYPE } from '../../saved_objects'; import { KibanaSavedObjectsSLORepository } from './slo_repository'; -import { createSLO } from './fixtures/slo'; - -const anSLO = createSLO({ - type: 'slo.apm.transaction_duration', - params: { - environment: 'irrelevant', - service: 'irrelevant', - transaction_type: 'irrelevant', - transaction_name: 'irrelevant', - 'threshold.us': 200000, - }, -}); +import { createAPMTransactionDurationIndicator, createSLO } from './fixtures/slo'; +import { SLONotFound } from '../../errors'; + +const SOME_SLO = createSLO(createAPMTransactionDurationIndicator()); function aStoredSLO(slo: SLO): SavedObject { return { @@ -45,38 +37,61 @@ describe('KibanaSavedObjectsSLORepository', () => { soClientMock = savedObjectsClientMock.create(); }); + describe('validation', () => { + it('findById throws when an SLO is not found', async () => { + soClientMock.get.mockRejectedValueOnce(SavedObjectsErrorHelpers.createGenericNotFoundError()); + const repository = new KibanaSavedObjectsSLORepository(soClientMock); + + await expect(repository.findById('inexistant-slo-id')).rejects.toThrowError( + new SLONotFound('SLO [inexistant-slo-id] not found') + ); + }); + + it('deleteById throws when an SLO is not found', async () => { + soClientMock.delete.mockRejectedValueOnce( + SavedObjectsErrorHelpers.createGenericNotFoundError() + ); + const repository = new KibanaSavedObjectsSLORepository(soClientMock); + + await expect(repository.deleteById('inexistant-slo-id')).rejects.toThrowError( + new SLONotFound('SLO [inexistant-slo-id] not found') + ); + }); + }); + it('saves the SLO', async () => { - soClientMock.create.mockResolvedValueOnce(aStoredSLO(anSLO)); + soClientMock.create.mockResolvedValueOnce(aStoredSLO(SOME_SLO)); const repository = new KibanaSavedObjectsSLORepository(soClientMock); - const savedSLO = await repository.save(anSLO); + const savedSLO = await repository.save(SOME_SLO); - expect(savedSLO).toEqual(anSLO); + expect(savedSLO).toEqual(SOME_SLO); expect(soClientMock.create).toHaveBeenCalledWith( SO_SLO_TYPE, expect.objectContaining({ - ...anSLO, + ...SOME_SLO, updated_at: expect.anything(), created_at: expect.anything(), - }) + }), + { id: SOME_SLO.id } ); }); it('finds an existing SLO', async () => { const repository = new KibanaSavedObjectsSLORepository(soClientMock); - soClientMock.get.mockResolvedValueOnce(aStoredSLO(anSLO)); + soClientMock.get.mockResolvedValueOnce(aStoredSLO(SOME_SLO)); - const foundSLO = await repository.findById(anSLO.id); + const foundSLO = await repository.findById(SOME_SLO.id); - expect(foundSLO).toEqual(anSLO); - expect(soClientMock.get).toHaveBeenCalledWith(SO_SLO_TYPE, anSLO.id); + expect(foundSLO).toEqual(SOME_SLO); + expect(soClientMock.get).toHaveBeenCalledWith(SO_SLO_TYPE, SOME_SLO.id); }); - it('removes an SLO', async () => { + it('deletes an SLO', async () => { const repository = new KibanaSavedObjectsSLORepository(soClientMock); - await repository.deleteById(anSLO.id); + await repository.deleteById(SOME_SLO.id); - expect(soClientMock.delete).toHaveBeenCalledWith(SO_SLO_TYPE, anSLO.id); + expect(soClientMock.delete).toHaveBeenCalledWith(SO_SLO_TYPE, SOME_SLO.id); }); }); diff --git a/x-pack/plugins/observability/server/services/slo/slo_repository.ts b/x-pack/plugins/observability/server/services/slo/slo_repository.ts index cdcfe6b49e760..ccb6b63b048ed 100644 --- a/x-pack/plugins/observability/server/services/slo/slo_repository.ts +++ b/x-pack/plugins/observability/server/services/slo/slo_repository.ts @@ -6,9 +6,11 @@ */ import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; +import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-utils-server'; import { StoredSLO, SLO } from '../../types/models'; import { SO_SLO_TYPE } from '../../saved_objects'; +import { SLONotFound } from '../../errors'; export interface SLORepository { save(slo: SLO): Promise; @@ -21,22 +23,40 @@ export class KibanaSavedObjectsSLORepository implements SLORepository { async save(slo: SLO): Promise { const now = new Date().toISOString(); - const savedSLO = await this.soClient.create(SO_SLO_TYPE, { - ...slo, - created_at: now, - updated_at: now, - }); + const savedSLO = await this.soClient.create( + SO_SLO_TYPE, + { + ...slo, + created_at: now, + updated_at: now, + }, + { id: slo.id } + ); return toSLOModel(savedSLO.attributes); } async findById(id: string): Promise { - const slo = await this.soClient.get(SO_SLO_TYPE, id); - return toSLOModel(slo.attributes); + try { + const slo = await this.soClient.get(SO_SLO_TYPE, id); + return toSLOModel(slo.attributes); + } catch (err) { + if (SavedObjectsErrorHelpers.isNotFoundError(err)) { + throw new SLONotFound(`SLO [${id}] not found`); + } + throw err; + } } async deleteById(id: string): Promise { - await this.soClient.delete(SO_SLO_TYPE, id); + try { + await this.soClient.delete(SO_SLO_TYPE, id); + } catch (err) { + if (SavedObjectsErrorHelpers.isNotFoundError(err)) { + throw new SLONotFound(`SLO [${id}] not found`); + } + throw err; + } } } diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts index c00ba8f69d805..274abe86e3a52 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_duration.ts @@ -10,7 +10,11 @@ import { MappingRuntimeFieldType, TransformPutTransformRequest, } from '@elastic/elasticsearch/lib/api/types'; -import { getSLODestinationIndexName, SLO_INGEST_PIPELINE_NAME } from '../../../assets/constants'; +import { + getSLODestinationIndexName, + getSLOTransformId, + SLO_INGEST_PIPELINE_NAME, +} from '../../../assets/constants'; import { getSLOTransformTemplate } from '../../../assets/transform_templates/slo_transform_template'; import { SLO, @@ -38,7 +42,7 @@ export class ApmTransactionDurationTransformGenerator implements TransformGenera } private buildTransformId(slo: APMTransactionDurationSLO): string { - return `slo-${slo.id}`; + return getSLOTransformId(slo.id); } private buildSource(slo: APMTransactionDurationSLO) { diff --git a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts index e418afbead7da..da65bca20392e 100644 --- a/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts +++ b/x-pack/plugins/observability/server/services/slo/transform_generators/apm_transaction_error_rate.ts @@ -12,7 +12,11 @@ import { } from '@elastic/elasticsearch/lib/api/types'; import { getSLOTransformTemplate } from '../../../assets/transform_templates/slo_transform_template'; import { TransformGenerator } from '.'; -import { getSLODestinationIndexName, SLO_INGEST_PIPELINE_NAME } from '../../../assets/constants'; +import { + getSLODestinationIndexName, + getSLOTransformId, + SLO_INGEST_PIPELINE_NAME, +} from '../../../assets/constants'; import { apmTransactionErrorRateSLOSchema, APMTransactionErrorRateSLO, @@ -40,7 +44,7 @@ export class ApmTransactionErrorRateTransformGenerator implements TransformGener } private buildTransformId(slo: APMTransactionErrorRateSLO): string { - return `slo-${slo.id}`; + return getSLOTransformId(slo.id); } private buildSource(slo: APMTransactionErrorRateSLO) { diff --git a/x-pack/plugins/observability/server/services/slo/transform_installer.test.ts b/x-pack/plugins/observability/server/services/slo/transform_installer.test.ts deleted file mode 100644 index fbe0e7545fc05..0000000000000 --- a/x-pack/plugins/observability/server/services/slo/transform_installer.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -/* eslint-disable max-classes-per-file */ - -import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; -import { ElasticsearchClient } from '@kbn/core/server'; -import { MockedLogger } from '@kbn/logging-mocks'; -import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; - -import { DefaultTransformInstaller } from './transform_installer'; -import { - ApmTransactionErrorRateTransformGenerator, - TransformGenerator, -} from './transform_generators'; -import { SLO, SLITypes } from '../../types/models'; -import { createAPMTransactionErrorRateIndicator, createSLO } from './fixtures/slo'; - -describe('TransformerGenerator', () => { - let esClientMock: jest.Mocked; - let loggerMock: jest.Mocked; - - beforeEach(() => { - esClientMock = elasticsearchServiceMock.createElasticsearchClient(); - loggerMock = loggingSystemMock.createLogger(); - }); - - describe('Unhappy path', () => { - it('throws when no generator exists for the slo indicator type', async () => { - // @ts-ignore defining only a subset of the possible SLI - const generators: Record = { - 'slo.apm.transaction_duration': new DummyTransformGenerator(), - }; - const service = new DefaultTransformInstaller(generators, esClientMock, loggerMock); - - await expect( - service.installAndStartTransform( - createSLO({ - type: 'slo.apm.transaction_error_rate', - params: { - environment: 'irrelevant', - service: 'irrelevant', - transaction_name: 'irrelevant', - transaction_type: 'irrelevant', - }, - }) - ) - ).rejects.toThrowError('Unsupported SLO type: slo.apm.transaction_error_rate'); - }); - - it('throws when transform generator fails', async () => { - // @ts-ignore defining only a subset of the possible SLI - const generators: Record = { - 'slo.apm.transaction_duration': new FailTransformGenerator(), - }; - const service = new DefaultTransformInstaller(generators, esClientMock, loggerMock); - - await expect( - service.installAndStartTransform( - createSLO({ - type: 'slo.apm.transaction_duration', - params: { - environment: 'irrelevant', - service: 'irrelevant', - transaction_name: 'irrelevant', - transaction_type: 'irrelevant', - 'threshold.us': 250000, - }, - }) - ) - ).rejects.toThrowError('Some error'); - }); - }); - - it('installs and starts the transform', async () => { - // @ts-ignore defining only a subset of the possible SLI - const generators: Record = { - 'slo.apm.transaction_error_rate': new ApmTransactionErrorRateTransformGenerator(), - }; - const service = new DefaultTransformInstaller(generators, esClientMock, loggerMock); - - await service.installAndStartTransform(createSLO(createAPMTransactionErrorRateIndicator())); - - expect(esClientMock.transform.putTransform).toHaveBeenCalledTimes(1); - expect(esClientMock.transform.startTransform).toHaveBeenCalledTimes(1); - }); -}); - -class DummyTransformGenerator implements TransformGenerator { - getTransformParams(slo: SLO): TransformPutTransformRequest { - return {} as TransformPutTransformRequest; - } -} - -class FailTransformGenerator implements TransformGenerator { - getTransformParams(slo: SLO): TransformPutTransformRequest { - throw new Error('Some error'); - } -} diff --git a/x-pack/plugins/observability/server/services/slo/transform_installer.ts b/x-pack/plugins/observability/server/services/slo/transform_installer.ts deleted file mode 100644 index 50fdf1b19ee55..0000000000000 --- a/x-pack/plugins/observability/server/services/slo/transform_installer.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { errors } from '@elastic/elasticsearch'; -import { ElasticsearchClient, Logger } from '@kbn/core/server'; - -import { SLO, SLITypes } from '../../types/models'; -import { TransformGenerator } from './transform_generators'; - -export interface TransformInstaller { - installAndStartTransform(slo: SLO, spaceId: string): Promise; -} - -export class DefaultTransformInstaller implements TransformInstaller { - constructor( - private generators: Record, - private esClient: ElasticsearchClient, - private logger: Logger - ) {} - - async installAndStartTransform(slo: SLO, spaceId: string = 'default'): Promise { - const generator = this.generators[slo.indicator.type]; - if (!generator) { - this.logger.error(`No transform generator found for ${slo.indicator.type} SLO type`); - throw new Error(`Unsupported SLO type: ${slo.indicator.type}`); - } - - const transformParams = generator.getTransformParams(slo, spaceId); - try { - await this.esClient.transform.putTransform(transformParams); - } catch (err) { - // swallow the error if the transform already exists. - const isAlreadyExistError = - err instanceof errors.ResponseError && - err?.body?.error?.type === 'resource_already_exists_exception'; - if (!isAlreadyExistError) { - this.logger.error(`Cannot create transform for ${slo.indicator.type} SLO type: ${err}`); - throw err; - } - } - - try { - await this.esClient.transform.startTransform( - { transform_id: transformParams.transform_id }, - { ignore: [409] } - ); - } catch (err) { - this.logger.error(`Cannot start transform id ${transformParams.transform_id}: ${err}`); - throw err; - } - } -} diff --git a/x-pack/plugins/observability/server/services/slo/transform_manager.test.ts b/x-pack/plugins/observability/server/services/slo/transform_manager.test.ts new file mode 100644 index 0000000000000..dafa4bff18a52 --- /dev/null +++ b/x-pack/plugins/observability/server/services/slo/transform_manager.test.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* eslint-disable max-classes-per-file */ + +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { ElasticsearchClient } from '@kbn/core/server'; +import { MockedLogger } from '@kbn/logging-mocks'; +import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types'; + +import { DefaultTransformManager } from './transform_manager'; +import { + ApmTransactionErrorRateTransformGenerator, + TransformGenerator, +} from './transform_generators'; +import { SLO, SLITypes } from '../../types/models'; +import { createAPMTransactionErrorRateIndicator, createSLO } from './fixtures/slo'; + +const SPACE_ID = 'space-id'; + +describe('TransformManager', () => { + let esClientMock: jest.Mocked; + let loggerMock: jest.Mocked; + + beforeEach(() => { + esClientMock = elasticsearchServiceMock.createElasticsearchClient(); + loggerMock = loggingSystemMock.createLogger(); + }); + + describe('Install', () => { + describe('Unhappy path', () => { + it('throws when no generator exists for the slo indicator type', async () => { + // @ts-ignore defining only a subset of the possible SLI + const generators: Record = { + 'slo.apm.transaction_duration': new DummyTransformGenerator(), + }; + const service = new DefaultTransformManager(generators, esClientMock, loggerMock, SPACE_ID); + + await expect( + service.install( + createSLO({ + type: 'slo.apm.transaction_error_rate', + params: { + environment: 'irrelevant', + service: 'irrelevant', + transaction_name: 'irrelevant', + transaction_type: 'irrelevant', + }, + }) + ) + ).rejects.toThrowError('Unsupported SLO type: slo.apm.transaction_error_rate'); + }); + + it('throws when transform generator fails', async () => { + // @ts-ignore defining only a subset of the possible SLI + const generators: Record = { + 'slo.apm.transaction_duration': new FailTransformGenerator(), + }; + const transformManager = new DefaultTransformManager( + generators, + esClientMock, + loggerMock, + SPACE_ID + ); + + await expect( + transformManager.install( + createSLO({ + type: 'slo.apm.transaction_duration', + params: { + environment: 'irrelevant', + service: 'irrelevant', + transaction_name: 'irrelevant', + transaction_type: 'irrelevant', + 'threshold.us': 250000, + }, + }) + ) + ).rejects.toThrowError('Some error'); + }); + }); + + it('installs the transform', async () => { + // @ts-ignore defining only a subset of the possible SLI + const generators: Record = { + 'slo.apm.transaction_error_rate': new ApmTransactionErrorRateTransformGenerator(), + }; + const transformManager = new DefaultTransformManager( + generators, + esClientMock, + loggerMock, + SPACE_ID + ); + const slo = createSLO(createAPMTransactionErrorRateIndicator()); + + const transformId = await transformManager.install(slo); + + expect(esClientMock.transform.putTransform).toHaveBeenCalledTimes(1); + expect(transformId).toBe(`slo-${slo.id}`); + }); + }); + + describe('Start', () => { + it('starts the transform', async () => { + // @ts-ignore defining only a subset of the possible SLI + const generators: Record = { + 'slo.apm.transaction_error_rate': new ApmTransactionErrorRateTransformGenerator(), + }; + const transformManager = new DefaultTransformManager( + generators, + esClientMock, + loggerMock, + SPACE_ID + ); + + await transformManager.start('slo-transform-id'); + + expect(esClientMock.transform.startTransform).toHaveBeenCalledTimes(1); + }); + }); + + describe('Stop', () => { + it('stops the transform', async () => { + // @ts-ignore defining only a subset of the possible SLI + const generators: Record = { + 'slo.apm.transaction_error_rate': new ApmTransactionErrorRateTransformGenerator(), + }; + const transformManager = new DefaultTransformManager( + generators, + esClientMock, + loggerMock, + SPACE_ID + ); + + await transformManager.stop('slo-transform-id'); + + expect(esClientMock.transform.stopTransform).toHaveBeenCalledTimes(1); + }); + }); + + describe('Uninstall', () => { + it('uninstalls the transform', async () => { + // @ts-ignore defining only a subset of the possible SLI + const generators: Record = { + 'slo.apm.transaction_error_rate': new ApmTransactionErrorRateTransformGenerator(), + }; + const transformManager = new DefaultTransformManager( + generators, + esClientMock, + loggerMock, + SPACE_ID + ); + + await transformManager.uninstall('slo-transform-id'); + + expect(esClientMock.transform.deleteTransform).toHaveBeenCalledTimes(1); + }); + }); +}); + +class DummyTransformGenerator implements TransformGenerator { + getTransformParams(slo: SLO): TransformPutTransformRequest { + return {} as TransformPutTransformRequest; + } +} + +class FailTransformGenerator implements TransformGenerator { + getTransformParams(slo: SLO): TransformPutTransformRequest { + throw new Error('Some error'); + } +} diff --git a/x-pack/plugins/observability/server/services/slo/transform_manager.ts b/x-pack/plugins/observability/server/services/slo/transform_manager.ts new file mode 100644 index 0000000000000..178d6dacaa433 --- /dev/null +++ b/x-pack/plugins/observability/server/services/slo/transform_manager.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient, Logger } from '@kbn/core/server'; + +import { SLO, SLITypes } from '../../types/models'; +import { TransformGenerator } from './transform_generators'; + +type TransformId = string; + +export interface TransformManager { + install(slo: SLO): Promise; + start(transformId: TransformId): Promise; + stop(transformId: TransformId): Promise; + uninstall(transformId: TransformId): Promise; +} + +export class DefaultTransformManager implements TransformManager { + constructor( + private generators: Record, + private esClient: ElasticsearchClient, + private logger: Logger, + private spaceId: string + ) {} + + async install(slo: SLO): Promise { + const generator = this.generators[slo.indicator.type]; + if (!generator) { + this.logger.error(`No transform generator found for ${slo.indicator.type} SLO type`); + throw new Error(`Unsupported SLO type: ${slo.indicator.type}`); + } + + const transformParams = generator.getTransformParams(slo, this.spaceId); + try { + await this.esClient.transform.putTransform(transformParams); + } catch (err) { + this.logger.error(`Cannot create transform for ${slo.indicator.type} SLO type: ${err}`); + throw err; + } + + return transformParams.transform_id; + } + + async start(transformId: TransformId): Promise { + try { + await this.esClient.transform.startTransform( + { transform_id: transformId }, + { ignore: [409] } + ); + } catch (err) { + this.logger.error(`Cannot start transform id ${transformId}: ${err}`); + throw err; + } + } + + async stop(transformId: TransformId): Promise { + try { + await this.esClient.transform.stopTransform( + { transform_id: transformId, wait_for_completion: true }, + { ignore: [404] } + ); + } catch (err) { + this.logger.error(`Cannot stop transform id ${transformId}: ${err}`); + throw err; + } + } + + async uninstall(transformId: TransformId): Promise { + try { + await this.esClient.transform.deleteTransform( + { transform_id: transformId, force: true }, + { ignore: [404] } + ); + } catch (err) { + this.logger.error(`Cannot delete transform id ${transformId}: ${err}`); + throw err; + } + } +} diff --git a/x-pack/plugins/observability/server/types/schema/slo.ts b/x-pack/plugins/observability/server/types/schema/slo.ts index 888808f129fc0..c601b49270f89 100644 --- a/x-pack/plugins/observability/server/types/schema/slo.ts +++ b/x-pack/plugins/observability/server/types/schema/slo.ts @@ -85,3 +85,9 @@ export type CreateSLOResponse = t.TypeOf; export const createSLOParamsSchema = t.type({ body: createSLOBodySchema, }); + +export const deleteSLOParamsSchema = t.type({ + path: t.type({ + id: t.string, + }), +}); From 92ca42f007492dbda15ef60c73b6a7e944a9f917 Mon Sep 17 00:00:00 2001 From: "Christiane (Tina) Heiligers" Date: Tue, 20 Sep 2022 07:51:25 -0700 Subject: [PATCH 06/76] [saved objects] Adds bulkDelete API (#139680) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../core-saved-objects-api-browser/index.ts | 3 + .../src/apis/bulk_delete.ts | 27 + .../src/apis/index.ts | 5 + .../src/saved_objects_client.ts | 14 + .../src/lib/repository.test.ts | 513 ++++++++++++++++++ .../src/lib/repository.ts | 296 +++++++++- .../repository_bulk_delete_internal_types.ts | 86 +++ .../src/mocks/repository.mock.ts | 1 + .../src/saved_objects_client.test.ts | 18 + .../src/saved_objects_client.ts | 11 + .../src/repository.mock.ts | 1 + .../src/saved_objects_client.mock.ts | 1 + .../core-saved-objects-api-server/index.ts | 4 + .../src/apis/bulk_delete.ts | 50 ++ .../src/apis/index.ts | 6 + .../src/saved_objects_client.ts | 13 + .../src/saved_objects_repository.ts | 14 + .../src/saved_objects_client.test.ts | 39 ++ .../src/saved_objects_client.ts | 28 + .../src/saved_objects_service.mock.ts | 1 + .../index.ts | 1 + .../src/routes/bulk_delete.ts | 48 ++ .../src/routes/index.ts | 2 + .../src/usage_stats_client.ts | 2 + .../src/core_usage_stats_client.test.ts | 76 +++ .../src/core_usage_stats_client.ts | 6 + .../src/core_usage_stats_client.mock.ts | 1 + .../src/core_usage_stats.ts | 7 + src/core/server/index.ts | 3 + .../saved_objects/routes/bulk_delete.test.ts | 97 ++++ .../service/lib/repository_with_proxy.test.ts | 37 ++ .../lib/repository_with_proxy_utils.ts | 3 +- .../collectors/core/core_usage_collector.ts | 40 ++ .../server/collectors/core/core_usage_data.ts | 7 + src/plugins/telemetry/schema/oss_plugins.json | 42 ++ .../apis/saved_objects/bulk_delete.ts | 114 ++++ .../apis/saved_objects/index.ts | 1 + ...ypted_saved_objects_client_wrapper.test.ts | 38 ++ .../encrypted_saved_objects_client_wrapper.ts | 9 + .../feature_privilege_builder/saved_object.ts | 1 + .../privileges/privileges.test.ts | 53 ++ ...ecure_saved_objects_client_wrapper.test.ts | 61 +++ .../secure_saved_objects_client_wrapper.ts | 45 ++ .../spaces_saved_objects_client.test.ts | 35 ++ .../spaces_saved_objects_client.ts | 13 + .../common/suites/bulk_delete.ts | 154 ++++++ .../security_and_spaces/apis/bulk_delete.ts | 105 ++++ .../security_and_spaces/apis/index.ts | 1 + .../spaces_only/apis/bulk_delete.ts | 68 +++ .../spaces_only/apis/index.ts | 1 + 50 files changed, 2200 insertions(+), 2 deletions(-) create mode 100644 packages/core/saved-objects/core-saved-objects-api-browser/src/apis/bulk_delete.ts create mode 100644 packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository_bulk_delete_internal_types.ts create mode 100644 packages/core/saved-objects/core-saved-objects-api-server/src/apis/bulk_delete.ts create mode 100644 packages/core/saved-objects/core-saved-objects-server-internal/src/routes/bulk_delete.ts create mode 100644 src/core/server/integration_tests/saved_objects/routes/bulk_delete.test.ts create mode 100644 test/api_integration/apis/saved_objects/bulk_delete.ts create mode 100644 x-pack/test/saved_object_api_integration/common/suites/bulk_delete.ts create mode 100644 x-pack/test/saved_object_api_integration/security_and_spaces/apis/bulk_delete.ts create mode 100644 x-pack/test/saved_object_api_integration/spaces_only/apis/bulk_delete.ts diff --git a/packages/core/saved-objects/core-saved-objects-api-browser/index.ts b/packages/core/saved-objects/core-saved-objects-api-browser/index.ts index 9fc3e6a78c5c0..e78c56d76556c 100644 --- a/packages/core/saved-objects/core-saved-objects-api-browser/index.ts +++ b/packages/core/saved-objects/core-saved-objects-api-browser/index.ts @@ -22,4 +22,7 @@ export type { SavedObjectsBulkUpdateOptions, SavedObjectsBulkResolveResponse, SavedObjectsBulkCreateObject, + SavedObjectsBulkDeleteOptions, + SavedObjectsBulkDeleteResponseItem, + SavedObjectsBulkDeleteResponse, } from './src/apis'; diff --git a/packages/core/saved-objects/core-saved-objects-api-browser/src/apis/bulk_delete.ts b/packages/core/saved-objects/core-saved-objects-api-browser/src/apis/bulk_delete.ts new file mode 100644 index 0000000000000..1e4b5d2268dea --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-api-browser/src/apis/bulk_delete.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SavedObjectError } from '@kbn/core-saved-objects-common'; + +/** @public */ +export interface SavedObjectsBulkDeleteOptions { + force?: boolean; +} + +/** @public */ +export interface SavedObjectsBulkDeleteResponseItem { + id: string; + type: string; + success: boolean; + error?: SavedObjectError; +} + +/** @public */ +export interface SavedObjectsBulkDeleteResponse { + statuses: SavedObjectsBulkDeleteResponseItem[]; +} diff --git a/packages/core/saved-objects/core-saved-objects-api-browser/src/apis/index.ts b/packages/core/saved-objects/core-saved-objects-api-browser/src/apis/index.ts index 4652facb972cc..afee0a01494e1 100644 --- a/packages/core/saved-objects/core-saved-objects-api-browser/src/apis/index.ts +++ b/packages/core/saved-objects/core-saved-objects-api-browser/src/apis/index.ts @@ -19,3 +19,8 @@ export type { } from './find'; export type { ResolvedSimpleSavedObject } from './resolve'; export type { SavedObjectsUpdateOptions } from './update'; +export type { + SavedObjectsBulkDeleteOptions, + SavedObjectsBulkDeleteResponseItem, + SavedObjectsBulkDeleteResponse, +} from './bulk_delete'; diff --git a/packages/core/saved-objects/core-saved-objects-api-browser/src/saved_objects_client.ts b/packages/core/saved-objects/core-saved-objects-api-browser/src/saved_objects_client.ts index 123b5c81d4064..d222770a8579d 100644 --- a/packages/core/saved-objects/core-saved-objects-api-browser/src/saved_objects_client.ts +++ b/packages/core/saved-objects/core-saved-objects-api-browser/src/saved_objects_client.ts @@ -19,7 +19,10 @@ import type { SavedObjectsFindOptions, SavedObjectsUpdateOptions, SavedObjectsDeleteOptions, + SavedObjectsBulkDeleteResponse, + SavedObjectsBulkDeleteOptions, } from './apis'; + import type { SimpleSavedObject } from './simple_saved_object'; /** @@ -52,6 +55,17 @@ export interface SavedObjectsClientContract { */ delete(type: string, id: string, options?: SavedObjectsDeleteOptions): Promise<{}>; + /** + * Deletes multiple documents at once + * @param objects - an array of objects containing id, type + * @param options - optional force argument to force deletion of objects in a namespace other than the scoped client + * @returns The bulk delete result for the saved objects for the given types and ids. + */ + bulkDelete( + objects: SavedObjectTypeIdTuple[], + options?: SavedObjectsBulkDeleteOptions + ): Promise; + /** * Search for objects * diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.test.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.test.ts index c86945c6acb5c..0739c9acab8f5 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.test.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.test.ts @@ -46,6 +46,8 @@ import type { SavedObjectsCollectMultiNamespaceReferencesResponse, SavedObjectsUpdateObjectsSpacesObject, SavedObjectsUpdateObjectsSpacesOptions, + SavedObjectsBulkDeleteObject, + SavedObjectsBulkDeleteOptions, } from '@kbn/core-saved-objects-api-server'; import type { SavedObjectsType, @@ -2044,6 +2046,517 @@ describe('SavedObjectsRepository', () => { }); }); + describe('#bulkDelete', () => { + const obj1: SavedObjectsBulkDeleteObject = { + type: 'config', + id: '6.0.0-alpha1', + }; + const obj2: SavedObjectsBulkDeleteObject = { + type: 'index-pattern', + id: 'logstash-*', + }; + + const namespace = 'foo-namespace'; + + const createNamespaceAwareGetId = (type: string, id: string) => + `${registry.isSingleNamespace(type) && namespace ? `${namespace}:` : ''}${type}:${id}`; + + const getMockEsBulkDeleteResponse = ( + objects: TypeIdTuple[], + options?: SavedObjectsBulkDeleteOptions + ) => + ({ + items: objects.map(({ type, id }) => ({ + // es response returns more fields than what we're interested in. + delete: { + _id: `${ + registry.isSingleNamespace(type) && options?.namespace ? `${options?.namespace}:` : '' + }${type}:${id}`, + ...mockVersionProps, + result: 'deleted', + }, + })), + } as estypes.BulkResponse); + + const repositoryBulkDeleteSuccess = async ( + objects: SavedObjectsBulkDeleteObject[] = [], + options?: SavedObjectsBulkDeleteOptions, + internalOptions: { + mockMGetResponseWithObject?: { initialNamespaces: string[]; type: string; id: string }; + } = {} + ) => { + const multiNamespaceObjects = objects.filter(({ type }) => { + return registry.isMultiNamespace(type); + }); + + const { mockMGetResponseWithObject } = internalOptions; + if (multiNamespaceObjects.length > 0) { + const mockedMGetResponse = mockMGetResponseWithObject + ? getMockMgetResponse([mockMGetResponseWithObject], options?.namespace) + : getMockMgetResponse(multiNamespaceObjects, options?.namespace); + client.mget.mockResponseOnce(mockedMGetResponse); + } + const mockedEsBulkDeleteResponse = getMockEsBulkDeleteResponse(objects, options); + + client.bulk.mockResponseOnce(mockedEsBulkDeleteResponse); + const result = await savedObjectsRepository.bulkDelete(objects, options); + + expect(client.mget).toHaveBeenCalledTimes(multiNamespaceObjects?.length ? 1 : 0); + return result; + }; + + // bulk delete calls only has one object for each source -- the action + const expectClientCallBulkDeleteArgsAction = ( + objects: TypeIdTuple[], + { + method, + _index = expect.any(String), + getId = () => expect.any(String), + overrides = {}, + }: { + method: string; + _index?: string; + getId?: (type: string, id: string) => string; + overrides?: Record; + } + ) => { + const body = []; + for (const { type, id } of objects) { + body.push({ + [method]: { + _index, + _id: getId(type, id), + ...overrides, + }, + }); + } + + expect(client.bulk).toHaveBeenCalledWith( + expect.objectContaining({ body }), + expect.anything() + ); + }; + + const createBulkDeleteFailStatus = ({ + type, + id, + error, + }: { + type: string; + id: string; + error?: ExpectedErrorResult['error']; + }) => ({ + type, + id, + success: false, + error: error ?? createBadRequestError(), + }); + + const createBulkDeleteSuccessStatus = ({ type, id }: { type: string; id: string }) => ({ + type, + id, + success: true, + }); + + // mocks a combination of success, error results for hidden and unknown object object types. + const repositoryBulkDeleteError = async ( + obj: SavedObjectsBulkDeleteObject, + isBulkError: boolean, + expectedErrorResult: ExpectedErrorResult + ) => { + const objects = [obj1, obj, obj2]; + const mockedBulkDeleteResponse = getMockEsBulkDeleteResponse(objects); + if (isBulkError) { + mockGetBulkOperationError.mockReturnValueOnce(undefined); + mockGetBulkOperationError.mockReturnValueOnce(expectedErrorResult.error as Payload); + } + client.bulk.mockResponseOnce(mockedBulkDeleteResponse); + + const result = await savedObjectsRepository.bulkDelete(objects); + expect(client.bulk).toHaveBeenCalled(); + expect(result).toEqual({ + statuses: [ + createBulkDeleteSuccessStatus(obj1), + createBulkDeleteFailStatus({ ...obj, error: expectedErrorResult.error }), + createBulkDeleteSuccessStatus(obj2), + ], + }); + }; + + const expectClientCallArgsAction = ( + objects: TypeIdTuple[], + { + method, + _index = expect.any(String), + getId = () => expect.any(String), + overrides = {}, + }: { + method: string; + _index?: string; + getId?: (type: string, id: string) => string; + overrides?: Record; + } + ) => { + const body = []; + for (const { type, id } of objects) { + body.push({ + [method]: { + _index, + _id: getId(type, id), + ...overrides, + }, + }); + } + expect(client.bulk).toHaveBeenCalledWith( + expect.objectContaining({ body }), + expect.anything() + ); + }; + + const bulkDeleteMultiNamespaceError = async ( + [obj1, _obj, obj2]: SavedObjectsBulkDeleteObject[], + options: SavedObjectsBulkDeleteOptions | undefined, + mgetResponse: estypes.MgetResponse, + mgetOptions?: { statusCode?: number } + ) => { + const getId = (type: string, id: string) => `${options?.namespace}:${type}:${id}`; + // mock the response for the not found doc + client.mget.mockResponseOnce(mgetResponse, { statusCode: mgetOptions?.statusCode }); + // get a mocked response for the valid docs + const bulkResponse = getMockEsBulkDeleteResponse([obj1, obj2], { namespace }); + client.bulk.mockResponseOnce(bulkResponse); + + const result = await savedObjectsRepository.bulkDelete([obj1, _obj, obj2], options); + expect(client.bulk).toHaveBeenCalledTimes(1); + expect(client.mget).toHaveBeenCalledTimes(1); + + expectClientCallArgsAction([obj1, obj2], { method: 'delete', getId }); + expect(result).toEqual({ + statuses: [ + createBulkDeleteSuccessStatus(obj1), + { ...expectErrorNotFound(_obj), success: false }, + createBulkDeleteSuccessStatus(obj2), + ], + }); + }; + + beforeEach(() => { + mockDeleteLegacyUrlAliases.mockClear(); + mockDeleteLegacyUrlAliases.mockResolvedValue(); + }); + + describe('client calls', () => { + it(`should use the ES bulk action by default`, async () => { + await repositoryBulkDeleteSuccess([obj1, obj2]); + expect(client.bulk).toHaveBeenCalled(); + }); + + it(`should use the ES mget action before bulk action for any types that are multi-namespace`, async () => { + const objects = [obj1, { ...obj2, type: MULTI_NAMESPACE_ISOLATED_TYPE }]; + await repositoryBulkDeleteSuccess(objects); + expect(client.bulk).toHaveBeenCalled(); + expect(client.mget).toHaveBeenCalled(); + + const docs = [ + expect.objectContaining({ _id: `${MULTI_NAMESPACE_ISOLATED_TYPE}:${obj2.id}` }), + ]; + expect(client.mget).toHaveBeenCalledWith( + expect.objectContaining({ body: { docs } }), + expect.anything() + ); + }); + + it(`should not use the ES bulk action when there are no valid documents to delete`, async () => { + const objects = [obj1, obj2].map((x) => ({ ...x, type: 'unknownType' })); + await savedObjectsRepository.bulkDelete(objects); + expect(client.bulk).toHaveBeenCalledTimes(0); + }); + + it(`formats the ES request`, async () => { + const getId = createNamespaceAwareGetId; + await repositoryBulkDeleteSuccess([obj1, obj2], { namespace }); + expectClientCallBulkDeleteArgsAction([obj1, obj2], { method: 'delete', getId }); + }); + + it(`formats the ES request for any types that are multi-namespace`, async () => { + const _obj2 = { ...obj2, type: MULTI_NAMESPACE_ISOLATED_TYPE }; + const getId = createNamespaceAwareGetId; + await repositoryBulkDeleteSuccess([obj1, _obj2], { namespace }); + expectClientCallBulkDeleteArgsAction([obj1, _obj2], { method: 'delete', getId }); + }); + + it(`defaults to a refresh setting of wait_for`, async () => { + await repositoryBulkDeleteSuccess([obj1, obj2]); + expect(client.bulk).toHaveBeenCalledWith( + expect.objectContaining({ refresh: 'wait_for' }), + expect.anything() + ); + }); + + it(`does not include the version of the existing document when not using a multi-namespace type`, async () => { + const objects = [obj1, { ...obj2, type: NAMESPACE_AGNOSTIC_TYPE }]; + await repositoryBulkDeleteSuccess(objects); + expectClientCallBulkDeleteArgsAction(objects, { method: 'delete' }); + }); + + it(`prepends namespace to the id when providing namespace for single-namespace type`, async () => { + const getId = createNamespaceAwareGetId; + await repositoryBulkDeleteSuccess([obj1, obj2], { namespace }); + expectClientCallBulkDeleteArgsAction([obj1, obj2], { method: 'delete', getId }); + }); + + it(`doesn't prepend namespace to the id when providing no namespace for single-namespace type`, async () => { + const getId = (type: string, id: string) => `${type}:${id}`; + await repositoryBulkDeleteSuccess([obj1, obj2]); + expectClientCallBulkDeleteArgsAction([obj1, obj2], { method: 'delete', getId }); + }); + + it(`normalizes options.namespace from 'default' to undefined`, async () => { + const getId = (type: string, id: string) => `${type}:${id}`; + await repositoryBulkDeleteSuccess([obj1, obj2], { namespace: 'default' }); + expectClientCallBulkDeleteArgsAction([obj1, obj2], { method: 'delete', getId }); + }); + + it(`doesn't prepend namespace to the id when not using single-namespace type`, async () => { + const getId = (type: string, id: string) => `${type}:${id}`; // not expecting namespace prefix; + const _obj1 = { ...obj1, type: NAMESPACE_AGNOSTIC_TYPE }; + const _obj2 = { ...obj2, type: MULTI_NAMESPACE_ISOLATED_TYPE }; + + await repositoryBulkDeleteSuccess([_obj1, _obj2], { namespace }); + expectClientCallBulkDeleteArgsAction([_obj1, _obj2], { method: 'delete', getId }); + }); + }); + + describe('legacy URL aliases', () => { + it(`doesn't delete legacy URL aliases for single-namespace object types`, async () => { + await repositoryBulkDeleteSuccess([obj1, obj2]); + expect(mockDeleteLegacyUrlAliases).not.toHaveBeenCalled(); + }); + + it(`deletes legacy URL aliases for multi-namespace object types (all spaces)`, async () => { + const testObject = { ...obj1, type: MULTI_NAMESPACE_TYPE }; + const internalOptions = { + mockMGetResponseWithObject: { + ...testObject, + initialNamespaces: [ALL_NAMESPACES_STRING], + }, + }; + await repositoryBulkDeleteSuccess( + [testObject], + { namespace, force: true }, + internalOptions + ); + expect(mockDeleteLegacyUrlAliases).toHaveBeenCalledWith( + expect.objectContaining({ + type: MULTI_NAMESPACE_TYPE, + id: testObject.id, + namespaces: [], + deleteBehavior: 'exclusive', + }) + ); + }); + + it(`deletes legacy URL aliases for multi-namespace object types (specific space)`, async () => { + const testObject = { ...obj1, type: MULTI_NAMESPACE_TYPE }; + const internalOptions = { + mockMGetResponseWithObject: { + ...testObject, + initialNamespaces: [namespace], + }, + }; + // specifically test against the current namespace + await repositoryBulkDeleteSuccess([testObject], { namespace }, internalOptions); + expect(mockDeleteLegacyUrlAliases).toHaveBeenCalledWith( + expect.objectContaining({ + type: MULTI_NAMESPACE_TYPE, + id: testObject.id, + namespaces: [namespace], + deleteBehavior: 'inclusive', + }) + ); + }); + + it(`deletes legacy URL aliases for multi-namespace object types shared to many specific spaces`, async () => { + const testObject = { ...obj1, type: MULTI_NAMESPACE_TYPE }; + const initialTestObjectNamespaces = [namespace, 'bar-namespace']; + const internalOptions = { + mockMGetResponseWithObject: { + ...testObject, + initialNamespaces: initialTestObjectNamespaces, + }, + }; + // specifically test against named spaces ('*' is handled specifically, this assures we also take care of named spaces) + await repositoryBulkDeleteSuccess( + [testObject], + { namespace, force: true }, + internalOptions + ); + expect(mockDeleteLegacyUrlAliases).toHaveBeenCalledWith( + expect.objectContaining({ + type: MULTI_NAMESPACE_TYPE, + id: testObject.id, + namespaces: initialTestObjectNamespaces, + deleteBehavior: 'inclusive', + }) + ); + }); + + it(`logs a message when deleteLegacyUrlAliases returns an error`, async () => { + const testObject = { type: MULTI_NAMESPACE_ISOLATED_TYPE, id: obj1.id }; + + client.mget.mockResolvedValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise( + getMockMgetResponse([testObject], namespace) + ) + ); + const mockedBulkResponse = getMockEsBulkDeleteResponse([testObject], { namespace }); + client.bulk.mockResolvedValueOnce(mockedBulkResponse); + + mockDeleteLegacyUrlAliases.mockRejectedValueOnce(new Error('Oh no!')); + + await savedObjectsRepository.bulkDelete([testObject], { namespace }); + + expect(client.mget).toHaveBeenCalledTimes(1); + expect(logger.error).toHaveBeenCalledTimes(1); + expect(logger.error).toHaveBeenCalledWith( + 'Unable to delete aliases when deleting an object: Oh no!' + ); + }); + }); + + describe('errors', () => { + it(`throws an error when options.namespace is '*'`, async () => { + await expect( + savedObjectsRepository.bulkDelete([obj1], { namespace: ALL_NAMESPACES_STRING }) + ).rejects.toThrowError(createBadRequestError('"options.namespace" cannot be "*"')); + }); + + it(`throws an error when client bulk response is not defined`, async () => { + client.mget.mockResolvedValueOnce( + elasticsearchClientMock.createSuccessTransportRequestPromise( + getMockMgetResponse([obj1], namespace) + ) + ); + const mockedBulkResponse = undefined; + // we have to cast here to test the assumption we always get a response. + client.bulk.mockResponseOnce(mockedBulkResponse as unknown as estypes.BulkResponse); + await expect(savedObjectsRepository.bulkDelete([obj1], { namespace })).rejects.toThrowError( + 'Unexpected error in bulkDelete saved objects: bulkDeleteResponse is undefined' + ); + }); + + it(`returns an error for the object when the object's type is invalid`, async () => { + const unknownObjType = { ...obj1, type: 'unknownType' }; + await repositoryBulkDeleteError( + unknownObjType, + false, + expectErrorInvalidType(unknownObjType) + ); + }); + + it(`returns an error for an object when the object's type is hidden`, async () => { + const hiddenObject = { ...obj1, type: HIDDEN_TYPE }; + await repositoryBulkDeleteError(hiddenObject, false, expectErrorInvalidType(hiddenObject)); + }); + + it(`returns an error when ES is unable to find the document during mget`, async () => { + const notFoundObj = { ...obj1, type: MULTI_NAMESPACE_ISOLATED_TYPE, found: false }; + const mgetResponse = getMockMgetResponse([notFoundObj], namespace); + await bulkDeleteMultiNamespaceError([obj1, notFoundObj, obj2], { namespace }, mgetResponse); + }); + + it(`returns an error when ES is unable to find the index during mget`, async () => { + const notFoundObj = { ...obj1, type: MULTI_NAMESPACE_ISOLATED_TYPE, found: false }; + await bulkDeleteMultiNamespaceError( + [obj1, notFoundObj, obj2], + { namespace }, + {} as estypes.MgetResponse, + { + statusCode: 404, + } + ); + }); + + it(`returns an error when the type is multi-namespace and the document exists, but not in this namespace`, async () => { + const obj = { + type: MULTI_NAMESPACE_ISOLATED_TYPE, + id: 'three', + namespace: 'bar-namespace', + }; + const mgetResponse = getMockMgetResponse([obj], namespace); + await bulkDeleteMultiNamespaceError([obj1, obj, obj2], { namespace }, mgetResponse); + }); + + it(`returns an error when the type is multi-namespace and the document has multiple namespaces and the force option is not enabled`, async () => { + const testObject = { ...obj1, type: MULTI_NAMESPACE_TYPE }; + const internalOptions = { + mockMGetResponseWithObject: { + ...testObject, + initialNamespaces: [namespace, 'bar-namespace'], + }, + }; + const result = await repositoryBulkDeleteSuccess( + [testObject], + { namespace }, + internalOptions + ); + expect(result.statuses[0]).toStrictEqual( + createBulkDeleteFailStatus({ + ...testObject, + error: createBadRequestError( + 'Unable to delete saved object that exists in multiple namespaces, use the "force" option to delete it anyway' + ), + }) + ); + }); + + it(`returns an error when the type is multi-namespace and the document has all namespaces and the force option is not enabled`, async () => { + const testObject = { ...obj1, type: ALL_NAMESPACES_STRING }; + const internalOptions = { + mockMGetResponseWithObject: { + ...testObject, + initialNamespaces: [namespace, 'bar-namespace'], + }, + }; + const result = await repositoryBulkDeleteSuccess( + [testObject], + { namespace }, + internalOptions + ); + expect(result.statuses[0]).toStrictEqual( + createBulkDeleteFailStatus({ + ...testObject, + error: createBadRequestError("Unsupported saved object type: '*'"), + }) + ); + }); + }); + + describe('returns', () => { + it(`returns early for empty objects argument`, async () => { + await savedObjectsRepository.bulkDelete([], { namespace }); + expect(client.bulk).toHaveBeenCalledTimes(0); + }); + + it(`formats the ES response`, async () => { + const response = await repositoryBulkDeleteSuccess([obj1, obj2], { namespace }); + expect(response).toEqual({ + statuses: [obj1, obj2].map(createBulkDeleteSuccessStatus), + }); + }); + + it(`handles a mix of successful deletes and errors`, async () => { + const notFoundObj = { ...obj1, type: MULTI_NAMESPACE_ISOLATED_TYPE, found: false }; + await bulkDeleteMultiNamespaceError( + [obj1, notFoundObj, obj2], + { namespace }, + {} as estypes.MgetResponse, + { statusCode: 404 } + ); + }); + }); + }); + describe('#checkConflicts', () => { const obj1 = { type: 'dashboard', id: 'one' }; const obj2 = { type: 'dashboard', id: 'two' }; diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.ts index 961b44a1cd688..5569141c7fa0e 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.ts @@ -54,6 +54,9 @@ import type { SavedObjectsClosePointInTimeOptions, SavedObjectsCreatePointInTimeFinderOptions, SavedObjectsFindOptions, + SavedObjectsBulkDeleteObject, + SavedObjectsBulkDeleteOptions, + SavedObjectsBulkDeleteResponse, } from '@kbn/core-saved-objects-api-server'; import type { SavedObjectSanitizedDoc, @@ -83,6 +86,7 @@ import { type IndexMapping, type IKibanaMigrator, } from '@kbn/core-saved-objects-base-server-internal'; +import pMap from 'p-map'; import { PointInTimeFinder } from './point_in_time_finder'; import { createRepositoryEsClient, RepositoryEsClient } from './repository_es_client'; import { getSearchDsl } from './search_dsl'; @@ -109,6 +113,16 @@ import { PreflightCheckForCreateObject, } from './preflight_check_for_create'; import { deleteLegacyUrlAliases } from './legacy_url_aliases'; +import type { + BulkDeleteParams, + ExpectedBulkDeleteResult, + BulkDeleteItemErrorResult, + NewBulkItemResponse, + BulkDeleteExpectedBulkGetResult, + PreflightCheckForBulkDeleteParams, + ExpectedBulkDeleteMultiNamespaceDocsParams, + ObjectToDeleteAliasesFor, +} from './repository_bulk_delete_internal_types'; // BEWARE: The SavedObjectClient depends on the implementation details of the SavedObjectsRepository // so any breaking changes to this repository are considered breaking changes to the SavedObjectsClient. @@ -127,6 +141,7 @@ export interface SavedObjectsRepositoryOptions { export const DEFAULT_REFRESH_SETTING = 'wait_for'; export const DEFAULT_RETRY_COUNT = 3; +const MAX_CONCURRENT_ALIAS_DELETIONS = 10; /** * @internal */ @@ -676,7 +691,6 @@ export class SavedObjectsRepository implements ISavedObjectsRepository { if (!this._allowedTypes.includes(type)) { throw SavedObjectsErrorHelpers.createGenericNotFoundError(type, id); } - const { refresh = DEFAULT_REFRESH_SETTING, force } = options; const namespace = normalizeNamespace(options.namespace); @@ -762,6 +776,286 @@ export class SavedObjectsRepository implements ISavedObjectsRepository { ); } + /** + * Performs initial checks on object type validity and flags multi-namespace objects for preflight checks by adding an `esRequestIndex` + * @param objects SavedObjectsBulkDeleteObject[] + * @returns array BulkDeleteExpectedBulkGetResult[] + * @internal + */ + private presortObjectsByNamespaceType(objects: SavedObjectsBulkDeleteObject[]) { + let bulkGetRequestIndexCounter = 0; + return objects.map((object) => { + const { type, id } = object; + if (!this._allowedTypes.includes(type)) { + return { + tag: 'Left', + value: { + id, + type, + error: errorContent(SavedObjectsErrorHelpers.createUnsupportedTypeError(type)), + }, + }; + } + const requiresNamespacesCheck = this._registry.isMultiNamespace(type); + return { + tag: 'Right', + value: { + type, + id, + ...(requiresNamespacesCheck && { esRequestIndex: bulkGetRequestIndexCounter++ }), + }, + }; + }); + } + + /** + * Fetch multi-namespace saved objects + * @returns MgetResponse + * @notes multi-namespace objects shared to more than one space require special handling. We fetch these docs to retrieve their namespaces. + * @internal + */ + private async preflightCheckForBulkDelete(params: PreflightCheckForBulkDeleteParams) { + const { expectedBulkGetResults, namespace } = params; + const bulkGetMultiNamespaceDocs = expectedBulkGetResults + .filter(isRight) + .filter(({ value }) => value.esRequestIndex !== undefined) + .map(({ value: { type, id } }) => ({ + _id: this._serializer.generateRawId(namespace, type, id), + _index: this.getIndexForType(type), + _source: ['type', 'namespaces'], + })); + + const bulkGetMultiNamespaceDocsResponse = bulkGetMultiNamespaceDocs.length + ? await this.client.mget( + { body: { docs: bulkGetMultiNamespaceDocs } }, + { ignore: [404], meta: true } + ) + : undefined; + // fail fast if we can't verify a 404 response is from Elasticsearch + if ( + bulkGetMultiNamespaceDocsResponse && + isNotFoundFromUnsupportedServer({ + statusCode: bulkGetMultiNamespaceDocsResponse.statusCode, + headers: bulkGetMultiNamespaceDocsResponse.headers, + }) + ) { + throw SavedObjectsErrorHelpers.createGenericNotFoundEsUnavailableError(); + } + return bulkGetMultiNamespaceDocsResponse; + } + + /** + * @returns array of objects sorted by expected delete success or failure result + * @internal + */ + private getExpectedBulkDeleteMultiNamespaceDocsResults( + params: ExpectedBulkDeleteMultiNamespaceDocsParams + ): ExpectedBulkDeleteResult[] { + const { expectedBulkGetResults, multiNamespaceDocsResponse, namespace, force } = params; + let indexCounter = 0; + const expectedBulkDeleteMultiNamespaceDocsResults = + expectedBulkGetResults.map((expectedBulkGetResult) => { + if (isLeft(expectedBulkGetResult)) { + return { ...expectedBulkGetResult }; + } + const { esRequestIndex: esBulkGetRequestIndex, id, type } = expectedBulkGetResult.value; + + let namespaces; + + if (esBulkGetRequestIndex !== undefined) { + const indexFound = multiNamespaceDocsResponse?.statusCode !== 404; + + const actualResult = indexFound + ? multiNamespaceDocsResponse?.body.docs[esBulkGetRequestIndex] + : undefined; + + const docFound = indexFound && isMgetDoc(actualResult) && actualResult.found; + + // return an error if the doc isn't found at all or the doc doesn't exist in the namespaces + if (!docFound) { + return { + tag: 'Left', + value: { + id, + type, + error: errorContent(SavedObjectsErrorHelpers.createGenericNotFoundError(type, id)), + }, + }; + } + // the following check should be redundant since we're retrieving the docs from elasticsearch but we check just to make sure + // @ts-expect-error MultiGetHit is incorrectly missing _id, _source + if (!this.rawDocExistsInNamespace(actualResult, namespace)) { + return { + tag: 'Left', + value: { + id, + type, + error: errorContent(SavedObjectsErrorHelpers.createGenericNotFoundError(type, id)), + }, + }; + } + // @ts-expect-error MultiGetHit is incorrectly missing _id, _source + namespaces = actualResult!._source.namespaces ?? [ + SavedObjectsUtils.namespaceIdToString(namespace), + ]; + const useForce = force && force === true ? true : false; + // the document is shared to more than one space and can only be deleted by force. + if (!useForce && (namespaces.length > 1 || namespaces.includes(ALL_NAMESPACES_STRING))) { + return { + tag: 'Left', + value: { + success: false, + id, + type, + error: errorContent( + SavedObjectsErrorHelpers.createBadRequestError( + `Unable to delete saved object that exists in multiple namespaces, use the "force" option to delete it anyway` + ) + ), + }, + }; + } + } + // contains all objects that passed initial preflight checks, including single namespace objects that skipped the mget call + // single namespace objects will have namespaces:undefined + const expectedResult = { + type, + id, + namespaces, + esRequestIndex: indexCounter++, + }; + + return { tag: 'Right', value: expectedResult }; + }); + return expectedBulkDeleteMultiNamespaceDocsResults; + } + + /** + * {@inheritDoc ISavedObjectsRepository.bulkDelete} + */ + async bulkDelete( + objects: SavedObjectsBulkDeleteObject[], + options: SavedObjectsBulkDeleteOptions = {} + ): Promise { + const { refresh = DEFAULT_REFRESH_SETTING, force } = options; + const namespace = normalizeNamespace(options.namespace); + const expectedBulkGetResults = this.presortObjectsByNamespaceType(objects); + const multiNamespaceDocsResponse = await this.preflightCheckForBulkDelete({ + expectedBulkGetResults, + namespace, + }); + const bulkDeleteParams: BulkDeleteParams[] = []; + + const expectedBulkDeleteMultiNamespaceDocsResults = + this.getExpectedBulkDeleteMultiNamespaceDocsResults({ + expectedBulkGetResults, + multiNamespaceDocsResponse, + namespace, + force, + }); + // bulk up the bulkDeleteParams + expectedBulkDeleteMultiNamespaceDocsResults.map((expectedResult) => { + if (isRight(expectedResult)) { + bulkDeleteParams.push({ + delete: { + _id: this._serializer.generateRawId( + namespace, + expectedResult.value.type, + expectedResult.value.id + ), + _index: this.getIndexForType(expectedResult.value.type), + ...getExpectedVersionProperties(undefined), + }, + }); + } + }); + + const bulkDeleteResponse = bulkDeleteParams.length + ? await this.client.bulk({ + refresh, + body: bulkDeleteParams, + require_alias: true, + }) + : undefined; + + // extracted to ensure consistency in the error results returned + let errorResult: BulkDeleteItemErrorResult; + const objectsToDeleteAliasesFor: ObjectToDeleteAliasesFor[] = []; + + const savedObjects = expectedBulkDeleteMultiNamespaceDocsResults.map((expectedResult) => { + if (isLeft(expectedResult)) { + return { ...expectedResult.value, success: false }; + } + const { + type, + id, + namespaces, + esRequestIndex: esBulkDeleteRequestIndex, + } = expectedResult.value; + // we assume this wouldn't happen but is needed to ensure type consistency + if (bulkDeleteResponse === undefined) { + throw new Error( + `Unexpected error in bulkDelete saved objects: bulkDeleteResponse is undefined` + ); + } + const rawResponse = Object.values( + bulkDeleteResponse.items[esBulkDeleteRequestIndex] + )[0] as NewBulkItemResponse; + + const error = getBulkOperationError(type, id, rawResponse); + if (error) { + errorResult = { success: false, type, id, error }; + return errorResult; + } + if (rawResponse.result === 'not_found') { + errorResult = { + success: false, + type, + id, + error: errorContent(SavedObjectsErrorHelpers.createGenericNotFoundError(type, id)), + }; + return errorResult; + } + + if (rawResponse.result === 'deleted') { + // `namespaces` should only exist in the expectedResult.value if the type is multi-namespace. + if (namespaces) { + objectsToDeleteAliasesFor.push({ + type, + id, + ...(namespaces.includes(ALL_NAMESPACES_STRING) + ? { namespaces: [], deleteBehavior: 'exclusive' } + : { namespaces, deleteBehavior: 'inclusive' }), + }); + } + } + const successfulResult = { + success: true, + id, + type, + }; + return successfulResult; + }); + + // Delete aliases if necessary, ensuring we don't have too many concurrent operations running. + const mapper = async ({ type, id, namespaces, deleteBehavior }: ObjectToDeleteAliasesFor) => + await deleteLegacyUrlAliases({ + mappings: this._mappings, + registry: this._registry, + client: this.client, + getIndexForType: this.getIndexForType.bind(this), + type, + id, + namespaces, + deleteBehavior, + }).catch((err) => { + this._logger.error(`Unable to delete aliases when deleting an object: ${err.message}`); + }); + await pMap(objectsToDeleteAliasesFor, mapper, { concurrency: MAX_CONCURRENT_ALIAS_DELETIONS }); + + return { statuses: [...savedObjects] }; + } + /** * {@inheritDoc ISavedObjectsRepository.deleteByNamespace} */ diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository_bulk_delete_internal_types.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository_bulk_delete_internal_types.ts new file mode 100644 index 0000000000000..93d4354d8d7e8 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository_bulk_delete_internal_types.ts @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import type { Payload } from '@hapi/boom'; +import { + BulkOperationBase, + BulkResponseItem, + ErrorCause, +} from '@elastic/elasticsearch/lib/api/types'; +import type { estypes, TransportResult } from '@elastic/elasticsearch'; +import { Either } from './internal_utils'; +import { DeleteLegacyUrlAliasesParams } from './legacy_url_aliases'; + +/** + * @internal + */ +export interface PreflightCheckForBulkDeleteParams { + expectedBulkGetResults: BulkDeleteExpectedBulkGetResult[]; + namespace?: string; +} + +/** + * @internal + */ +export interface ExpectedBulkDeleteMultiNamespaceDocsParams { + // contains the type and id of all objects to delete + expectedBulkGetResults: BulkDeleteExpectedBulkGetResult[]; + // subset of multi-namespace only expectedBulkGetResults + multiNamespaceDocsResponse: TransportResult, unknown> | undefined; + // current namespace in which the bulkDelete call is made + namespace: string | undefined; + // optional parameter used to force delete multinamespace objects that exist in more than the current space + force?: boolean; +} +/** + * @internal + */ +export interface BulkDeleteParams { + delete: Omit; +} + +/** + * @internal + */ +export type ExpectedBulkDeleteResult = Either< + { type: string; id: string; error: Payload }, + { + type: string; + id: string; + namespaces: string[]; + esRequestIndex: number; + } +>; + +/** + * @internal + */ +export interface BulkDeleteItemErrorResult { + success: boolean; + type: string; + id: string; + error: Payload; +} + +/** + * @internal + */ +export type NewBulkItemResponse = BulkResponseItem & { error: ErrorCause & { index: string } }; + +/** + * @internal + * @note Contains all documents for bulk delete, regardless of namespace type + */ +export type BulkDeleteExpectedBulkGetResult = Either< + { type: string; id: string; error: Payload }, + { type: string; id: string; version?: string; esRequestIndex?: number } +>; + +export type ObjectToDeleteAliasesFor = Pick< + DeleteLegacyUrlAliasesParams, + 'type' | 'id' | 'namespaces' | 'deleteBehavior' +>; diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/mocks/repository.mock.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/mocks/repository.mock.ts index 2cdfcf1710ad4..dc6c06c0c828d 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/mocks/repository.mock.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/mocks/repository.mock.ts @@ -16,6 +16,7 @@ const createRepositoryMock = () => { create: jest.fn(), bulkCreate: jest.fn(), bulkUpdate: jest.fn(), + bulkDelete: jest.fn(), delete: jest.fn(), bulkGet: jest.fn(), find: jest.fn(), diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/saved_objects_client.test.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/saved_objects_client.test.ts index 5829f34a6ba79..38d4e75a0c528 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/saved_objects_client.test.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/saved_objects_client.test.ts @@ -23,6 +23,8 @@ import type { SavedObjectsFindOptions, SavedObjectsUpdateObjectsSpacesObject, SavedObjectsUpdateObjectsSpacesOptions, + SavedObjectsBulkDeleteOptions, + SavedObjectsBulkDeleteObject, } from '@kbn/core-saved-objects-api-server'; import { SavedObjectsClient } from './saved_objects_client'; import { repositoryMock, savedObjectsPointInTimeFinderMock } from './mocks'; @@ -119,6 +121,22 @@ describe('SavedObjectsClient', () => { }); }); + test(`#bulkDelete`, async () => { + const returnValue: any = Symbol(); + mockRepository.bulkDelete.mockResolvedValueOnce(returnValue); + const client = new SavedObjectsClient(mockRepository); + + const objects: SavedObjectsBulkDeleteObject[] = [ + { type: 'foo', id: '1' }, + { type: 'bar', id: '2' }, + ]; + const options: SavedObjectsBulkDeleteOptions = { namespace: 'ns-1', refresh: true }; + const result = await client.bulkDelete(objects, options); + + expect(mockRepository.bulkDelete).toHaveBeenCalledWith(objects, options); + expect(result).toBe(returnValue); + }); + test(`#delete`, async () => { const returnValue: any = Symbol(); mockRepository.delete.mockResolvedValueOnce(returnValue); diff --git a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/saved_objects_client.ts b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/saved_objects_client.ts index 7c2b3a205b76d..50f78f09dd684 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-internal/src/saved_objects_client.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-internal/src/saved_objects_client.ts @@ -39,6 +39,9 @@ import type { SavedObjectsClosePointInTimeOptions, SavedObjectsCreatePointInTimeFinderOptions, SavedObjectsFindOptions, + SavedObjectsBulkDeleteObject, + SavedObjectsBulkDeleteOptions, + SavedObjectsBulkDeleteResponse, } from '@kbn/core-saved-objects-api-server'; import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-utils-server'; @@ -83,6 +86,14 @@ export class SavedObjectsClient implements SavedObjectsClientContract { return await this._repository.delete(type, id, options); } + /** {@inheritDoc SavedObjectsClientContract.bulkDelete} */ + async bulkDelete( + objects: SavedObjectsBulkDeleteObject[], + options: SavedObjectsBulkDeleteOptions = {} + ): Promise { + return await this._repository.bulkDelete(objects, options); + } + /** {@inheritDoc SavedObjectsClientContract.find} */ async find( options: SavedObjectsFindOptions diff --git a/packages/core/saved-objects/core-saved-objects-api-server-mocks/src/repository.mock.ts b/packages/core/saved-objects/core-saved-objects-api-server-mocks/src/repository.mock.ts index d950b041d2432..168f4c8de6b59 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-mocks/src/repository.mock.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-mocks/src/repository.mock.ts @@ -15,6 +15,7 @@ const create = () => { create: jest.fn(), bulkCreate: jest.fn(), bulkUpdate: jest.fn(), + bulkDelete: jest.fn(), delete: jest.fn(), bulkGet: jest.fn(), find: jest.fn(), diff --git a/packages/core/saved-objects/core-saved-objects-api-server-mocks/src/saved_objects_client.mock.ts b/packages/core/saved-objects/core-saved-objects-api-server-mocks/src/saved_objects_client.mock.ts index 75ee540cb7d8a..523e5003e650f 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server-mocks/src/saved_objects_client.mock.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server-mocks/src/saved_objects_client.mock.ts @@ -18,6 +18,7 @@ const create = () => { checkConflicts: jest.fn(), bulkUpdate: jest.fn(), delete: jest.fn(), + bulkDelete: jest.fn(), bulkGet: jest.fn(), find: jest.fn(), get: jest.fn(), diff --git a/packages/core/saved-objects/core-saved-objects-api-server/index.ts b/packages/core/saved-objects/core-saved-objects-api-server/index.ts index 1c9688a236920..fdaa5685fbde0 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server/index.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server/index.ts @@ -52,4 +52,8 @@ export type { SavedObjectsCreatePointInTimeFinderOptions, SavedObjectsFindOptions, SavedObjectsPointInTimeFinderClient, + SavedObjectsBulkDeleteObject, + SavedObjectsBulkDeleteOptions, + SavedObjectsBulkDeleteStatus, + SavedObjectsBulkDeleteResponse, } from './src/apis'; diff --git a/packages/core/saved-objects/core-saved-objects-api-server/src/apis/bulk_delete.ts b/packages/core/saved-objects/core-saved-objects-api-server/src/apis/bulk_delete.ts new file mode 100644 index 0000000000000..76d490925c580 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-api-server/src/apis/bulk_delete.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectError } from '@kbn/core-saved-objects-common'; +import type { MutatingOperationRefreshSetting, SavedObjectsBaseOptions } from './base'; + +/** + * + * @public + */ +export interface SavedObjectsBulkDeleteObject { + type: string; + id: string; +} + +/** + * @public + */ +export interface SavedObjectsBulkDeleteOptions extends SavedObjectsBaseOptions { + /** The Elasticsearch Refresh setting for this operation */ + refresh?: MutatingOperationRefreshSetting; + /** + * Force deletion of all objects that exists in multiple namespaces, applied to all objects. + */ + force?: boolean; +} + +/** + * @public + */ +export interface SavedObjectsBulkDeleteStatus { + id: string; + type: string; + /** The status of deleting the object: true for deleted, false for error */ + success: boolean; + /** Reason the object could not be deleted (success is false) */ + error?: SavedObjectError; +} + +/** + * @public + */ +export interface SavedObjectsBulkDeleteResponse { + statuses: SavedObjectsBulkDeleteStatus[]; +} diff --git a/packages/core/saved-objects/core-saved-objects-api-server/src/apis/index.ts b/packages/core/saved-objects/core-saved-objects-api-server/src/apis/index.ts index 7dc8e7ab09fc6..d311f2316885d 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server/src/apis/index.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server/src/apis/index.ts @@ -72,3 +72,9 @@ export type { SavedObjectsUpdateObjectsSpacesOptions, SavedObjectsUpdateObjectsSpacesResponseObject, } from './update_objects_spaces'; +export type { + SavedObjectsBulkDeleteObject, + SavedObjectsBulkDeleteOptions, + SavedObjectsBulkDeleteStatus, + SavedObjectsBulkDeleteResponse, +} from './bulk_delete'; diff --git a/packages/core/saved-objects/core-saved-objects-api-server/src/saved_objects_client.ts b/packages/core/saved-objects/core-saved-objects-api-server/src/saved_objects_client.ts index 82808e4024c73..cfe1f4e6a146b 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server/src/saved_objects_client.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server/src/saved_objects_client.ts @@ -41,6 +41,9 @@ import type { SavedObjectsRemoveReferencesToResponse, SavedObjectsCollectMultiNamespaceReferencesOptions, SavedObjectsBulkResponse, + SavedObjectsBulkDeleteObject, + SavedObjectsBulkDeleteOptions, + SavedObjectsBulkDeleteResponse, } from './apis'; /** @@ -151,6 +154,16 @@ export interface SavedObjectsClientContract { */ delete(type: string, id: string, options?: SavedObjectsDeleteOptions): Promise<{}>; + /** + * Deletes multiple SavedObjects batched together as a single request + * + * @param objects + * @param options + */ + bulkDelete( + objects: SavedObjectsBulkDeleteObject[], + options?: SavedObjectsBulkDeleteOptions + ): Promise; /** * Find all SavedObjects matching the search query * diff --git a/packages/core/saved-objects/core-saved-objects-api-server/src/saved_objects_repository.ts b/packages/core/saved-objects/core-saved-objects-api-server/src/saved_objects_repository.ts index 102afce9dd73d..d7d2ca57ae3a2 100644 --- a/packages/core/saved-objects/core-saved-objects-api-server/src/saved_objects_repository.ts +++ b/packages/core/saved-objects/core-saved-objects-api-server/src/saved_objects_repository.ts @@ -44,6 +44,9 @@ import type { SavedObjectsDeleteByNamespaceOptions, SavedObjectsIncrementCounterField, SavedObjectsIncrementCounterOptions, + SavedObjectsBulkDeleteOptions, + SavedObjectsBulkDeleteObject, + SavedObjectsBulkDeleteResponse, } from './apis'; /** @@ -105,6 +108,17 @@ export interface ISavedObjectsRepository { */ delete(type: string, id: string, options?: SavedObjectsDeleteOptions): Promise<{}>; + /** + * Deletes multiple documents at once + * @param {array} objects - an array of objects containing id and type + * @param {object} [options={}] + * @returns {promise} - { statuses: [{ id, type, success, error: { message } }] } + */ + bulkDelete( + objects: SavedObjectsBulkDeleteObject[], + options?: SavedObjectsBulkDeleteOptions + ): Promise; + /** * Deletes all objects from the provided namespace. * diff --git a/packages/core/saved-objects/core-saved-objects-browser-internal/src/saved_objects_client.test.ts b/packages/core/saved-objects/core-saved-objects-browser-internal/src/saved_objects_client.test.ts index 77391343cd033..6c2966ee9775f 100644 --- a/packages/core/saved-objects/core-saved-objects-browser-internal/src/saved_objects_client.test.ts +++ b/packages/core/saved-objects/core-saved-objects-browser-internal/src/saved_objects_client.test.ts @@ -308,6 +308,45 @@ describe('SavedObjectsClient', () => { }); }); + describe('#bulk_delete', () => { + const bulkDeleteDoc = { + id: 'AVwSwFxtcMV38qjDZoQg', + type: 'config', + }; + beforeEach(() => { + http.fetch.mockResolvedValue({ + statuses: [{ id: bulkDeleteDoc.id, type: bulkDeleteDoc.type, success: true }], + }); + }); + + test('deletes with an array of id, type and success status for deleted docs', async () => { + const response = savedObjectsClient.bulkDelete([bulkDeleteDoc]); + await expect(response).resolves.toHaveProperty('statuses'); + + const result = await response; + expect(result.statuses).toHaveLength(1); + expect(result.statuses[0]).toHaveProperty('success'); + }); + + test('makes HTTP call', async () => { + await savedObjectsClient.bulkDelete([bulkDeleteDoc]); + expect(http.fetch.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + "/api/saved_objects/_bulk_delete", + Object { + "body": "[{\\"type\\":\\"config\\",\\"id\\":\\"AVwSwFxtcMV38qjDZoQg\\"}]", + "method": "POST", + "query": Object { + "force": false, + }, + }, + ], + ] + `); + }); + }); + describe('#update', () => { const attributes = { foo: 'Foo', bar: 'Bar' }; const options = { version: '1' }; diff --git a/packages/core/saved-objects/core-saved-objects-browser-internal/src/saved_objects_client.ts b/packages/core/saved-objects/core-saved-objects-browser-internal/src/saved_objects_client.ts index 3a16030983fa9..dd2feed58123f 100644 --- a/packages/core/saved-objects/core-saved-objects-browser-internal/src/saved_objects_client.ts +++ b/packages/core/saved-objects/core-saved-objects-browser-internal/src/saved_objects_client.ts @@ -11,9 +11,11 @@ import type { HttpSetup, HttpFetchOptions } from '@kbn/core-http-browser'; import type { SavedObject, SavedObjectTypeIdTuple } from '@kbn/core-saved-objects-common'; import type { SavedObjectsBulkResolveResponse as SavedObjectsBulkResolveResponseServer, + SavedObjectsBulkDeleteResponse as SavedObjectsBulkDeleteResponseServer, SavedObjectsClientContract as SavedObjectsApi, SavedObjectsFindResponse as SavedObjectsFindResponseServer, SavedObjectsResolveResponse, + SavedObjectsBulkDeleteOptions, } from '@kbn/core-saved-objects-api-server'; import type { SavedObjectsClientContract, @@ -28,6 +30,7 @@ import type { SavedObjectsBulkCreateOptions, SavedObjectsBulkCreateObject, SimpleSavedObject, + SavedObjectsBulkDeleteResponse, } from '@kbn/core-saved-objects-api-browser'; import { SimpleSavedObjectImpl } from './simple_saved_object'; @@ -255,6 +258,31 @@ export class SavedObjectsClient implements SavedObjectsClientContract { return this.savedObjectsFetch(this.getPath([type, id]), { method: 'DELETE', query }); }; + public bulkDelete = async ( + objects: SavedObjectTypeIdTuple[], + options?: SavedObjectsBulkDeleteOptions + ): Promise => { + const filteredObjects = objects.map(({ type, id }) => ({ type, id })); + const queryOptions = { force: !!options?.force }; + const response = await this.performBulkDelete(filteredObjects, queryOptions); + return { + statuses: response.statuses, + }; + }; + + private async performBulkDelete( + objects: SavedObjectTypeIdTuple[], + queryOptions: { force: boolean } + ) { + const path = this.getPath(['_bulk_delete']); + const request: Promise = this.savedObjectsFetch(path, { + method: 'POST', + body: JSON.stringify(objects), + query: queryOptions, + }); + return request; + } + public find = ( options: SavedObjectsFindOptions ): Promise> => { diff --git a/packages/core/saved-objects/core-saved-objects-browser-mocks/src/saved_objects_service.mock.ts b/packages/core/saved-objects/core-saved-objects-browser-mocks/src/saved_objects_service.mock.ts index 0caa572238807..2239b94d7e2eb 100644 --- a/packages/core/saved-objects/core-saved-objects-browser-mocks/src/saved_objects_service.mock.ts +++ b/packages/core/saved-objects/core-saved-objects-browser-mocks/src/saved_objects_service.mock.ts @@ -19,6 +19,7 @@ const createStartContractMock = () => { bulkCreate: jest.fn(), bulkResolve: jest.fn(), bulkUpdate: jest.fn(), + bulkDelete: jest.fn(), delete: jest.fn(), bulkGet: jest.fn(), find: jest.fn(), diff --git a/packages/core/saved-objects/core-saved-objects-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-server-internal/index.ts index caeb029e037f7..f7d6fa7918031 100644 --- a/packages/core/saved-objects/core-saved-objects-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server-internal/index.ts @@ -22,6 +22,7 @@ export { registerBulkCreateRoute } from './src/routes/bulk_create'; export { registerBulkGetRoute } from './src/routes/bulk_get'; export { registerBulkResolveRoute } from './src/routes/bulk_resolve'; export { registerBulkUpdateRoute } from './src/routes/bulk_update'; +export { registerBulkDeleteRoute } from './src/routes/bulk_delete'; export { registerCreateRoute } from './src/routes/create'; export { registerDeleteRoute } from './src/routes/delete'; export { registerExportRoute } from './src/routes/export'; diff --git a/packages/core/saved-objects/core-saved-objects-server-internal/src/routes/bulk_delete.ts b/packages/core/saved-objects/core-saved-objects-server-internal/src/routes/bulk_delete.ts new file mode 100644 index 0000000000000..f435eadebd066 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-server-internal/src/routes/bulk_delete.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema } from '@kbn/config-schema'; +import type { InternalCoreUsageDataSetup } from '@kbn/core-usage-data-base-server-internal'; +import type { InternalSavedObjectRouter } from '../internal_types'; +import { catchAndReturnBoomErrors } from './utils'; + +interface RouteDependencies { + coreUsageData: InternalCoreUsageDataSetup; +} + +export const registerBulkDeleteRoute = ( + router: InternalSavedObjectRouter, + { coreUsageData }: RouteDependencies +) => { + router.post( + { + path: '/_bulk_delete', + validate: { + body: schema.arrayOf( + schema.object({ + type: schema.string(), + id: schema.string(), + }) + ), + query: schema.object({ + force: schema.maybe(schema.boolean()), + }), + }, + }, + catchAndReturnBoomErrors(async (context, req, res) => { + const { force } = req.query; + const usageStatsClient = coreUsageData.getClient(); + usageStatsClient.incrementSavedObjectsBulkDelete({ request: req }).catch(() => {}); + + const { savedObjects } = await context.core; + + const statuses = await savedObjects.client.bulkDelete(req.body, { force }); + return res.ok({ body: statuses }); + }) + ); +}; diff --git a/packages/core/saved-objects/core-saved-objects-server-internal/src/routes/index.ts b/packages/core/saved-objects/core-saved-objects-server-internal/src/routes/index.ts index 528793e8539bd..89d5b41dd8885 100644 --- a/packages/core/saved-objects/core-saved-objects-server-internal/src/routes/index.ts +++ b/packages/core/saved-objects/core-saved-objects-server-internal/src/routes/index.ts @@ -23,6 +23,7 @@ import { registerUpdateRoute } from './update'; import { registerBulkGetRoute } from './bulk_get'; import { registerBulkCreateRoute } from './bulk_create'; import { registerBulkUpdateRoute } from './bulk_update'; +import { registerBulkDeleteRoute } from './bulk_delete'; import { registerExportRoute } from './export'; import { registerImportRoute } from './import'; import { registerResolveImportErrorsRoute } from './resolve_import_errors'; @@ -62,6 +63,7 @@ export function registerRoutes({ registerBulkCreateRoute(router, { coreUsageData }); registerBulkResolveRoute(router, { coreUsageData }); registerBulkUpdateRoute(router, { coreUsageData }); + registerBulkDeleteRoute(router, { coreUsageData }); registerExportRoute(router, { config, coreUsageData }); registerImportRoute(router, { config, coreUsageData }); registerResolveImportErrorsRoute(router, { config, coreUsageData }); diff --git a/packages/core/usage-data/core-usage-data-base-server-internal/src/usage_stats_client.ts b/packages/core/usage-data/core-usage-data-base-server-internal/src/usage_stats_client.ts index 3a603ebfdf5f0..735a4ab261658 100644 --- a/packages/core/usage-data/core-usage-data-base-server-internal/src/usage_stats_client.ts +++ b/packages/core/usage-data/core-usage-data-base-server-internal/src/usage_stats_client.ts @@ -43,6 +43,8 @@ export interface ICoreUsageStatsClient { incrementSavedObjectsBulkUpdate(options: BaseIncrementOptions): Promise; + incrementSavedObjectsBulkDelete(options: BaseIncrementOptions): Promise; + incrementSavedObjectsCreate(options: BaseIncrementOptions): Promise; incrementSavedObjectsDelete(options: BaseIncrementOptions): Promise; diff --git a/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_stats_client.test.ts b/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_stats_client.test.ts index f00341fdad0a7..1b7d332743697 100644 --- a/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_stats_client.test.ts +++ b/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_stats_client.test.ts @@ -20,6 +20,7 @@ import { BULK_CREATE_STATS_PREFIX, BULK_GET_STATS_PREFIX, BULK_UPDATE_STATS_PREFIX, + BULK_DELETE_STATS_PREFIX, CREATE_STATS_PREFIX, DELETE_STATS_PREFIX, FIND_STATS_PREFIX, @@ -452,6 +453,81 @@ describe('CoreUsageStatsClient', () => { }); }); + describe('#incrementSavedObjectsBulkDelete', () => { + it('does not throw an error if repository incrementCounter operation fails', async () => { + const { usageStatsClient, repositoryMock } = setup(); + repositoryMock.incrementCounter.mockRejectedValue(new Error('Oh no!')); + + const request = httpServerMock.createKibanaRequest(); + await expect( + usageStatsClient.incrementSavedObjectsBulkDelete({ + request, + } as BaseIncrementOptions) + ).resolves.toBeUndefined(); + expect(repositoryMock.incrementCounter).toHaveBeenCalled(); + }); + + it('handles falsy options appropriately', async () => { + const { usageStatsClient, repositoryMock } = setup(); + + const request = httpServerMock.createKibanaRequest(); + await usageStatsClient.incrementSavedObjectsBulkDelete({ + request, + } as BaseIncrementOptions); + expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(1); + expect(repositoryMock.incrementCounter).toHaveBeenCalledWith( + CORE_USAGE_STATS_TYPE, + CORE_USAGE_STATS_ID, + [ + `${BULK_DELETE_STATS_PREFIX}.total`, + `${BULK_DELETE_STATS_PREFIX}.namespace.default.total`, + `${BULK_DELETE_STATS_PREFIX}.namespace.default.kibanaRequest.no`, + ], + incrementOptions + ); + }); + + it('handles truthy options and the default namespace string appropriately', async () => { + const { usageStatsClient, repositoryMock } = setup(DEFAULT_NAMESPACE_STRING); + + const request = httpServerMock.createKibanaRequest({ headers: firstPartyRequestHeaders }); + await usageStatsClient.incrementSavedObjectsBulkDelete({ + request, + } as BaseIncrementOptions); + expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(1); + expect(repositoryMock.incrementCounter).toHaveBeenCalledWith( + CORE_USAGE_STATS_TYPE, + CORE_USAGE_STATS_ID, + [ + `${BULK_DELETE_STATS_PREFIX}.total`, + `${BULK_DELETE_STATS_PREFIX}.namespace.default.total`, + `${BULK_DELETE_STATS_PREFIX}.namespace.default.kibanaRequest.yes`, + ], + incrementOptions + ); + }); + + it('handles a non-default space appropriately', async () => { + const { usageStatsClient, repositoryMock } = setup('foo'); + + const request = httpServerMock.createKibanaRequest(); + await usageStatsClient.incrementSavedObjectsBulkDelete({ + request, + } as BaseIncrementOptions); + expect(repositoryMock.incrementCounter).toHaveBeenCalledTimes(1); + expect(repositoryMock.incrementCounter).toHaveBeenCalledWith( + CORE_USAGE_STATS_TYPE, + CORE_USAGE_STATS_ID, + [ + `${BULK_DELETE_STATS_PREFIX}.total`, + `${BULK_DELETE_STATS_PREFIX}.namespace.custom.total`, + `${BULK_DELETE_STATS_PREFIX}.namespace.custom.kibanaRequest.no`, + ], + incrementOptions + ); + }); + }); + describe('#incrementSavedObjectsDelete', () => { it('does not throw an error if repository incrementCounter operation fails', async () => { const { usageStatsClient, repositoryMock } = setup(); diff --git a/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_stats_client.ts b/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_stats_client.ts index 49c7333bea772..3bafa2e20e562 100644 --- a/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_stats_client.ts +++ b/packages/core/usage-data/core-usage-data-server-internal/src/core_usage_stats_client.ts @@ -25,6 +25,7 @@ export const BULK_CREATE_STATS_PREFIX = 'apiCalls.savedObjectsBulkCreate'; export const BULK_GET_STATS_PREFIX = 'apiCalls.savedObjectsBulkGet'; export const BULK_RESOLVE_STATS_PREFIX = 'apiCalls.savedObjectsBulkResolve'; export const BULK_UPDATE_STATS_PREFIX = 'apiCalls.savedObjectsBulkUpdate'; +export const BULK_DELETE_STATS_PREFIX = 'apiCalls.savedObjectsBulkDelete'; export const CREATE_STATS_PREFIX = 'apiCalls.savedObjectsCreate'; export const DELETE_STATS_PREFIX = 'apiCalls.savedObjectsDelete'; export const FIND_STATS_PREFIX = 'apiCalls.savedObjectsFind'; @@ -43,6 +44,7 @@ const ALL_COUNTER_FIELDS = [ ...getFieldsForCounter(BULK_GET_STATS_PREFIX), ...getFieldsForCounter(BULK_RESOLVE_STATS_PREFIX), ...getFieldsForCounter(BULK_UPDATE_STATS_PREFIX), + ...getFieldsForCounter(BULK_DELETE_STATS_PREFIX), ...getFieldsForCounter(CREATE_STATS_PREFIX), ...getFieldsForCounter(DELETE_STATS_PREFIX), ...getFieldsForCounter(FIND_STATS_PREFIX), @@ -114,6 +116,10 @@ export class CoreUsageStatsClient implements ICoreUsageStatsClient { await this.updateUsageStats([], BULK_UPDATE_STATS_PREFIX, options); } + public async incrementSavedObjectsBulkDelete(options: BaseIncrementOptions) { + await this.updateUsageStats([], BULK_DELETE_STATS_PREFIX, options); + } + public async incrementSavedObjectsCreate(options: BaseIncrementOptions) { await this.updateUsageStats([], CREATE_STATS_PREFIX, options); } diff --git a/packages/core/usage-data/core-usage-data-server-mocks/src/core_usage_stats_client.mock.ts b/packages/core/usage-data/core-usage-data-server-mocks/src/core_usage_stats_client.mock.ts index a6e76e33057dc..6da6da69f4962 100644 --- a/packages/core/usage-data/core-usage-data-server-mocks/src/core_usage_stats_client.mock.ts +++ b/packages/core/usage-data/core-usage-data-server-mocks/src/core_usage_stats_client.mock.ts @@ -15,6 +15,7 @@ const createUsageStatsClientMock = () => incrementSavedObjectsBulkGet: jest.fn().mockResolvedValue(null), incrementSavedObjectsBulkResolve: jest.fn().mockResolvedValue(null), incrementSavedObjectsBulkUpdate: jest.fn().mockResolvedValue(null), + incrementSavedObjectsBulkDelete: jest.fn().mockResolvedValue(null), incrementSavedObjectsCreate: jest.fn().mockResolvedValue(null), incrementSavedObjectsDelete: jest.fn().mockResolvedValue(null), incrementSavedObjectsFind: jest.fn().mockResolvedValue(null), diff --git a/packages/core/usage-data/core-usage-data-server/src/core_usage_stats.ts b/packages/core/usage-data/core-usage-data-server/src/core_usage_stats.ts index aef5b657fb6f7..279d5c68cd733 100644 --- a/packages/core/usage-data/core-usage-data-server/src/core_usage_stats.ts +++ b/packages/core/usage-data/core-usage-data-server/src/core_usage_stats.ts @@ -42,6 +42,13 @@ export interface CoreUsageStats { 'apiCalls.savedObjectsBulkUpdate.namespace.custom.total'?: number; 'apiCalls.savedObjectsBulkUpdate.namespace.custom.kibanaRequest.yes'?: number; 'apiCalls.savedObjectsBulkUpdate.namespace.custom.kibanaRequest.no'?: number; + 'apiCalls.savedObjectsBulkDelete.total'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.default.total'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.default.kibanaRequest.yes'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.default.kibanaRequest.no'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.custom.total'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.custom.kibanaRequest.yes'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.custom.kibanaRequest.no'?: number; 'apiCalls.savedObjectsCreate.total'?: number; 'apiCalls.savedObjectsCreate.namespace.default.total'?: number; 'apiCalls.savedObjectsCreate.namespace.default.kibanaRequest.yes'?: number; diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 67c29f0fb747c..6e72b1d2623cf 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -338,6 +338,9 @@ export type { SavedObjectsFindOptions, SavedObjectsFindOptionsReference, SavedObjectsPitParams, + SavedObjectsBulkDeleteObject, + SavedObjectsBulkDeleteOptions, + SavedObjectsBulkDeleteResponse, } from '@kbn/core-saved-objects-api-server'; export type { SavedObjectsServiceSetup, diff --git a/src/core/server/integration_tests/saved_objects/routes/bulk_delete.test.ts b/src/core/server/integration_tests/saved_objects/routes/bulk_delete.test.ts new file mode 100644 index 0000000000000..2536915f6f068 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/routes/bulk_delete.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import supertest from 'supertest'; +import { savedObjectsClientMock } from '../../../mocks'; +import type { ICoreUsageStatsClient } from '@kbn/core-usage-data-base-server-internal'; +import { + coreUsageStatsClientMock, + coreUsageDataServiceMock, +} from '@kbn/core-usage-data-server-mocks'; +import { setupServer } from './test_utils'; +import { + registerBulkDeleteRoute, + type InternalSavedObjectsRequestHandlerContext, +} from '@kbn/core-saved-objects-server-internal'; + +type SetupServerReturn = Awaited>; + +describe('POST /api/saved_objects/_bulk_delete', () => { + let server: SetupServerReturn['server']; + let httpSetup: SetupServerReturn['httpSetup']; + let handlerContext: SetupServerReturn['handlerContext']; + let savedObjectsClient: ReturnType; + let coreUsageStatsClient: jest.Mocked; + + beforeEach(async () => { + ({ server, httpSetup, handlerContext } = await setupServer()); + savedObjectsClient = handlerContext.savedObjects.client; + + savedObjectsClient.bulkDelete.mockResolvedValue({ + statuses: [], + }); + const router = + httpSetup.createRouter('/api/saved_objects/'); + coreUsageStatsClient = coreUsageStatsClientMock.create(); + coreUsageStatsClient.incrementSavedObjectsBulkDelete.mockRejectedValue(new Error('Oh no!')); // intentionally throw this error, which is swallowed, so we can assert that the operation does not fail + const coreUsageData = coreUsageDataServiceMock.createSetupContract(coreUsageStatsClient); + registerBulkDeleteRoute(router, { coreUsageData }); + + await server.start(); + }); + + afterEach(async () => { + await server.stop(); + }); + + it('formats successful response and records usage stats', async () => { + const clientResponse = { + statuses: [ + { + id: 'abc123', + type: 'index-pattern', + success: true, + }, + ], + }; + savedObjectsClient.bulkDelete.mockImplementation(() => Promise.resolve(clientResponse)); + + const result = await supertest(httpSetup.server.listener) + .post('/api/saved_objects/_bulk_delete') + .send([ + { + id: 'abc123', + type: 'index-pattern', + }, + ]) + .expect(200); + + expect(result.body).toEqual(clientResponse); + expect(coreUsageStatsClient.incrementSavedObjectsBulkDelete).toHaveBeenCalledWith({ + request: expect.anything(), + }); + }); + + it('calls upon savedObjectClient.bulkDelete with query options', async () => { + const docs = [ + { + id: 'abc123', + type: 'index-pattern', + }, + ]; + + await supertest(httpSetup.server.listener) + .post('/api/saved_objects/_bulk_delete') + .send(docs) + .query({ force: true }) + .expect(200); + + expect(savedObjectsClient.bulkDelete).toHaveBeenCalledTimes(1); + expect(savedObjectsClient.bulkDelete).toHaveBeenCalledWith(docs, { force: true }); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts index 7581e5f3639a2..f0fdc609d8915 100644 --- a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts +++ b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy.test.ts @@ -54,6 +54,17 @@ const registerSOTypes = (setup: InternalCoreSetup) => { }, namespaceType: 'single', }); + setup.savedObjects.registerType({ + name: 'my_bulk_delete_type', + hidden: false, + mappings: { + dynamic: false, + properties: { + title: { type: 'text' }, + }, + }, + namespaceType: 'single', + }); }; describe('404s from proxies', () => { @@ -124,6 +135,7 @@ describe('404s from proxies', () => { let repository: ISavedObjectsRepository; let myOtherType: SavedObject; const myOtherTypeDocs: SavedObject[] = []; + const myBulkDeleteTypeDocs: SavedObject[] = []; beforeAll(async () => { repository = start.savedObjects.createInternalRepository(); @@ -145,6 +157,19 @@ describe('404s from proxies', () => { overwrite: true, namespace: 'default', }); + + for (let i = 1; i < 11; i++) { + myBulkDeleteTypeDocs.push({ + type: 'my_bulk_delete_type', + id: `myOtherTypeId${i}`, + attributes: { title: `MyOtherTypeTitle${i}` }, + references: [], + }); + } + await repository.bulkCreate(myBulkDeleteTypeDocs, { + overwrite: true, + namespace: 'default', + }); }); beforeEach(() => { @@ -237,6 +262,18 @@ describe('404s from proxies', () => { ); }); + it('handles `bulkDelete` requests that are successful when the proxy passes through the product header', async () => { + const docsToDelete = myBulkDeleteTypeDocs; + const bulkDeleteDocs = docsToDelete.map((doc) => ({ + id: doc.id, + type: 'my_bulk_delete_type', + })); + + const docsFound = await repository.bulkDelete(bulkDeleteDocs, { force: false }); + expect(docsFound.statuses.length).toBeGreaterThan(0); + expect(docsFound.statuses[0].success).toBe(true); + }); + it('handles `bulkGet` requests that are successful when the proxy passes through the product header', async () => { const docsToGet = myOtherTypeDocs; const docsFound = await repository.bulkGet( diff --git a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts index 6f1c2c523226c..251c7608b6299 100644 --- a/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts +++ b/src/core/server/integration_tests/saved_objects/service/lib/repository_with_proxy_utils.ts @@ -122,7 +122,8 @@ export const declarePostMgetRoute = (hapiServer: Hapi.Server, hostname: string, if ( proxyInterrupt === 'bulkGetMyType' || proxyInterrupt === 'checkConficts' || - proxyInterrupt === 'internalBulkResolve' + proxyInterrupt === 'internalBulkResolve' || + proxyInterrupt === 'bulkDeleteMyDocs' ) { return proxyResponseHandler(h, hostname, port); } else { diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts index ce8e27f318cfd..28a921dd20162 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts @@ -822,6 +822,46 @@ export function getCoreUsageCollector( 'How many times this API has been called by a non-Kibana client in a custom space.', }, }, + 'apiCalls.savedObjectsBulkDelete.total': { + type: 'long', + _meta: { description: 'How many times this API has been called.' }, + }, + 'apiCalls.savedObjectsBulkDelete.namespace.default.total': { + type: 'long', + _meta: { description: 'How many times this API has been called in the Default space.' }, + }, + 'apiCalls.savedObjectsBulkDelete.namespace.default.kibanaRequest.yes': { + type: 'long', + _meta: { + description: + 'How many times this API has been called by the Kibana client in the Default space.', + }, + }, + 'apiCalls.savedObjectsBulkDelete.namespace.default.kibanaRequest.no': { + type: 'long', + _meta: { + description: + 'How many times this API has been called by a non-Kibana client in the Default space.', + }, + }, + 'apiCalls.savedObjectsBulkDelete.namespace.custom.total': { + type: 'long', + _meta: { description: 'How many times this API has been called in a custom space.' }, + }, + 'apiCalls.savedObjectsBulkDelete.namespace.custom.kibanaRequest.yes': { + type: 'long', + _meta: { + description: + 'How many times this API has been called by the Kibana client in a custom space.', + }, + }, + 'apiCalls.savedObjectsBulkDelete.namespace.custom.kibanaRequest.no': { + type: 'long', + _meta: { + description: + 'How many times this API has been called by a non-Kibana client in a custom space.', + }, + }, // Saved Objects Management APIs 'apiCalls.savedObjectsImport.total': { type: 'long', diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_data.ts b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_data.ts index 36aac8ec0511a..2e5084f1b9ae5 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_data.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_data.ts @@ -38,6 +38,13 @@ export interface CoreUsageStats { 'apiCalls.savedObjectsBulkResolve.namespace.custom.total'?: number; 'apiCalls.savedObjectsBulkResolve.namespace.custom.kibanaRequest.yes'?: number; 'apiCalls.savedObjectsBulkResolve.namespace.custom.kibanaRequest.no'?: number; + 'apiCalls.savedObjectsBulkDelete.total'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.default.total'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.default.kibanaRequest.yes'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.default.kibanaRequest.no'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.custom.total'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.custom.kibanaRequest.yes'?: number; + 'apiCalls.savedObjectsBulkDelete.namespace.custom.kibanaRequest.no'?: number; 'apiCalls.savedObjectsBulkUpdate.total'?: number; 'apiCalls.savedObjectsBulkUpdate.namespace.default.total'?: number; 'apiCalls.savedObjectsBulkUpdate.namespace.default.kibanaRequest.yes'?: number; diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json index 5d962699345b0..30e476afd20cf 100644 --- a/src/plugins/telemetry/schema/oss_plugins.json +++ b/src/plugins/telemetry/schema/oss_plugins.json @@ -7222,6 +7222,48 @@ "description": "How many times this API has been called by a non-Kibana client in a custom space." } }, + "apiCalls.savedObjectsBulkDelete.total": { + "type": "long", + "_meta": { + "description": "How many times this API has been called." + } + }, + "apiCalls.savedObjectsBulkDelete.namespace.default.total": { + "type": "long", + "_meta": { + "description": "How many times this API has been called in the Default space." + } + }, + "apiCalls.savedObjectsBulkDelete.namespace.default.kibanaRequest.yes": { + "type": "long", + "_meta": { + "description": "How many times this API has been called by the Kibana client in the Default space." + } + }, + "apiCalls.savedObjectsBulkDelete.namespace.default.kibanaRequest.no": { + "type": "long", + "_meta": { + "description": "How many times this API has been called by a non-Kibana client in the Default space." + } + }, + "apiCalls.savedObjectsBulkDelete.namespace.custom.total": { + "type": "long", + "_meta": { + "description": "How many times this API has been called in a custom space." + } + }, + "apiCalls.savedObjectsBulkDelete.namespace.custom.kibanaRequest.yes": { + "type": "long", + "_meta": { + "description": "How many times this API has been called by the Kibana client in a custom space." + } + }, + "apiCalls.savedObjectsBulkDelete.namespace.custom.kibanaRequest.no": { + "type": "long", + "_meta": { + "description": "How many times this API has been called by a non-Kibana client in a custom space." + } + }, "apiCalls.savedObjectsImport.total": { "type": "long", "_meta": { diff --git a/test/api_integration/apis/saved_objects/bulk_delete.ts b/test/api_integration/apis/saved_objects/bulk_delete.ts new file mode 100644 index 0000000000000..5b5292b97ddde --- /dev/null +++ b/test/api_integration/apis/saved_objects/bulk_delete.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const kibanaServer = getService('kibanaServer'); + + describe('bulk_delete', () => { + before(async () => { + await kibanaServer.importExport.load( + 'test/api_integration/fixtures/kbn_archiver/saved_objects/basic.json' + ); + }); + after(async () => { + await kibanaServer.importExport.unload( + 'test/api_integration/fixtures/kbn_archiver/saved_objects/basic.json' + ); + }); + + it('should return 200 with individual responses when deleting many docs', async () => + await supertest + .post(`/api/saved_objects/_bulk_delete`) + .send([ + { + type: 'visualization', + id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', + }, + { + type: 'dashboard', + id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', + }, + ]) + .expect(200) + .then((resp) => { + expect(resp.body).to.eql({ + statuses: [ + { + success: true, + type: 'visualization', + id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab', + }, + { + success: true, + type: 'dashboard', + id: 'be3733a0-9efe-11e7-acb3-3dab96693fab', + }, + ], + }); + })); + + it('should return generic 404 when deleting an unknown doc', async () => + await supertest + .post(`/api/saved_objects/_bulk_delete`) + .send([{ type: 'dashboard', id: 'not-a-real-id' }]) + .expect(200) + .then((resp) => { + expect(resp.body).to.eql({ + statuses: [ + { + error: { + error: 'Not Found', + message: 'Saved object [dashboard/not-a-real-id] not found', + statusCode: 404, + }, + id: 'not-a-real-id', + type: 'dashboard', + success: false, + }, + ], + }); + })); + + it('should return the result of deleting valid and invalid objects in the same request', async () => + await supertest + .post(`/api/saved_objects/_bulk_delete`) + .send([ + { type: 'visualization', id: 'not-a-real-vis-id' }, + { + type: 'index-pattern', + id: '91200a00-9efd-11e7-acb3-3dab96693fab', + }, + ]) + .expect(200) + .then((resp) => { + expect(resp.body).to.eql({ + statuses: [ + { + error: { + error: 'Not Found', + message: 'Saved object [visualization/not-a-real-vis-id] not found', + statusCode: 404, + }, + id: 'not-a-real-vis-id', + type: 'visualization', + success: false, + }, + { + success: true, + type: 'index-pattern', + id: '91200a00-9efd-11e7-acb3-3dab96693fab', + }, + ], + }); + })); + }); +} diff --git a/test/api_integration/apis/saved_objects/index.ts b/test/api_integration/apis/saved_objects/index.ts index 44ee3d8d7d76b..c981add4540b3 100644 --- a/test/api_integration/apis/saved_objects/index.ts +++ b/test/api_integration/apis/saved_objects/index.ts @@ -11,6 +11,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('saved_objects', () => { loadTestFile(require.resolve('./bulk_create')); + loadTestFile(require.resolve('./bulk_delete')); loadTestFile(require.resolve('./bulk_get')); loadTestFile(require.resolve('./bulk_update')); loadTestFile(require.resolve('./create')); diff --git a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts index 75dd3fdfe8dce..ab8b03840c819 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts @@ -870,6 +870,44 @@ describe('#delete', () => { }); }); +describe('#bulkDelete', () => { + const obj1 = Object.freeze({ type: 'unknown-type', id: 'unknown-type-id-1' }); + const obj2 = Object.freeze({ type: 'unknown-type', id: 'unknown-type-id-2' }); + const namespace = 'some-ns'; + + it('redirects request to underlying base client if type is not registered', async () => { + await wrapper.bulkDelete([obj1, obj2], { namespace }); + expect(mockBaseClient.bulkDelete).toHaveBeenCalledTimes(1); + expect(mockBaseClient.bulkDelete).toHaveBeenCalledWith([obj1, obj2], { namespace }); + }); + + it('redirects request to underlying base client if type is registered', async () => { + const knownObj1 = Object.freeze({ type: 'known-type', id: 'known-type-id-1' }); + const knownObj2 = Object.freeze({ type: 'known-type', id: 'known-type-id-2' }); + const options = { namespace: 'some-ns' }; + + await wrapper.bulkDelete([knownObj1, knownObj2], options); + + expect(mockBaseClient.bulkDelete).toHaveBeenCalledTimes(1); + expect(mockBaseClient.bulkDelete).toHaveBeenCalledWith([knownObj1, knownObj2], { namespace }); + }); + + it('fails if base client fails', async () => { + const failureReason = new Error('Something bad happened...'); + mockBaseClient.bulkDelete.mockRejectedValue(failureReason); + + await expect(wrapper.bulkDelete([{ type: 'known-type', id: 'some-id' }])).rejects.toThrowError( + failureReason + ); + + expect(mockBaseClient.bulkDelete).toHaveBeenCalledTimes(1); + expect(mockBaseClient.bulkDelete).toHaveBeenCalledWith( + [{ type: 'known-type', id: 'some-id' }], + undefined + ); + }); +}); + describe('#find', () => { it('redirects request to underlying base client and does not alter response if type is not registered', async () => { const mockedResponse = { diff --git a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts index 0b9c81cc33334..e2fcfd2a6ef25 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.ts @@ -10,6 +10,8 @@ import type { SavedObject, SavedObjectsBaseOptions, SavedObjectsBulkCreateObject, + SavedObjectsBulkDeleteObject, + SavedObjectsBulkDeleteOptions, SavedObjectsBulkGetObject, SavedObjectsBulkResolveObject, SavedObjectsBulkResponse, @@ -166,6 +168,13 @@ export class EncryptedSavedObjectsClientWrapper implements SavedObjectsClientCon return await this.options.baseClient.delete(type, id, options); } + public async bulkDelete( + objects: SavedObjectsBulkDeleteObject[], + options?: SavedObjectsBulkDeleteOptions + ) { + return await this.options.baseClient.bulkDelete(objects, options); + } + public async find(options: SavedObjectsFindOptions) { return await this.handleEncryptedAttributesInBulkResponse( await this.options.baseClient.find(options), diff --git a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts index 330eebc1602b4..3ba81321ca9ee 100644 --- a/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts +++ b/x-pack/plugins/security/server/authorization/privileges/feature_privilege_builder/saved_object.ts @@ -24,6 +24,7 @@ const writeOperations: string[] = [ 'update', 'bulk_update', 'delete', + 'bulk_delete', 'share_to_space', ]; const allOperations: string[] = [...readOperations, ...writeOperations]; diff --git a/x-pack/plugins/security/server/authorization/privileges/privileges.test.ts b/x-pack/plugins/security/server/authorization/privileges/privileges.test.ts index d9c82403a582a..1a8d7814e5a38 100644 --- a/x-pack/plugins/security/server/authorization/privileges/privileges.test.ts +++ b/x-pack/plugins/security/server/authorization/privileges/privileges.test.ts @@ -109,6 +109,7 @@ describe('features', () => { actions.savedObject.get('all-savedObject-all-1', 'update'), actions.savedObject.get('all-savedObject-all-1', 'bulk_update'), actions.savedObject.get('all-savedObject-all-1', 'delete'), + actions.savedObject.get('all-savedObject-all-1', 'bulk_delete'), actions.savedObject.get('all-savedObject-all-1', 'share_to_space'), actions.savedObject.get('all-savedObject-all-2', 'bulk_get'), actions.savedObject.get('all-savedObject-all-2', 'get'), @@ -120,6 +121,7 @@ describe('features', () => { actions.savedObject.get('all-savedObject-all-2', 'update'), actions.savedObject.get('all-savedObject-all-2', 'bulk_update'), actions.savedObject.get('all-savedObject-all-2', 'delete'), + actions.savedObject.get('all-savedObject-all-2', 'bulk_delete'), actions.savedObject.get('all-savedObject-all-2', 'share_to_space'), actions.savedObject.get('all-savedObject-read-1', 'bulk_get'), actions.savedObject.get('all-savedObject-read-1', 'get'), @@ -148,6 +150,7 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-1', 'update'), actions.savedObject.get('read-savedObject-all-1', 'bulk_update'), actions.savedObject.get('read-savedObject-all-1', 'delete'), + actions.savedObject.get('read-savedObject-all-1', 'bulk_delete'), actions.savedObject.get('read-savedObject-all-1', 'share_to_space'), actions.savedObject.get('read-savedObject-all-2', 'bulk_get'), actions.savedObject.get('read-savedObject-all-2', 'get'), @@ -159,6 +162,7 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-2', 'update'), actions.savedObject.get('read-savedObject-all-2', 'bulk_update'), actions.savedObject.get('read-savedObject-all-2', 'delete'), + actions.savedObject.get('read-savedObject-all-2', 'bulk_delete'), actions.savedObject.get('read-savedObject-all-2', 'share_to_space'), actions.savedObject.get('read-savedObject-read-1', 'bulk_get'), actions.savedObject.get('read-savedObject-read-1', 'get'), @@ -301,6 +305,7 @@ describe('features', () => { actions.savedObject.get('all-savedObject-all-1', 'update'), actions.savedObject.get('all-savedObject-all-1', 'bulk_update'), actions.savedObject.get('all-savedObject-all-1', 'delete'), + actions.savedObject.get('all-savedObject-all-1', 'bulk_delete'), actions.savedObject.get('all-savedObject-all-1', 'share_to_space'), actions.savedObject.get('all-savedObject-all-2', 'bulk_get'), actions.savedObject.get('all-savedObject-all-2', 'get'), @@ -312,6 +317,7 @@ describe('features', () => { actions.savedObject.get('all-savedObject-all-2', 'update'), actions.savedObject.get('all-savedObject-all-2', 'bulk_update'), actions.savedObject.get('all-savedObject-all-2', 'delete'), + actions.savedObject.get('all-savedObject-all-2', 'bulk_delete'), actions.savedObject.get('all-savedObject-all-2', 'share_to_space'), actions.savedObject.get('all-savedObject-read-1', 'bulk_get'), actions.savedObject.get('all-savedObject-read-1', 'get'), @@ -339,6 +345,7 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-1', 'update'), actions.savedObject.get('read-savedObject-all-1', 'bulk_update'), actions.savedObject.get('read-savedObject-all-1', 'delete'), + actions.savedObject.get('read-savedObject-all-1', 'bulk_delete'), actions.savedObject.get('read-savedObject-all-1', 'share_to_space'), actions.savedObject.get('read-savedObject-all-2', 'bulk_get'), actions.savedObject.get('read-savedObject-all-2', 'get'), @@ -350,6 +357,7 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-2', 'update'), actions.savedObject.get('read-savedObject-all-2', 'bulk_update'), actions.savedObject.get('read-savedObject-all-2', 'delete'), + actions.savedObject.get('read-savedObject-all-2', 'bulk_delete'), actions.savedObject.get('read-savedObject-all-2', 'share_to_space'), actions.savedObject.get('read-savedObject-read-1', 'bulk_get'), actions.savedObject.get('read-savedObject-read-1', 'get'), @@ -427,6 +435,7 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-1', 'update'), actions.savedObject.get('read-savedObject-all-1', 'bulk_update'), actions.savedObject.get('read-savedObject-all-1', 'delete'), + actions.savedObject.get('read-savedObject-all-1', 'bulk_delete'), actions.savedObject.get('read-savedObject-all-1', 'share_to_space'), actions.savedObject.get('read-savedObject-all-2', 'bulk_get'), actions.savedObject.get('read-savedObject-all-2', 'get'), @@ -438,6 +447,7 @@ describe('features', () => { actions.savedObject.get('read-savedObject-all-2', 'update'), actions.savedObject.get('read-savedObject-all-2', 'bulk_update'), actions.savedObject.get('read-savedObject-all-2', 'delete'), + actions.savedObject.get('read-savedObject-all-2', 'bulk_delete'), actions.savedObject.get('read-savedObject-all-2', 'share_to_space'), actions.savedObject.get('read-savedObject-read-1', 'bulk_get'), actions.savedObject.get('read-savedObject-read-1', 'get'), @@ -732,6 +742,7 @@ describe('reserved', () => { actions.savedObject.get('savedObject-all-1', 'update'), actions.savedObject.get('savedObject-all-1', 'bulk_update'), actions.savedObject.get('savedObject-all-1', 'delete'), + actions.savedObject.get('savedObject-all-1', 'bulk_delete'), actions.savedObject.get('savedObject-all-1', 'share_to_space'), actions.savedObject.get('savedObject-all-2', 'bulk_get'), actions.savedObject.get('savedObject-all-2', 'get'), @@ -743,6 +754,7 @@ describe('reserved', () => { actions.savedObject.get('savedObject-all-2', 'update'), actions.savedObject.get('savedObject-all-2', 'bulk_update'), actions.savedObject.get('savedObject-all-2', 'delete'), + actions.savedObject.get('savedObject-all-2', 'bulk_delete'), actions.savedObject.get('savedObject-all-2', 'share_to_space'), actions.savedObject.get('savedObject-read-1', 'bulk_get'), actions.savedObject.get('savedObject-read-1', 'get'), @@ -862,6 +874,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -993,6 +1006,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1015,6 +1029,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1044,6 +1059,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1081,6 +1097,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1104,6 +1121,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1127,6 +1145,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1149,6 +1168,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1227,6 +1247,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1249,6 +1270,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1278,6 +1300,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1384,6 +1407,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1406,6 +1430,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1455,6 +1480,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1484,6 +1510,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1567,6 +1594,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1589,6 +1617,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1709,6 +1738,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1738,6 +1768,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1775,6 +1806,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1798,6 +1830,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1821,6 +1854,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1843,6 +1877,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1941,6 +1976,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -1970,6 +2006,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -2007,6 +2044,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -2030,6 +2068,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -2053,6 +2092,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -2075,6 +2115,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -2173,6 +2214,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), @@ -2184,6 +2226,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'update'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-licensed-sub-feature-type', 'delete'), + actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-licensed-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -2219,6 +2262,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), @@ -2230,6 +2274,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'update'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-licensed-sub-feature-type', 'delete'), + actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-licensed-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -2273,6 +2318,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), @@ -2284,6 +2330,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'update'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-licensed-sub-feature-type', 'delete'), + actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-licensed-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -2313,6 +2360,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), @@ -2324,6 +2372,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'update'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-licensed-sub-feature-type', 'delete'), + actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-licensed-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -2353,6 +2402,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), @@ -2364,6 +2414,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'update'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-licensed-sub-feature-type', 'delete'), + actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-licensed-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), @@ -2392,6 +2443,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-sub-feature-type', 'update'), actions.savedObject.get('all-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-sub-feature-type', 'delete'), + actions.savedObject.get('all-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-sub-feature-type', 'share_to_space'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_get'), actions.savedObject.get('all-licensed-sub-feature-type', 'get'), @@ -2403,6 +2455,7 @@ describe('subFeatures', () => { actions.savedObject.get('all-licensed-sub-feature-type', 'update'), actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_update'), actions.savedObject.get('all-licensed-sub-feature-type', 'delete'), + actions.savedObject.get('all-licensed-sub-feature-type', 'bulk_delete'), actions.savedObject.get('all-licensed-sub-feature-type', 'share_to_space'), actions.savedObject.get('read-sub-feature-type', 'bulk_get'), actions.savedObject.get('read-sub-feature-type', 'get'), diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts index 52702c014f0cb..bce6786f06775 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.test.ts @@ -591,6 +591,67 @@ describe('#bulkUpdate', () => { }); }); +describe('#bulkDelete', () => { + const obj1 = Object.freeze({ type: 'foo', id: 'foo-id' }); + const obj2 = Object.freeze({ type: 'bar', id: 'bar-id' }); + const namespace = 'some-ns'; + + test(`throws decorated GeneralError when hasPrivileges rejects promise`, async () => { + const objects = [obj1]; + await expectGeneralError(client.bulkDelete, { objects }); + }); + + test(`throws decorated ForbiddenError when unauthorized`, async () => { + const objects = [obj1, obj2]; + const options = { namespace }; + await expectForbiddenError(client.bulkDelete, { objects, options }); + }); + + test(`returns result of baseClient.bulkDelete when authorized`, async () => { + const apiCallReturnValue = { + statuses: [obj1, obj2].map((obj) => { + return { ...obj, success: true }; + }), + }; + clientOpts.baseClient.bulkDelete.mockReturnValue(apiCallReturnValue as any); + + const objects = [obj1, obj2]; + const options = { namespace }; + const result = await expectSuccess(client.bulkDelete, { objects, options }); + expect(result).toEqual(apiCallReturnValue); + }); + + test(`checks privileges for user, actions, and namespace`, async () => { + const objects = [obj1, obj2]; + const options = { namespace }; + await expectPrivilegeCheck(client.bulkDelete, { objects, options }, namespace); + }); + + test(`adds audit event when successful`, async () => { + const apiCallReturnValue = { + statuses: [obj1, obj2].map((obj) => { + return { ...obj, success: true }; + }), + }; + clientOpts.baseClient.bulkDelete.mockReturnValue(apiCallReturnValue as any); + + const objects = [obj1, obj2]; + const options = { namespace }; + await expectSuccess(client.bulkDelete, { objects, options }); + expect(clientOpts.auditLogger.log).toHaveBeenCalledTimes(2); + expectAuditEvent('saved_object_delete', 'success', { type: obj1.type, id: obj1.id }); + expectAuditEvent('saved_object_delete', 'success', { type: obj2.type, id: obj2.id }); + }); + + test(`adds audit event when not successful`, async () => { + clientOpts.checkSavedObjectsPrivilegesAsCurrentUser.mockRejectedValue(new Error()); + await expect(() => client.bulkDelete([obj1, obj2], { namespace })).rejects.toThrow(); + expect(clientOpts.auditLogger.log).toHaveBeenCalledTimes(2); + expectAuditEvent('saved_object_delete', 'failure', { type: obj1.type, id: obj1.id }); + expectAuditEvent('saved_object_delete', 'failure', { type: obj2.type, id: obj2.id }); + }); +}); + describe('#checkConflicts', () => { const obj1 = Object.freeze({ type: 'foo', id: 'foo-id' }); const obj2 = Object.freeze({ type: 'bar', id: 'bar-id' }); diff --git a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts index db0645992b2f7..b8e253b7f3160 100644 --- a/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts +++ b/x-pack/plugins/security/server/saved_objects/secure_saved_objects_client_wrapper.ts @@ -9,6 +9,9 @@ import type { SavedObjectReferenceWithContext, SavedObjectsBaseOptions, SavedObjectsBulkCreateObject, + SavedObjectsBulkDeleteObject, + SavedObjectsBulkDeleteOptions, + SavedObjectsBulkDeleteResponse, SavedObjectsBulkGetObject, SavedObjectsBulkResolveObject, SavedObjectsBulkUpdateObject, @@ -224,6 +227,48 @@ export class SecureSavedObjectsClientWrapper implements SavedObjectsClientContra return await this.baseClient.delete(type, id, options); } + public async bulkDelete( + objects: SavedObjectsBulkDeleteObject[], + options: SavedObjectsBulkDeleteOptions + ): Promise { + try { + const args = { objects, options }; + await this.legacyEnsureAuthorized( + this.getUniqueObjectTypes(objects), + 'bulk_delete', + options?.namespace, + { + args, + } + ); + } catch (error) { + objects.forEach(({ type, id }) => + this.auditLogger.log( + savedObjectEvent({ + action: SavedObjectAction.DELETE, + savedObject: { type, id }, + error, + }) + ) + ); + throw error; + } + const response = await this.baseClient.bulkDelete(objects, options); + response?.statuses.forEach(({ id, type, success, error }) => { + const auditEventOutcome = success === true ? 'success' : 'failure'; + const auditEventOutcomeError = error ? (error as unknown as Error) : undefined; + this.auditLogger.log( + savedObjectEvent({ + action: SavedObjectAction.DELETE, + savedObject: { type, id }, + outcome: auditEventOutcome, + error: auditEventOutcomeError, + }) + ); + }); + return response; + } + public async find(options: SavedObjectsFindOptions) { if ( this.getSpacesService() == null && diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts index 0060039f6f2ca..70a8628246b71 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts @@ -586,6 +586,41 @@ const ERROR_NAMESPACE_SPECIFIED = 'Spaces currently determines the namespaces'; }); }); + describe('#bulkDelete', () => { + test(`throws error if options.namespace is specified`, async () => { + const { client } = createSpacesSavedObjectsClient(); + + await expect( + // @ts-expect-error + client.bulkDelete(null, { namespace: 'bar' }) + ).rejects.toThrow(ERROR_NAMESPACE_SPECIFIED); + }); + + test(`supplements options with the current namespace`, async () => { + const { client, baseClient } = createSpacesSavedObjectsClient(); + const expectedReturnValue = { statuses: [{ id: 'id', type: 'type', success: true }] }; + baseClient.bulkDelete.mockReturnValue(Promise.resolve(expectedReturnValue)); + + const actualReturnValue = await client.bulkDelete([{ id: 'id', type: 'foo' }], { + force: true, + }); + + expect(actualReturnValue).toBe(expectedReturnValue); + expect(baseClient.bulkDelete).toHaveBeenCalledWith( + [ + { + id: 'id', + type: 'foo', + }, + ], + { + namespace: currentSpace.expectedNamespace, + force: true, + } + ); + }); + }); + describe('#removeReferencesTo', () => { test(`throws error if options.namespace is specified`, async () => { const { client } = createSpacesSavedObjectsClient(); diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts index a7ef2dae5b386..52ca1f2604e88 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.ts @@ -12,6 +12,8 @@ import type { SavedObject, SavedObjectsBaseOptions, SavedObjectsBulkCreateObject, + SavedObjectsBulkDeleteObject, + SavedObjectsBulkDeleteOptions, SavedObjectsBulkGetObject, SavedObjectsBulkResolveObject, SavedObjectsBulkUpdateObject, @@ -139,6 +141,17 @@ export class SpacesSavedObjectsClient implements SavedObjectsClientContract { }); } + async bulkDelete( + objects: SavedObjectsBulkDeleteObject[] = [], + options: SavedObjectsBulkDeleteOptions = {} + ) { + throwErrorIfNamespaceSpecified(options); + return await this.client.bulkDelete(objects, { + ...options, + namespace: spaceIdToNamespace(this.spaceId), + }); + } + async find(options: SavedObjectsFindOptions) { let namespaces: string[]; try { diff --git a/x-pack/test/saved_object_api_integration/common/suites/bulk_delete.ts b/x-pack/test/saved_object_api_integration/common/suites/bulk_delete.ts new file mode 100644 index 0000000000000..d0c90ccc3c255 --- /dev/null +++ b/x-pack/test/saved_object_api_integration/common/suites/bulk_delete.ts @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SuperTest } from 'supertest'; +import type { Client } from '@elastic/elasticsearch'; +import expect from '@kbn/expect'; +import type { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types'; +import { SAVED_OBJECT_TEST_CASES as CASES } from '../lib/saved_object_test_cases'; +import { SPACES } from '../lib/spaces'; +import { expectResponses, getUrlPrefix, getTestTitle } from '../lib/saved_object_test_utils'; +import { ExpectResponseBody, TestCase, TestDefinition, TestSuite, TestUser } from '../lib/types'; + +export interface BulkDeleteTestDefinition extends TestDefinition { + request: { type: string; id: string; force?: boolean }; + force?: boolean; +} +export type BulkDeleteTestSuite = TestSuite; + +export interface BulkDeleteTestCase extends TestCase { + force?: boolean; + failure?: 400 | 403 | 404; +} + +const ALIAS_DELETE_INCLUSIVE = Object.freeze({ + type: 'resolvetype', + id: 'alias-match-newid', +}); // exists in three specific spaces; deleting this should also delete the aliases that target it in the default space and space_1 +const ALIAS_DELETE_EXCLUSIVE = Object.freeze({ + type: 'resolvetype', + id: 'all_spaces', +}); // exists in all spaces; deleting this should also delete the aliases that target it in the default space and space_1 +const DOES_NOT_EXIST = Object.freeze({ type: 'dashboard', id: 'does-not-exist' }); +export const TEST_CASES: Record = Object.freeze({ + ...CASES, + ALIAS_DELETE_INCLUSIVE, + ALIAS_DELETE_EXCLUSIVE, + DOES_NOT_EXIST, +}); + +/** + * Test cases have additional properties that we don't want to send in HTTP Requests + */ +const createRequest = ({ type, id, force }: BulkDeleteTestCase) => ({ type, id, force }); + +export function bulkDeleteTestSuiteFactory(es: Client, esArchiver: any, supertest: SuperTest) { + const expectSavedObjectForbidden = expectResponses.forbiddenTypes('bulk_delete'); + const expectResponseBody = + (testCase: BulkDeleteTestCase, statusCode: 200 | 403, user?: TestUser): ExpectResponseBody => + async (response: Record) => { + if (statusCode === 403) { + await expectSavedObjectForbidden(testCase.type)(response); + } else { + // permitted + const statuses = response.body.statuses; + expect(statuses).length([testCase].length); + for (let i = 0; i < statuses.length; i++) { + const object = statuses[i]; + expect(object).to.have.keys(['id', 'type', 'success']); + if (testCase.failure) { + const { type, id } = testCase; + expect(object.type).to.eql(type); + expect(object.id).to.eql(id); + await expectResponses.permitted(object, testCase); + } else { + await es.indices.refresh({ index: '.kibana' }); // alias deletion uses refresh: false, so we need to manually refresh the index before searching + const searchResponse = await es.search({ + index: '.kibana', + body: { + size: 0, + query: { terms: { type: ['legacy-url-alias'] } }, + track_total_hits: true, + }, + }); + + const expectAliasWasDeleted = !![ALIAS_DELETE_INCLUSIVE, ALIAS_DELETE_EXCLUSIVE].find( + ({ type, id }) => testCase.type === type && testCase.id === id + ); + // Eight aliases exist and they are all deleted in the bulk operation. + // The delete behavior for multinamespace objects shared to more than one space when using force is to delete the object from all the spaces it is shared to. + expect((searchResponse.hits.total as SearchTotalHits).value).to.eql( + expectAliasWasDeleted ? 6 : 8 + ); + } + } + } + }; + + const createTestDefinitions = ( + testCases: BulkDeleteTestCase | BulkDeleteTestCase[], + forbidden: boolean, + options?: { + spaceId?: string; + responseBodyOverride?: ExpectResponseBody; + } + ): BulkDeleteTestDefinition[] => { + let cases = Array.isArray(testCases) ? testCases : [testCases]; + const responseStatusCode = forbidden ? 403 : 200; + if (forbidden) { + // override the expected result in each test case + cases = cases.map((x) => ({ ...x, failure: 403 })); + } + return cases.map((x) => ({ + title: getTestTitle(x, responseStatusCode), + responseStatusCode, + request: createRequest(x), + responseBody: options?.responseBodyOverride || expectResponseBody(x, responseStatusCode), + })); + }; + + const makeBulkDeleteTest = + (describeFn: Mocha.SuiteFunction) => (description: string, definition: BulkDeleteTestSuite) => { + const { user, spaceId = SPACES.DEFAULT.spaceId, tests } = definition; + + describeFn(description, () => { + before(() => + esArchiver.load( + 'x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces' + ) + ); + after(() => + esArchiver.unload( + 'x-pack/test/saved_object_api_integration/common/fixtures/es_archiver/saved_objects/spaces' + ) + ); + + for (const test of tests) { + it(`should return ${test.responseStatusCode} ${test.title} `, async () => { + const { type: testType, id: testId, force: testForce } = test.request; + const requestBody = [{ type: testType, id: testId }]; + const query = testForce && testForce === true ? '?force=true' : ''; + await supertest + .post(`${getUrlPrefix(spaceId)}/api/saved_objects/_bulk_delete${query}`) + .auth(user?.username, user?.password) + .send(requestBody) + .expect(test.responseStatusCode) + .then(test.responseBody); + }); + } + }); + }; + + const addTests = makeBulkDeleteTest(describe); + // @ts-ignore + addTests.only = makeBulkDeleteTest(describe.only); + + return { + addTests, + createTestDefinitions, + }; +} diff --git a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/bulk_delete.ts b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/bulk_delete.ts new file mode 100644 index 0000000000000..7a40ea564fb82 --- /dev/null +++ b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/bulk_delete.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SPACES } from '../../common/lib/spaces'; +import { testCaseFailures, getTestScenarios } from '../../common/lib/saved_object_test_utils'; +import { TestUser } from '../../common/lib/types'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + bulkDeleteTestSuiteFactory, + TEST_CASES as CASES, + BulkDeleteTestDefinition, +} from '../../common/suites/bulk_delete'; + +const { + DEFAULT: { spaceId: DEFAULT_SPACE_ID }, + SPACE_1: { spaceId: SPACE_1_ID }, + SPACE_2: { spaceId: SPACE_2_ID }, +} = SPACES; +const { fail400, fail404 } = testCaseFailures; + +const createTestCases = (spaceId: string) => { + // for each permitted (non-403) outcome, if failure !== undefined then we expect + // to receive an error; otherwise, we expect to receive a success result + const normalTypes = [ + { ...CASES.SINGLE_NAMESPACE_DEFAULT_SPACE, ...fail404(spaceId !== DEFAULT_SPACE_ID) }, + { ...CASES.SINGLE_NAMESPACE_SPACE_1, ...fail404(spaceId !== SPACE_1_ID) }, + { ...CASES.SINGLE_NAMESPACE_SPACE_2, ...fail404(spaceId !== SPACE_2_ID) }, + { ...CASES.MULTI_NAMESPACE_ALL_SPACES, ...fail400() }, + // try to delete this object again, this time using the `force` option + { ...CASES.MULTI_NAMESPACE_ALL_SPACES, force: true }, + { + ...CASES.MULTI_NAMESPACE_DEFAULT_AND_SPACE_1, + ...fail400(spaceId === DEFAULT_SPACE_ID || spaceId === SPACE_1_ID), + ...fail404(spaceId !== DEFAULT_SPACE_ID && spaceId !== SPACE_1_ID), + }, + // try to delete this object again, this time using the `force` option + { + ...CASES.MULTI_NAMESPACE_DEFAULT_AND_SPACE_1, + force: true, + ...fail404(spaceId !== DEFAULT_SPACE_ID && spaceId !== SPACE_1_ID), + }, + { ...CASES.MULTI_NAMESPACE_ONLY_SPACE_1, ...fail404(spaceId !== SPACE_1_ID) }, + { ...CASES.MULTI_NAMESPACE_ONLY_SPACE_2, ...fail404(spaceId !== SPACE_2_ID) }, + { + ...CASES.MULTI_NAMESPACE_ISOLATED_ONLY_DEFAULT_SPACE, + ...fail404(spaceId !== DEFAULT_SPACE_ID), + }, + { ...CASES.MULTI_NAMESPACE_ISOLATED_ONLY_SPACE_1, ...fail404(spaceId !== SPACE_1_ID) }, + CASES.NAMESPACE_AGNOSTIC, + { ...CASES.ALIAS_DELETE_INCLUSIVE, force: true }, + { ...CASES.ALIAS_DELETE_EXCLUSIVE, force: true }, + { ...CASES.DOES_NOT_EXIST, ...fail404() }, + ]; + const hiddenType = [{ ...CASES.HIDDEN, ...fail400() }]; // this behavior diverges from `delete`, which throws 404 + const allTypes = normalTypes.concat(hiddenType); + return { normalTypes, hiddenType, allTypes }; +}; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertestWithoutAuth'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + + const { addTests, createTestDefinitions } = bulkDeleteTestSuiteFactory(es, esArchiver, supertest); + const createTests = (spaceId: string) => { + const { normalTypes, hiddenType, allTypes } = createTestCases(spaceId); + return { + unauthorized: createTestDefinitions(allTypes, true, { spaceId }), + authorized: [ + createTestDefinitions(normalTypes, false, { spaceId }), + createTestDefinitions(hiddenType, true, { spaceId }), + ].flat(), + superuser: createTestDefinitions(allTypes, false, { spaceId }), + }; + }; + + describe('_bulk_delete', () => { + getTestScenarios().securityAndSpaces.forEach(({ spaceId, users }) => { + const suffix = ` within the ${spaceId} space`; + const { unauthorized, authorized, superuser } = createTests(spaceId); + const _addTests = (user: TestUser, tests: BulkDeleteTestDefinition[]) => { + addTests(`${user.description}${suffix}`, { user, spaceId, tests }); + }; + + [ + users.noAccess, + users.legacyAll, + users.dualRead, + users.readGlobally, + users.readAtSpace, + users.allAtOtherSpace, + ].forEach((user) => { + _addTests(user, unauthorized); + }); + [users.dualAll, users.allGlobally, users.allAtSpace].forEach((user) => { + _addTests(user, authorized); + }); + _addTests(users.superuser, superuser); + }); + }); +} diff --git a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts index 4eb0b90480314..5cbb560427281 100644 --- a/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts +++ b/x-pack/test/saved_object_api_integration/security_and_spaces/apis/index.ts @@ -21,6 +21,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./bulk_get')); loadTestFile(require.resolve('./bulk_update')); loadTestFile(require.resolve('./bulk_resolve')); + loadTestFile(require.resolve('./bulk_delete')); loadTestFile(require.resolve('./create')); loadTestFile(require.resolve('./delete')); loadTestFile(require.resolve('./export')); diff --git a/x-pack/test/saved_object_api_integration/spaces_only/apis/bulk_delete.ts b/x-pack/test/saved_object_api_integration/spaces_only/apis/bulk_delete.ts new file mode 100644 index 0000000000000..84f0c048f1a28 --- /dev/null +++ b/x-pack/test/saved_object_api_integration/spaces_only/apis/bulk_delete.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SPACES } from '../../common/lib/spaces'; +import { testCaseFailures, getTestScenarios } from '../../common/lib/saved_object_test_utils'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { bulkDeleteTestSuiteFactory, TEST_CASES as CASES } from '../../common/suites/bulk_delete'; + +const { + DEFAULT: { spaceId: DEFAULT_SPACE_ID }, + SPACE_1: { spaceId: SPACE_1_ID }, + SPACE_2: { spaceId: SPACE_2_ID }, +} = SPACES; +const { fail400, fail404 } = testCaseFailures; + +const createTestCases = (spaceId: string) => [ + { ...CASES.SINGLE_NAMESPACE_DEFAULT_SPACE, ...fail404(spaceId !== DEFAULT_SPACE_ID) }, + { ...CASES.SINGLE_NAMESPACE_SPACE_1, ...fail404(spaceId !== SPACE_1_ID) }, + { ...CASES.SINGLE_NAMESPACE_SPACE_2, ...fail404(spaceId !== SPACE_2_ID) }, + { ...CASES.MULTI_NAMESPACE_ALL_SPACES, ...fail400() }, + // // try to delete this object again, this time using the `force` option + { ...CASES.MULTI_NAMESPACE_ALL_SPACES, force: true }, + { + ...CASES.MULTI_NAMESPACE_DEFAULT_AND_SPACE_1, + ...fail400(spaceId === DEFAULT_SPACE_ID || spaceId === SPACE_1_ID), + ...fail404(spaceId !== DEFAULT_SPACE_ID && spaceId !== SPACE_1_ID), + }, + // try to delete this object again, this time using the `force` option + { + ...CASES.MULTI_NAMESPACE_DEFAULT_AND_SPACE_1, + force: true, + }, + { ...CASES.MULTI_NAMESPACE_ONLY_SPACE_1, ...fail404(spaceId !== SPACE_1_ID) }, + { ...CASES.MULTI_NAMESPACE_ONLY_SPACE_2, ...fail404(spaceId !== SPACE_2_ID) }, + { + ...CASES.MULTI_NAMESPACE_ISOLATED_ONLY_DEFAULT_SPACE, + ...fail404(spaceId !== DEFAULT_SPACE_ID), + }, + { ...CASES.MULTI_NAMESPACE_ISOLATED_ONLY_SPACE_1, ...fail404(spaceId !== SPACE_1_ID) }, + CASES.NAMESPACE_AGNOSTIC, + { ...CASES.ALIAS_DELETE_INCLUSIVE, force: true }, + { ...CASES.ALIAS_DELETE_EXCLUSIVE, force: true }, + { ...CASES.HIDDEN, ...fail400() }, + { ...CASES.DOES_NOT_EXIST, ...fail404() }, +]; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + + const { addTests, createTestDefinitions } = bulkDeleteTestSuiteFactory(es, esArchiver, supertest); + const createTests = (spaceId: string) => { + const testCases = createTestCases(spaceId); + return createTestDefinitions(testCases, false, { spaceId }); + }; + + describe('_bulk_delete', () => { + getTestScenarios().spaces.forEach(({ spaceId }) => { + const tests = createTests(spaceId); + addTests(`within the ${spaceId} space`, { spaceId, tests }); + }); + }); +} diff --git a/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts b/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts index 1be7ed754a971..e17cdc43c16f9 100644 --- a/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts +++ b/x-pack/test/saved_object_api_integration/spaces_only/apis/index.ts @@ -15,6 +15,7 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./bulk_update')); loadTestFile(require.resolve('./create')); loadTestFile(require.resolve('./delete')); + loadTestFile(require.resolve('./bulk_delete')); loadTestFile(require.resolve('./export')); loadTestFile(require.resolve('./find')); loadTestFile(require.resolve('./get')); From b5a35d74e7c36d02e58bbd6447e8ab039b31926d Mon Sep 17 00:00:00 2001 From: Rodney Norris Date: Tue, 20 Sep 2022 10:10:02 -0500 Subject: [PATCH 07/76] [Enterprise Search] delete ml inference pipeline (#141009) * [Enterprise Search] delete ml inference pipeline added a confirm modal and delete call for the delete pipeline action on the index pipelines page. * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../ml_models/delete_ml_inference_pipeline.ts | 31 +++++++++++ .../pipelines/inference_pipeline_card.tsx | 55 ++++++++++++++++++- .../search_index/pipelines/pipelines_logic.ts | 40 ++++++++++++++ 3 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/ml_models/delete_ml_inference_pipeline.ts diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/ml_models/delete_ml_inference_pipeline.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/ml_models/delete_ml_inference_pipeline.ts new file mode 100644 index 0000000000000..4abef52979380 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/ml_models/delete_ml_inference_pipeline.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { createApiLogic } from '../../../shared/api_logic/create_api_logic'; +import { HttpLogic } from '../../../shared/http'; + +export interface DeleteMlInferencePipelineApiLogicArgs { + indexName: string; + pipelineName: string; +} + +export interface DeleteMlInferencePipelineResponse { + deleted?: string; + updated?: string; +} + +export const deleteMlInferencePipeline = async ( + args: DeleteMlInferencePipelineApiLogicArgs +): Promise => { + const route = `/internal/enterprise_search/indices/${args.indexName}/ml_inference/pipeline_processors/${args.pipelineName}`; + + return await HttpLogic.values.http.delete(route); +}; + +export const DeleteMlInferencePipelineApiLogic = createApiLogic( + ['delete_ml_inference_pipeline_api_logic'], + deleteMlInferencePipeline +); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.tsx index 50a8082322817..40339a90ab423 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.tsx @@ -7,17 +7,19 @@ import React, { useState } from 'react'; -import { useValues } from 'kea'; +import { useActions, useValues } from 'kea'; import { EuiBadge, EuiButtonEmpty, + EuiConfirmModal, EuiFlexGroup, EuiFlexItem, EuiHealth, EuiPanel, EuiPopover, EuiPopoverTitle, + EuiText, EuiTextColor, EuiTitle, } from '@elastic/eui'; @@ -25,7 +27,11 @@ import { import { i18n } from '@kbn/i18n'; import { InferencePipeline } from '../../../../../../common/types/pipelines'; +import { CANCEL_BUTTON_LABEL, DELETE_BUTTON_LABEL } from '../../../../shared/constants'; import { HttpLogic } from '../../../../shared/http'; +import { IndexNameLogic } from '../index_name_logic'; + +import { PipelinesLogic } from './pipelines_logic'; export const InferencePipelineCard: React.FC = ({ pipelineName, @@ -34,7 +40,10 @@ export const InferencePipelineCard: React.FC = ({ modelType, }) => { const { http } = useValues(HttpLogic); + const { indexName } = useValues(IndexNameLogic); const [isPopOverOpen, setIsPopOverOpen] = useState(false); + const [showConfirmDelete, setShowConfirmDelete] = useState(false); + const { deleteMlPipeline } = useActions(PipelinesLogic); const deployedText = i18n.translate('xpack.enterpriseSearch.inferencePipelineCard.isDeployed', { defaultMessage: 'Deployed', @@ -100,7 +109,13 @@ export const InferencePipelineCard: React.FC = ({
- + setShowConfirmDelete(true)} + > {i18n.translate( 'xpack.enterpriseSearch.inferencePipelineCard.action.delete', { defaultMessage: 'Delete pipeline' } @@ -137,6 +152,42 @@ export const InferencePipelineCard: React.FC = ({ + {showConfirmDelete && ( + setShowConfirmDelete(false)} + onConfirm={() => { + setShowConfirmDelete(false); + deleteMlPipeline({ + indexName, + pipelineName, + }); + }} + title={i18n.translate( + 'xpack.enterpriseSearch.inferencePipelineCard.deleteConfirm.title', + { defaultMessage: 'Delete Pipeline' } + )} + buttonColor="danger" + cancelButtonText={CANCEL_BUTTON_LABEL} + confirmButtonText={DELETE_BUTTON_LABEL} + defaultFocusedButton="confirm" + maxWidth + > + +

+ {i18n.translate( + 'xpack.enterpriseSearch.inferencePipelineCard.deleteConfirm.description', + { + defaultMessage: + 'You are removing the pipeline "{pipelineName}" from the Machine Learning Inference Pipeline and deleting it.', + values: { + pipelineName, + }, + } + )} +

+ + + )} ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/pipelines_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/pipelines_logic.ts index 290b4e469764f..e0ca6574b63d6 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/pipelines_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/pipelines_logic.ts @@ -47,6 +47,11 @@ import { FetchIndexApiResponse, } from '../../../api/index/fetch_index_api_logic'; import { CreateMlInferencePipelineApiLogic } from '../../../api/ml_models/create_ml_inference_pipeline'; +import { + DeleteMlInferencePipelineApiLogic, + DeleteMlInferencePipelineApiLogicArgs, + DeleteMlInferencePipelineResponse, +} from '../../../api/ml_models/delete_ml_inference_pipeline'; import { FetchMlInferencePipelineProcessorsApiLogic } from '../../../api/pipelines/fetch_ml_inference_pipeline_processors'; import { isApiIndex, isConnectorIndex, isCrawlerIndex } from '../../../utils/indices'; @@ -68,6 +73,18 @@ type PipelinesActions = Pick< CreateCustomPipelineApiLogicArgs, CreateCustomPipelineApiLogicResponse >['apiSuccess']; + deleteMlPipeline: Actions< + DeleteMlInferencePipelineApiLogicArgs, + DeleteMlInferencePipelineResponse + >['makeRequest']; + deleteMlPipelineError: Actions< + DeleteMlInferencePipelineApiLogicArgs, + DeleteMlInferencePipelineResponse + >['apiError']; + deleteMlPipelineSuccess: Actions< + DeleteMlInferencePipelineApiLogicArgs, + DeleteMlInferencePipelineResponse + >['apiSuccess']; fetchCustomPipeline: Actions< FetchCustomPipelineApiLogicArgs, FetchCustomPipelineApiLogicResponse @@ -129,6 +146,12 @@ export const PipelinesLogic = kea { actions.fetchMlInferenceProcessors({ indexName: values.index.name }); }, + deleteMlPipelineError: (error) => flashAPIErrors(error), + deleteMlPipelineSuccess: (value) => { + if (value.deleted) { + flashSuccessToast( + i18n.translate( + 'xpack.enterpriseSearch.content.indices.pipelines.successToastDeleteMlPipeline.title', + { + defaultMessage: 'Deleted machine learning inference pipeline "{pipelineName}"', + values: { + pipelineName: value.deleted, + }, + } + ) + ); + } + actions.fetchMlInferenceProcessors({ indexName: values.index.name }); + }, fetchIndexApiSuccess: (index) => { if (!values.showModal) { // Don't do this when the modal is open to avoid overwriting the values while editing From 49b39dff2a7d13d8f3a1a3ec1c7c09037dee06da Mon Sep 17 00:00:00 2001 From: Jordan <51442161+JordanSh@users.noreply.github.com> Date: Tue, 20 Sep 2022 18:13:58 +0300 Subject: [PATCH 08/76] [Cloud Posture] Tables shared values (#140295) --- .../public/common/constants.ts | 36 +++++++++ .../csp_inline_description_list.tsx | 48 ++++++++++++ .../deployment_type_select.tsx | 20 +---- .../latest_findings_table.test.tsx | 2 +- .../resource_findings_container.tsx | 52 ++++++++++++- .../resource_findings_table.tsx | 10 --- .../use_resource_findings.ts | 42 +++++++++-- .../public/pages/rules/index.tsx | 75 +++++++++++++++---- .../public/pages/rules/rules.test.tsx | 16 +++- .../public/pages/rules/test_subjects.ts | 3 + .../translations/translations/fr-FR.json | 2 - .../translations/translations/ja-JP.json | 2 - .../translations/translations/zh-CN.json | 2 - 13 files changed, 249 insertions(+), 61 deletions(-) create mode 100644 x-pack/plugins/cloud_security_posture/public/components/csp_inline_description_list.tsx diff --git a/x-pack/plugins/cloud_security_posture/public/common/constants.ts b/x-pack/plugins/cloud_security_posture/public/common/constants.ts index 8033d16d476c8..597e037cb68d8 100644 --- a/x-pack/plugins/cloud_security_posture/public/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/public/common/constants.ts @@ -6,8 +6,44 @@ */ import { euiPaletteForStatus } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { CLOUDBEAT_EKS, CLOUDBEAT_VANILLA } from '../../common/constants'; const [success, warning, danger] = euiPaletteForStatus(3); export const statusColors = { success, warning, danger }; export const CSP_MOMENT_FORMAT = 'MMMM D, YYYY @ HH:mm:ss.SSS'; + +export type CloudPostureIntegrations = typeof cloudPostureIntegrations; + +export const cloudPostureIntegrations = { + kspm: { + policyTemplate: 'kspm', + name: i18n.translate('xpack.csp.kspmIntegration.integration.nameTitle', { + defaultMessage: 'Kubernetes Security Posture Management', + }), + shortName: i18n.translate('xpack.csp.kspmIntegration.integration.shortNameTitle', { + defaultMessage: 'KSPM', + }), + options: [ + { + type: CLOUDBEAT_VANILLA, + name: i18n.translate('xpack.csp.kspmIntegration.vanillaOption.nameTitle', { + defaultMessage: 'Unmanaged Kubernetes', + }), + benchmark: i18n.translate('xpack.csp.kspmIntegration.vanillaOption.benchmarkTitle', { + defaultMessage: 'CIS Kubernetes', + }), + }, + { + type: CLOUDBEAT_EKS, + name: i18n.translate('xpack.csp.kspmIntegration.eksOption.nameTitle', { + defaultMessage: 'EKS (Elastic Kubernetes Service)', + }), + benchmark: i18n.translate('xpack.csp.kspmIntegration.eksOption.benchmarkTitle', { + defaultMessage: 'CIS EKS', + }), + }, + ], + }, +} as const; diff --git a/x-pack/plugins/cloud_security_posture/public/components/csp_inline_description_list.tsx b/x-pack/plugins/cloud_security_posture/public/components/csp_inline_description_list.tsx new file mode 100644 index 0000000000000..7acb06d2eabb6 --- /dev/null +++ b/x-pack/plugins/cloud_security_posture/public/components/csp_inline_description_list.tsx @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiDescriptionList, useEuiTheme, type EuiDescriptionListProps } from '@elastic/eui'; + +const getModifiedTitlesListItems = (listItems: EuiDescriptionListProps['listItems']) => + listItems + ?.filter((item) => !!item?.title && !!item?.description) + .map((item) => ({ ...item, title: `${item.title}:` })); + +// eui size m is 12px which is too small, and next after it is base which is 16px which is too big +const fontSize = '1rem'; + +export const CspInlineDescriptionList = ({ + listItems, +}: { + listItems: EuiDescriptionListProps['listItems']; +}) => { + const { euiTheme } = useEuiTheme(); + const modifiedTitlesListItems = getModifiedTitlesListItems(listItems); + + return ( + + ); +}; diff --git a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/deployment_type_select.tsx b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/deployment_type_select.tsx index 3085f9d83ea86..b01b5073a0e1b 100644 --- a/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/deployment_type_select.tsx +++ b/x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/deployment_type_select.tsx @@ -16,7 +16,7 @@ import { EuiDescribedFormGroup, } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; -import { i18n } from '@kbn/i18n'; +import { cloudPostureIntegrations } from '../../common/constants'; import { CLOUDBEAT_EKS, CLOUDBEAT_VANILLA } from '../../../common/constants'; export type InputType = typeof CLOUDBEAT_EKS | typeof CLOUDBEAT_VANILLA; @@ -27,22 +27,8 @@ interface Props { isDisabled?: boolean; } -const kubeDeployOptions: Array> = [ - { - value: CLOUDBEAT_VANILLA, - label: i18n.translate( - 'xpack.csp.createPackagePolicy.stepConfigure.integrationSettingsSection.vanillaKubernetesDeploymentOption', - { defaultMessage: 'Unmanaged Kubernetes' } - ), - }, - { - value: CLOUDBEAT_EKS, - label: i18n.translate( - 'xpack.csp.createPackagePolicy.stepConfigure.integrationSettingsSection.eksKubernetesDeploymentOption', - { defaultMessage: 'EKS (Elastic Kubernetes Service)' } - ), - }, -]; +const kubeDeployOptions: Array> = + cloudPostureIntegrations.kspm.options.map((o) => ({ value: o.type, label: o.name })); const KubernetesDeploymentFieldLabel = () => ( ({ - id: chance.word(), cluster_id: chance.guid(), + id: chance.word(), result: { expected: { source: {}, diff --git a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_container.tsx b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_container.tsx index aa86a586faf7e..75997efaf6294 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_container.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_container.tsx @@ -5,11 +5,17 @@ * 2.0. */ import React from 'react'; -import { EuiSpacer, EuiButtonEmpty } from '@elastic/eui'; +import { + EuiSpacer, + EuiButtonEmpty, + EuiPageHeader, + type EuiDescriptionListProps, +} from '@elastic/eui'; import { Link, useParams } from 'react-router-dom'; import { FormattedMessage } from '@kbn/i18n-react'; import { generatePath } from 'react-router-dom'; import { i18n } from '@kbn/i18n'; +import { CspInlineDescriptionList } from '../../../../components/csp_inline_description_list'; import type { Evaluation } from '../../../../../common/types'; import { CspFinding } from '../../../../../common/schemas/csp_finding'; import { CloudPosturePageTitle } from '../../../../components/cloud_posture_page_title'; @@ -54,6 +60,32 @@ const BackToResourcesButton = () => ( ); +const getResourceFindingSharedValues = (sharedValues: { + resourceId: string; + resourceSubType: string; + resourceName: string; + clusterId: string; +}): EuiDescriptionListProps['listItems'] => [ + { + title: i18n.translate('xpack.csp.findings.resourceFindingsSharedValues.resourceTypeTitle', { + defaultMessage: 'Resource Type', + }), + description: sharedValues.resourceSubType, + }, + { + title: i18n.translate('xpack.csp.findings.resourceFindingsSharedValues.resourceIdTitle', { + defaultMessage: 'Resource ID', + }), + description: sharedValues.resourceId, + }, + { + title: i18n.translate('xpack.csp.findings.resourceFindingsSharedValues.clusterIdTitle', { + defaultMessage: 'Cluster ID', + }), + description: sharedValues.clusterId, + }, +]; + export const ResourceFindings = ({ dataView }: FindingsBaseProps) => { const params = useParams<{ resourceId: string }>(); const getPersistedDefaultQuery = usePersistedQuery(getDefaultQuery); @@ -114,14 +146,28 @@ export const ResourceFindings = ({ dataView }: FindingsBaseProps) => { title={i18n.translate( 'xpack.csp.findings.resourceFindings.resourceFindingsPageTitle', { - defaultMessage: '{resourceId} - Findings', - values: { resourceId: params.resourceId }, + defaultMessage: '{resourceName} - Findings', + values: { resourceName: resourceFindings.data?.resourceName }, } )} /> } /> + + ) + } + /> {error && } {!error && ( diff --git a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_table.tsx b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_table.tsx index c8dddf5ebd84e..07c84000fb607 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_table.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_table.tsx @@ -49,21 +49,11 @@ const ResourceFindingsTableComponent = ({ ] = useMemo( () => [ getExpandColumn({ onClick: setSelectedFinding }), - baseFindingsColumns['resource.id'], createColumnWithFilters(baseFindingsColumns['result.evaluation'], { onAddFilter }), - createColumnWithFilters( - { ...baseFindingsColumns['resource.sub_type'], sortable: false }, - { onAddFilter } - ), - createColumnWithFilters( - { ...baseFindingsColumns['resource.name'], sortable: false }, - { onAddFilter } - ), createColumnWithFilters(baseFindingsColumns['rule.name'], { onAddFilter }), createColumnWithFilters(baseFindingsColumns['rule.benchmark.name'], { onAddFilter }), baseFindingsColumns['rule.section'], baseFindingsColumns['rule.tags'], - createColumnWithFilters(baseFindingsColumns.cluster_id, { onAddFilter }), baseFindingsColumns['@timestamp'], ], [onAddFilter] diff --git a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/use_resource_findings.ts b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/use_resource_findings.ts index d5b77f0e732d2..39cc7e985b77d 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/use_resource_findings.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/use_resource_findings.ts @@ -34,11 +34,14 @@ export interface ResourceFindingsQuery { } type ResourceFindingsRequest = IKibanaSearchRequest; -type ResourceFindingsResponse = IKibanaSearchResponse>; +type ResourceFindingsResponse = IKibanaSearchResponse< + estypes.SearchResponse +>; -interface Aggs { - count: estypes.AggregationsMultiBucketAggregateBase; -} +export type ResourceFindingsResponseAggs = Record< + 'count' | 'clusterId' | 'resourceSubType' | 'resourceName', + estypes.AggregationsMultiBucketAggregateBase +>; const getResourceFindingsQuery = ({ query, @@ -60,7 +63,18 @@ const getResourceFindingsQuery = ({ }, sort: [{ [sort.field]: sort.direction }], pit: { id: pitId }, - aggs: getFindingsCountAggQuery(), + aggs: { + ...getFindingsCountAggQuery(), + clusterId: { + terms: { field: 'cluster_id' }, + }, + resourceSubType: { + terms: { field: 'resource.sub_type' }, + }, + resourceName: { + terms: { field: 'resource.name' }, + }, + }, }, ignore_unavailable: false, }); @@ -90,13 +104,18 @@ export const useResourceFindings = (options: UseResourceFindingsOptions) => { }: ResourceFindingsResponse) => { if (!aggregations) throw new Error('expected aggregations to exists'); - if (!Array.isArray(aggregations?.count.buckets)) - throw new Error('expected buckets to be an array'); + assertNonEmptyArray(aggregations.count.buckets); + assertNonEmptyArray(aggregations.clusterId.buckets); + assertNonEmptyArray(aggregations.resourceSubType.buckets); + assertNonEmptyArray(aggregations.resourceName.buckets); return { page: hits.hits.map((hit) => hit._source!), total: number.is(hits.total) ? hits.total : 0, count: getAggregationCount(aggregations.count.buckets), + clusterId: getFirstBucketKey(aggregations.clusterId.buckets), + resourceSubType: getFirstBucketKey(aggregations.resourceSubType.buckets), + resourceName: getFirstBucketKey(aggregations.resourceName.buckets), newPitId: newPitId!, }; }, @@ -110,3 +129,12 @@ export const useResourceFindings = (options: UseResourceFindingsOptions) => { } ); }; + +function assertNonEmptyArray(arr: unknown): asserts arr is T[] { + if (!Array.isArray(arr) || arr.length === 0) { + throw new Error('expected a non empty array'); + } +} + +const getFirstBucketKey = (buckets: estypes.AggregationsStringRareTermsBucketKeys[]) => + buckets[0].key; diff --git a/x-pack/plugins/cloud_security_posture/public/pages/rules/index.tsx b/x-pack/plugins/cloud_security_posture/public/pages/rules/index.tsx index 57e3da3658a2e..af42704b9b9f3 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/rules/index.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/rules/index.tsx @@ -7,10 +7,18 @@ import React, { useContext, useMemo } from 'react'; import { generatePath, Link, type RouteComponentProps } from 'react-router-dom'; -import { EuiTextColor, EuiButtonEmpty, EuiFlexGroup, EuiPageHeader, EuiSpacer } from '@elastic/eui'; +import { + EuiButtonEmpty, + type EuiDescriptionListProps, + EuiFlexGroup, + EuiPageHeader, + EuiSpacer, +} from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import { pagePathGetters } from '@kbn/fleet-plugin/public'; import { i18n } from '@kbn/i18n'; +import { PackagePolicy } from '@kbn/fleet-plugin/common'; +import { CspInlineDescriptionList } from '../../components/csp_inline_description_list'; import { CloudPosturePageTitle } from '../../components/cloud_posture_page_title'; import type { BreadcrumbEntry } from '../../common/navigation/types'; import { RulesContainer, type PageUrlParams } from './rules_container'; @@ -20,6 +28,8 @@ import { useCspIntegrationInfo } from './use_csp_integration'; import { useKibana } from '../../common/hooks/use_kibana'; import { CloudPosturePage } from '../../components/cloud_posture_page'; import { SecuritySolutionContext } from '../../application/security_solution_context'; +import { CloudPostureIntegrations, cloudPostureIntegrations } from '../../common/constants'; +import * as TEST_SUBJECTS from './test_subjects'; const getRulesBreadcrumbs = ( name?: string, @@ -41,12 +51,55 @@ const getRulesBreadcrumbs = ( return breadCrumbs; }; +const isPolicyTemplate = (name: unknown): name is keyof CloudPostureIntegrations => + typeof name === 'string' && name in cloudPostureIntegrations; + +const getRulesSharedValues = ( + packageInfo?: PackagePolicy +): NonNullable => { + const enabledInput = packageInfo?.inputs.find((input) => input.enabled); + if (!enabledInput || !isPolicyTemplate(enabledInput.policy_template)) return []; + + const integration = cloudPostureIntegrations[enabledInput.policy_template]; + const enabledIntegrationOption = integration.options.find( + (option) => option.type === enabledInput.type + ); + + const values = [ + { + title: i18n.translate('xpack.csp.rules.rulesPageSharedValues.integrationTitle', { + defaultMessage: 'Integration', + }), + description: integration.shortName, + }, + ]; + + if (!enabledIntegrationOption) return values; + + values.push( + { + title: i18n.translate('xpack.csp.rules.rulesPageSharedValues.deploymentTypeTitle', { + defaultMessage: 'Deployment Type', + }), + description: enabledIntegrationOption.name, + }, + { + title: i18n.translate('xpack.csp.rules.rulesPageSharedValues.benchmarkTitle', { + defaultMessage: 'Benchmark', + }), + description: enabledIntegrationOption.benchmark, + } + ); + + return values; +}; + export const Rules = ({ match: { params } }: RouteComponentProps) => { const { http } = useKibana().services; const integrationInfo = useCspIntegrationInfo(params); const securitySolutionContext = useContext(SecuritySolutionContext); - const [packageInfo, agentInfo] = integrationInfo.data || []; + const [packageInfo] = integrationInfo.data || []; const breadcrumbs = useMemo( () => @@ -56,6 +109,8 @@ export const Rules = ({ match: { params } }: RouteComponentProps) useCspBreadcrumbs(breadcrumbs); + const sharedValues = getRulesSharedValues(packageInfo); + return ( ) } description={ - packageInfo?.package && - agentInfo?.name && ( - - - + sharedValues.length && ( +
+ +
) } bottomBorder diff --git a/x-pack/plugins/cloud_security_posture/public/pages/rules/rules.test.tsx b/x-pack/plugins/cloud_security_posture/public/pages/rules/rules.test.tsx index 70d803ff4ac5e..45e19b9fba5bd 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/rules/rules.test.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/rules/rules.test.tsx @@ -105,6 +105,18 @@ describe('', () => { package: { title: 'my package', }, + inputs: [ + { + enabled: true, + policy_template: 'kspm', + type: 'cloudbeat/cis_k8s', + }, + { + enabled: false, + policy_template: 'kspm', + type: 'cloudbeat/cis_eks', + }, + ], }, { name: 'my agent' }, ], @@ -114,9 +126,7 @@ describe('', () => { render(); - expect( - await screen.findByText(`${response.data?.[0]?.package?.title}, ${response.data?.[1].name}`) - ).toBeInTheDocument(); expect(await screen.findByTestId(TEST_SUBJECTS.CSP_RULES_CONTAINER)).toBeInTheDocument(); + expect(await screen.findByTestId(TEST_SUBJECTS.CSP_RULES_SHARED_VALUES)).toBeInTheDocument(); }); }); diff --git a/x-pack/plugins/cloud_security_posture/public/pages/rules/test_subjects.ts b/x-pack/plugins/cloud_security_posture/public/pages/rules/test_subjects.ts index 9092bc1707fdc..a40ccff5b5e6e 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/rules/test_subjects.ts +++ b/x-pack/plugins/cloud_security_posture/public/pages/rules/test_subjects.ts @@ -6,6 +6,9 @@ */ export const CSP_RULES_CONTAINER = 'csp_rules_container'; +export const CSP_RULES_SHARED_VALUES = 'csp_rules_shared_values'; +export const CSP_RULES_TABLE_ITEM_SWITCH = 'csp_rules_table_item_switch'; +export const CSP_RULES_SAVE_BUTTON = 'csp_rules_table_save_button'; export const CSP_RULES_TABLE = 'csp_rules_table'; export const CSP_RULES_TABLE_ROW_ITEM_NAME = 'csp_rules_table_row_item_name'; export const CSP_RULES_FLYOUT_CONTAINER = 'csp_rules_flyout_container'; diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index c9b80baccbf82..c19761a44a557 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -9535,10 +9535,8 @@ "xpack.csp.findings.findingsTableCell.addFilterButton": "Ajouter un filtre {field}", "xpack.csp.findings.findingsTableCell.addNegateFilterButton": "Ajouter un filtre {field} négatif", "xpack.csp.findings.latestFindings.bottomBarLabel": "Voici les {maxItems} premiers résultats correspondant à votre recherche. Veuillez l'affiner pour en voir davantage.", - "xpack.csp.findings.resourceFindings.resourceFindingsPageTitle": "{resourceId} - Résultats", "xpack.csp.rules.header.rulesCountLabel": "{count, plural, one { règle} other { règles}}", "xpack.csp.rules.header.totalRulesCount": "Affichage des {rules}", - "xpack.csp.rules.rulePageHeader.pageDescriptionTitle": "{integrationType}, {agentPolicyName}", "xpack.csp.rules.rulePageHeader.pageHeaderTitle": "Règles - {integrationName}", "xpack.csp.benchmarks.benchmarkEmptyState.integrationsNotFoundTitle": "Aucune intégration Benchmark trouvée", "xpack.csp.benchmarks.benchmarkEmptyState.integrationsNotFoundWithFiltersTitle": "Nous n'avons trouvé aucune intégration Benchmark avec les filtres ci-dessus.", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 7dce528b507f4..140eb400615e5 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -9522,10 +9522,8 @@ "xpack.csp.findings.findingsTableCell.addFilterButton": "{field}フィルターを追加", "xpack.csp.findings.findingsTableCell.addNegateFilterButton": "{field}否定フィルターを追加", "xpack.csp.findings.latestFindings.bottomBarLabel": "これらは検索条件に一致した初めの{maxItems}件の調査結果です。他の結果を表示するには検索条件を絞ってください。", - "xpack.csp.findings.resourceFindings.resourceFindingsPageTitle": "{resourceId} - 調査結果", "xpack.csp.rules.header.rulesCountLabel": "{count, plural, other {個のルール}}", "xpack.csp.rules.header.totalRulesCount": "{rules}を表示しています", - "xpack.csp.rules.rulePageHeader.pageDescriptionTitle": "{integrationType}, {agentPolicyName}", "xpack.csp.rules.rulePageHeader.pageHeaderTitle": "ルール - {integrationName}", "xpack.csp.benchmarks.benchmarkEmptyState.integrationsNotFoundTitle": "ベンチマーク統合が見つかりません", "xpack.csp.benchmarks.benchmarkEmptyState.integrationsNotFoundWithFiltersTitle": "上記のフィルターでベンチマーク統合が見つかりませんでした。", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 8c7e8ed8e7d30..10aa7ea3f37e9 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -9537,10 +9537,8 @@ "xpack.csp.findings.findingsTableCell.addFilterButton": "添加 {field} 筛选", "xpack.csp.findings.findingsTableCell.addNegateFilterButton": "添加 {field} 作废筛选", "xpack.csp.findings.latestFindings.bottomBarLabel": "这些是匹配您的搜索的前 {maxItems} 个结果，请优化搜索以查看其他结果。", - "xpack.csp.findings.resourceFindings.resourceFindingsPageTitle": "{resourceId} - 结果", "xpack.csp.rules.header.rulesCountLabel": "{count, plural, other { 规则}}", "xpack.csp.rules.header.totalRulesCount": "正在显示 {rules}", - "xpack.csp.rules.rulePageHeader.pageDescriptionTitle": "{integrationType}，{agentPolicyName}", "xpack.csp.rules.rulePageHeader.pageHeaderTitle": "规则 - {integrationName}", "xpack.csp.benchmarks.benchmarkEmptyState.integrationsNotFoundTitle": "找不到基准集成", "xpack.csp.benchmarks.benchmarkEmptyState.integrationsNotFoundWithFiltersTitle": "使用上述筛选，我们无法找到任何基准集成。", From b93dc5f8c213130af5f30d14d4457a20acef9cb9 Mon Sep 17 00:00:00 2001 From: Katerina Patticha Date: Tue, 20 Sep 2022 17:17:33 +0200 Subject: [PATCH 09/76] [APM] [experimental] Service metrics on inventory page (#140868) * Service metrics on inventory page * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * Create seperate query for service aggregegated transaction stats * Fix typo and eslint errors * Fix failed transaction rate function * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * Use service metrics for details stats * Fix throughput calculation * Fix types * Update snapshot * Clean up code * Create a helper to get aggregated metrics * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * Add test for configuration * Rename getAggregatedMetrics to getServiceInventorySearchSource * query on metricset.name * Address PR feedback * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * Sort servcice metrics bucker by `value_count` * fix broken path * fix types Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- docs/settings/apm-settings.asciidoc | 3 + .../elasticsearch_fieldnames.test.ts.snap | 18 ++ .../apm/common/elasticsearch_fieldnames.ts | 3 + x-pack/plugins/apm/server/index.ts | 3 + .../index.ts | 7 +- .../get_service_inventory_search_source.ts | 45 ++++ ...get_has_aggregated_service_metrics.test.ts | 124 +++++++++ .../lib/helpers/service_metrics/index.ts | 83 ++++++ .../helpers/transaction_error_rate.test.ts | 54 ++++ .../lib/helpers/transaction_error_rate.ts | 30 ++- ...et_service_aggregated_transaction_stats.ts | 174 +++++++++++++ .../get_services/get_services_items.ts | 8 +- .../routes/services/get_services/index.ts | 3 + ...regated_transaction_detailed_statistics.ts | 244 ++++++++++++++++++ ...service_transaction_detailed_statistics.ts | 50 +++- .../get_services_detailed_statistics/index.ts | 46 ++-- .../server/routes/services/queries.test.ts | 1 + .../apm/server/routes/services/route.ts | 34 ++- 18 files changed, 886 insertions(+), 44 deletions(-) create mode 100644 x-pack/plugins/apm/server/lib/helpers/get_service_inventory_search_source.ts create mode 100644 x-pack/plugins/apm/server/lib/helpers/service_metrics/get_has_aggregated_service_metrics.test.ts create mode 100644 x-pack/plugins/apm/server/lib/helpers/service_metrics/index.ts create mode 100644 x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.test.ts create mode 100644 x-pack/plugins/apm/server/routes/services/get_services/get_service_aggregated_transaction_stats.ts create mode 100644 x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_aggregated_transaction_detailed_statistics.ts diff --git a/docs/settings/apm-settings.asciidoc b/docs/settings/apm-settings.asciidoc index de5e8c686c61b..fba7a32e17f1b 100644 --- a/docs/settings/apm-settings.asciidoc +++ b/docs/settings/apm-settings.asciidoc @@ -77,6 +77,9 @@ Maximum number of child items displayed when viewing trace details. Defaults to `xpack.observability.annotations.index` {ess-icon}:: Index name where Observability annotations are stored. Defaults to `observability-annotations`. +`xpack.apm.searchAggregatedServiceMetrics` {ess-icon}:: + Enables Service metrics. Defaults to `false`. When set to `true`, additional configuration in APM Server is required. + `xpack.apm.searchAggregatedTransactions` {ess-icon}:: Enables Transaction histogram metrics. Defaults to `auto` so the UI will use metric indices over transaction indices for transactions if aggregated transactions are found. When set to `always`, additional configuration in APM Server is required. When set to `never` and aggregated transactions are not used. + diff --git a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index 3ac48a1e367ce..95c36d24aad5b 100644 --- a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -221,6 +221,10 @@ exports[`Error TRANSACTION_DURATION 1`] = `undefined`; exports[`Error TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; +exports[`Error TRANSACTION_DURATION_SUMMARY 1`] = `undefined`; + +exports[`Error TRANSACTION_FAILURE_COUNT 1`] = `undefined`; + exports[`Error TRANSACTION_ID 1`] = `"transaction id"`; exports[`Error TRANSACTION_NAME 1`] = `undefined`; @@ -233,6 +237,8 @@ exports[`Error TRANSACTION_ROOT 1`] = `undefined`; exports[`Error TRANSACTION_SAMPLED 1`] = `undefined`; +exports[`Error TRANSACTION_SUCCESS_COUNT 1`] = `undefined`; + exports[`Error TRANSACTION_TYPE 1`] = `"request"`; exports[`Error TRANSACTION_URL 1`] = `undefined`; @@ -462,6 +468,10 @@ exports[`Span TRANSACTION_DURATION 1`] = `undefined`; exports[`Span TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; +exports[`Span TRANSACTION_DURATION_SUMMARY 1`] = `undefined`; + +exports[`Span TRANSACTION_FAILURE_COUNT 1`] = `undefined`; + exports[`Span TRANSACTION_ID 1`] = `"transaction id"`; exports[`Span TRANSACTION_NAME 1`] = `undefined`; @@ -474,6 +484,8 @@ exports[`Span TRANSACTION_ROOT 1`] = `undefined`; exports[`Span TRANSACTION_SAMPLED 1`] = `undefined`; +exports[`Span TRANSACTION_SUCCESS_COUNT 1`] = `undefined`; + exports[`Span TRANSACTION_TYPE 1`] = `undefined`; exports[`Span TRANSACTION_URL 1`] = `undefined`; @@ -721,6 +733,10 @@ exports[`Transaction TRANSACTION_DURATION 1`] = `1337`; exports[`Transaction TRANSACTION_DURATION_HISTOGRAM 1`] = `undefined`; +exports[`Transaction TRANSACTION_DURATION_SUMMARY 1`] = `undefined`; + +exports[`Transaction TRANSACTION_FAILURE_COUNT 1`] = `undefined`; + exports[`Transaction TRANSACTION_ID 1`] = `"transaction id"`; exports[`Transaction TRANSACTION_NAME 1`] = `"transaction name"`; @@ -733,6 +749,8 @@ exports[`Transaction TRANSACTION_ROOT 1`] = `undefined`; exports[`Transaction TRANSACTION_SAMPLED 1`] = `true`; +exports[`Transaction TRANSACTION_SUCCESS_COUNT 1`] = `undefined`; + exports[`Transaction TRANSACTION_TYPE 1`] = `"transaction type"`; exports[`Transaction TRANSACTION_URL 1`] = `"http://www.elastic.co"`; diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts index 575588018b369..f29bf3c607e86 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts @@ -46,12 +46,15 @@ export const PROCESSOR_EVENT = 'processor.event'; export const TRANSACTION_DURATION = 'transaction.duration.us'; export const TRANSACTION_DURATION_HISTOGRAM = 'transaction.duration.histogram'; +export const TRANSACTION_DURATION_SUMMARY = 'transaction.duration.summary'; export const TRANSACTION_TYPE = 'transaction.type'; export const TRANSACTION_RESULT = 'transaction.result'; export const TRANSACTION_NAME = 'transaction.name'; export const TRANSACTION_ID = 'transaction.id'; export const TRANSACTION_SAMPLED = 'transaction.sampled'; export const TRANSACTION_PAGE_URL = 'transaction.page.url'; +export const TRANSACTION_FAILURE_COUNT = 'transaction.failure_count'; +export const TRANSACTION_SUCCESS_COUNT = 'transaction.success_count'; // for transaction metrics export const TRANSACTION_ROOT = 'transaction.root'; diff --git a/x-pack/plugins/apm/server/index.ts b/x-pack/plugins/apm/server/index.ts index 6071c455ad84e..7f2f7d208d896 100644 --- a/x-pack/plugins/apm/server/index.ts +++ b/x-pack/plugins/apm/server/index.ts @@ -31,6 +31,9 @@ const configSchema = schema.object({ transactionGroupBucketSize: schema.number({ defaultValue: 1000 }), maxTraceItems: schema.number({ defaultValue: 1000 }), }), + searchAggregatedServiceMetrics: schema.boolean({ + defaultValue: false, + }), searchAggregatedTransactions: schema.oneOf( [ schema.literal(SearchAggregatedTransactionSetting.auto), diff --git a/x-pack/plugins/apm/server/lib/helpers/get_bucket_size_for_aggregated_transactions/index.ts b/x-pack/plugins/apm/server/lib/helpers/get_bucket_size_for_aggregated_transactions/index.ts index c34948d9f797e..266f11f0eff3f 100644 --- a/x-pack/plugins/apm/server/lib/helpers/get_bucket_size_for_aggregated_transactions/index.ts +++ b/x-pack/plugins/apm/server/lib/helpers/get_bucket_size_for_aggregated_transactions/index.ts @@ -12,12 +12,17 @@ export function getBucketSizeForAggregatedTransactions({ end, numBuckets = 50, searchAggregatedTransactions, + searchAggregatedServiceMetrics, }: { start: number; end: number; numBuckets?: number; searchAggregatedTransactions?: boolean; + searchAggregatedServiceMetrics?: boolean; }) { - const minBucketSize = searchAggregatedTransactions ? 60 : undefined; + const minBucketSize = + searchAggregatedTransactions || searchAggregatedServiceMetrics + ? 60 + : undefined; return getBucketSize({ start, end, numBuckets, minBucketSize }); } diff --git a/x-pack/plugins/apm/server/lib/helpers/get_service_inventory_search_source.ts b/x-pack/plugins/apm/server/lib/helpers/get_service_inventory_search_source.ts new file mode 100644 index 0000000000000..c668eda669e29 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/helpers/get_service_inventory_search_source.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { APMEventClient } from './create_es_client/create_apm_event_client'; +import { getSearchAggregatedTransactions } from './transactions'; +import { getSearchAggregatedServiceMetrics } from './service_metrics'; +import { APMConfig } from '../..'; + +export async function getServiceInventorySearchSource({ + config, + apmEventClient, + start, + end, + kuery, +}: { + config: APMConfig; + apmEventClient: APMEventClient; + start: number; + end: number; + kuery: string; +}): Promise<{ + searchAggregatedTransactions: boolean; + searchAggregatedServiceMetrics: boolean; +}> { + const commonProps = { + config, + apmEventClient, + kuery, + start, + end, + }; + const [searchAggregatedTransactions, searchAggregatedServiceMetrics] = + await Promise.all([ + getSearchAggregatedTransactions(commonProps), + getSearchAggregatedServiceMetrics(commonProps), + ]); + + return { + searchAggregatedTransactions, + searchAggregatedServiceMetrics, + }; +} diff --git a/x-pack/plugins/apm/server/lib/helpers/service_metrics/get_has_aggregated_service_metrics.test.ts b/x-pack/plugins/apm/server/lib/helpers/service_metrics/get_has_aggregated_service_metrics.test.ts new file mode 100644 index 0000000000000..d78190d56e20a --- /dev/null +++ b/x-pack/plugins/apm/server/lib/helpers/service_metrics/get_has_aggregated_service_metrics.test.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getSearchAggregatedServiceMetrics } from '.'; +import { + SearchParamsMock, + inspectSearchParams, +} from '../../../utils/test_helpers'; +import { Setup } from '../setup_request'; + +const mockResponseWithServiceMetricsHits = { + took: 398, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 1, + relation: 'gte' as const, + }, + hits: [], + }, +}; + +const mockResponseWithServiceMetricsNoHits = { + took: 398, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 0, + relation: 'gte' as const, + }, + hits: [], + }, +}; + +describe('get default configuration for aggregated service metrics', () => { + it('should be false by default', async () => { + const mockSetup = { + apmEventClient: { search: () => Promise.resolve(response) }, + config: {}, + } as unknown as Setup; + + const response = await getSearchAggregatedServiceMetrics({ + apmEventClient: mockSetup.apmEventClient, + config: mockSetup.config, + kuery: '', + }); + expect(response).toBeFalsy(); + }); +}); + +describe('get has aggregated', () => { + it('should be false when xpack.apm.searchAggregatedServiceMetrics=false ', async () => { + const mockSetup = { + apmEventClient: { search: () => Promise.resolve(response) }, + config: { 'xpack.apm.searchAggregatedServiceMetrics': false }, + } as unknown as Setup; + + const response = await getSearchAggregatedServiceMetrics({ + apmEventClient: mockSetup.apmEventClient, + config: mockSetup.config, + kuery: '', + }); + expect(response).toBeFalsy(); + }); + + describe('with xpack.apm.searchAggregatedServiceMetrics=true', () => { + let mock: SearchParamsMock; + + const config = { + searchAggregatedServiceMetrics: true, + }; + + afterEach(() => { + mock.teardown(); + }); + it('should be true when service metrics data are found', async () => { + mock = await inspectSearchParams( + (setup) => + getSearchAggregatedServiceMetrics({ + apmEventClient: setup.apmEventClient, + config: setup.config, + kuery: '', + }), + { + config, + mockResponse: () => mockResponseWithServiceMetricsHits, + } + ); + expect(mock.response).toBeTruthy(); + }); + + it('should be false when service metrics data are not found', async () => { + mock = await inspectSearchParams( + (setup) => + getSearchAggregatedServiceMetrics({ + apmEventClient: setup.apmEventClient, + config: setup.config, + kuery: '', + }), + { + config, + mockResponse: () => mockResponseWithServiceMetricsNoHits, + } + ); + expect(mock.response).toBeFalsy(); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/helpers/service_metrics/index.ts b/x-pack/plugins/apm/server/lib/helpers/service_metrics/index.ts new file mode 100644 index 0000000000000..fd449734e9772 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/helpers/service_metrics/index.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { METRICSET_NAME } from '../../../../common/elasticsearch_fieldnames'; +import { APMConfig } from '../../..'; +import { APMEventClient } from '../create_es_client/create_apm_event_client'; + +export async function getSearchAggregatedServiceMetrics({ + config, + start, + end, + apmEventClient, + kuery, +}: { + config: APMConfig; + start?: number; + end?: number; + apmEventClient: APMEventClient; + kuery: string; +}): Promise { + if (config.searchAggregatedServiceMetrics) { + return getHasAggregatedServicesMetrics({ + start, + end, + apmEventClient, + kuery, + }); + } + + return false; +} + +export async function getHasAggregatedServicesMetrics({ + start, + end, + apmEventClient, + kuery, +}: { + start?: number; + end?: number; + apmEventClient: APMEventClient; + kuery: string; +}) { + const response = await apmEventClient.search( + 'get_has_aggregated_service_metrics', + { + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 1, + query: { + bool: { + filter: [ + ...getDocumentTypeFilterForServiceMetrics(), + ...(start && end ? rangeQuery(start, end) : []), + ...kqlQuery(kuery), + ], + }, + }, + }, + terminate_after: 1, + } + ); + + return response.hits.total.value > 0; +} + +export function getDocumentTypeFilterForServiceMetrics() { + return [ + { + term: { + [METRICSET_NAME]: 'service', + }, + }, + ]; +} diff --git a/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.test.ts b/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.test.ts new file mode 100644 index 0000000000000..0905f39f183b4 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.test.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { calculateFailedTransactionRateFromServiceMetrics } from './transaction_error_rate'; + +describe('calculateFailedTransactionRateFromServiceMetrics', () => { + it('should return 0 when all params are null', () => { + expect( + calculateFailedTransactionRateFromServiceMetrics({ + failedTransactions: null, + successfulTransactions: null, + }) + ).toBe(0); + }); + + it('should return 9 when failedTransactions:null', () => { + expect( + calculateFailedTransactionRateFromServiceMetrics({ + failedTransactions: null, + successfulTransactions: 2, + }) + ).toBe(0); + }); + + it('should return 0 when failedTransactions:0', () => { + expect( + calculateFailedTransactionRateFromServiceMetrics({ + failedTransactions: 0, + successfulTransactions: null, + }) + ).toBe(0); + }); + + it('should return 1 when failedTransactions:10 and successfulTransactions:0', () => { + expect( + calculateFailedTransactionRateFromServiceMetrics({ + failedTransactions: 10, + successfulTransactions: 0, + }) + ).toBe(1); + }); + it('should return 0,5 when failedTransactions:10 and successfulTransactions:10', () => { + expect( + calculateFailedTransactionRateFromServiceMetrics({ + failedTransactions: 10, + successfulTransactions: 10, + }) + ).toBe(0.5); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts b/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts index 5631633b41b72..665784ce53454 100644 --- a/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts +++ b/x-pack/plugins/apm/server/lib/helpers/transaction_error_rate.ts @@ -9,15 +9,18 @@ import type { AggregationOptionsByType, AggregationResultOf, } from '@kbn/es-types'; +import { isNull } from 'lodash'; import { EVENT_OUTCOME } from '../../../common/elasticsearch_fieldnames'; import { EventOutcome } from '../../../common/event_outcome'; -export const getOutcomeAggregation = () => ({ - terms: { - field: EVENT_OUTCOME, - include: [EventOutcome.failure, EventOutcome.success], - }, -}); +export const getOutcomeAggregation = () => { + return { + terms: { + field: EVENT_OUTCOME, + include: [EventOutcome.failure, EventOutcome.success], + }, + }; +}; type OutcomeAggregation = ReturnType; @@ -48,6 +51,21 @@ export function calculateFailedTransactionRate( return failedTransactions / (successfulTransactions + failedTransactions); } +export function calculateFailedTransactionRateFromServiceMetrics({ + failedTransactions, + successfulTransactions, +}: { + failedTransactions: number | null; + successfulTransactions: number | null; +}) { + if (isNull(failedTransactions) || failedTransactions === 0) { + return 0; + } + + successfulTransactions = successfulTransactions ?? 0; + return failedTransactions / (successfulTransactions + failedTransactions); +} + export function getFailedTransactionRateTimeSeries( buckets: AggregationResultOf< { diff --git a/x-pack/plugins/apm/server/routes/services/get_services/get_service_aggregated_transaction_stats.ts b/x-pack/plugins/apm/server/routes/services/get_services/get_service_aggregated_transaction_stats.ts new file mode 100644 index 0000000000000..d3364062eb2fb --- /dev/null +++ b/x-pack/plugins/apm/server/routes/services/get_services/get_service_aggregated_transaction_stats.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { + AGENT_NAME, + SERVICE_ENVIRONMENT, + SERVICE_NAME, + TRANSACTION_TYPE, + TRANSACTION_DURATION_SUMMARY, + TRANSACTION_FAILURE_COUNT, + TRANSACTION_SUCCESS_COUNT, +} from '../../../../common/elasticsearch_fieldnames'; +import { + TRANSACTION_PAGE_LOAD, + TRANSACTION_REQUEST, +} from '../../../../common/transaction_types'; +import { environmentQuery } from '../../../../common/utils/environment_query'; +import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; +import { calculateThroughputWithRange } from '../../../lib/helpers/calculate_throughput'; +import { calculateFailedTransactionRateFromServiceMetrics } from '../../../lib/helpers/transaction_error_rate'; +import { ServicesItemsSetup } from './get_services_items'; +import { serviceGroupQuery } from '../../../lib/service_group_query'; +import { ServiceGroup } from '../../../../common/service_groups'; +import { RandomSampler } from '../../../lib/helpers/get_random_sampler'; +import { getDocumentTypeFilterForServiceMetrics } from '../../../lib/helpers/service_metrics'; +interface AggregationParams { + environment: string; + kuery: string; + setup: ServicesItemsSetup; + maxNumServices: number; + start: number; + end: number; + serviceGroup: ServiceGroup | null; + randomSampler: RandomSampler; +} + +export async function getServiceAggregatedTransactionStats({ + environment, + kuery, + setup, + maxNumServices, + start, + end, + serviceGroup, + randomSampler, +}: AggregationParams) { + const { apmEventClient } = setup; + + const response = await apmEventClient.search( + 'get_service_aggregated_transaction_stats', + { + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + ...getDocumentTypeFilterForServiceMetrics(), + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...serviceGroupQuery(serviceGroup), + ], + }, + }, + aggs: { + sample: { + random_sampler: randomSampler, + aggs: { + services: { + terms: { + field: SERVICE_NAME, + size: maxNumServices, + }, + aggs: { + transactionType: { + terms: { + field: TRANSACTION_TYPE, + }, + aggs: { + avg_duration: { + avg: { + field: TRANSACTION_DURATION_SUMMARY, + }, + }, + total_doc: { + value_count: { + field: TRANSACTION_DURATION_SUMMARY, + }, + }, + failure_count: { + sum: { + field: TRANSACTION_FAILURE_COUNT, + }, + }, + success_count: { + sum: { + field: TRANSACTION_SUCCESS_COUNT, + }, + }, + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + }, + }, + sample: { + top_metrics: { + metrics: [{ field: AGENT_NAME } as const], + sort: { + '@timestamp': 'desc' as const, + }, + }, + }, + bucket_sort: { + bucket_sort: { + sort: [ + { + total_doc: { + order: 'desc', + }, + }, + ], + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + ); + + return ( + response.aggregations?.sample.services.buckets.map((bucket) => { + const topTransactionTypeBucket = + bucket.transactionType.buckets.find( + ({ key }) => + key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD + ) ?? bucket.transactionType.buckets[0]; + + return { + serviceName: bucket.key as string, + transactionType: topTransactionTypeBucket.key as string, + environments: topTransactionTypeBucket.environments.buckets.map( + (environmentBucket) => environmentBucket.key as string + ), + agentName: topTransactionTypeBucket.sample.top[0].metrics[ + AGENT_NAME + ] as AgentName, + latency: topTransactionTypeBucket.avg_duration.value, + transactionErrorRate: calculateFailedTransactionRateFromServiceMetrics({ + failedTransactions: topTransactionTypeBucket.failure_count.value, + successfulTransactions: topTransactionTypeBucket.success_count.value, + }), + throughput: calculateThroughputWithRange({ + start, + end, + value: topTransactionTypeBucket.total_doc.value, + }), + }; + }) ?? [] + ); +} diff --git a/x-pack/plugins/apm/server/routes/services/get_services/get_services_items.ts b/x-pack/plugins/apm/server/routes/services/get_services/get_services_items.ts index c8d83df9a700e..7ee41a2bea0e0 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services/get_services_items.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services/get_services_items.ts @@ -11,6 +11,7 @@ import { Setup } from '../../../lib/helpers/setup_request'; import { getHealthStatuses } from './get_health_statuses'; import { getServicesFromErrorAndMetricDocuments } from './get_services_from_error_and_metric_documents'; import { getServiceTransactionStats } from './get_service_transaction_stats'; +import { getServiceAggregatedTransactionStats } from './get_service_aggregated_transaction_stats'; import { mergeServiceStats } from './merge_service_stats'; import { ServiceGroup } from '../../../../common/service_groups'; import { RandomSampler } from '../../../lib/helpers/get_random_sampler'; @@ -24,6 +25,7 @@ export async function getServicesItems({ kuery, setup, searchAggregatedTransactions, + searchAggregatedServiceMetrics, logger, start, end, @@ -34,6 +36,7 @@ export async function getServicesItems({ kuery: string; setup: ServicesItemsSetup; searchAggregatedTransactions: boolean; + searchAggregatedServiceMetrics: boolean; logger: Logger; start: number; end: number; @@ -46,6 +49,7 @@ export async function getServicesItems({ kuery, setup, searchAggregatedTransactions, + searchAggregatedServiceMetrics, maxNumServices: MAX_NUMBER_OF_SERVICES, start, end, @@ -58,7 +62,9 @@ export async function getServicesItems({ servicesFromErrorAndMetricDocuments, healthStatuses, ] = await Promise.all([ - getServiceTransactionStats(params), + searchAggregatedServiceMetrics + ? getServiceAggregatedTransactionStats(params) + : getServiceTransactionStats(params), getServicesFromErrorAndMetricDocuments(params), getHealthStatuses(params).catch((err) => { logger.error(err); diff --git a/x-pack/plugins/apm/server/routes/services/get_services/index.ts b/x-pack/plugins/apm/server/routes/services/get_services/index.ts index 954da17c7249d..83932f630357e 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services/index.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services/index.ts @@ -17,6 +17,7 @@ export async function getServices({ kuery, setup, searchAggregatedTransactions, + searchAggregatedServiceMetrics, logger, start, end, @@ -27,6 +28,7 @@ export async function getServices({ kuery: string; setup: Setup; searchAggregatedTransactions: boolean; + searchAggregatedServiceMetrics: boolean; logger: Logger; start: number; end: number; @@ -39,6 +41,7 @@ export async function getServices({ kuery, setup, searchAggregatedTransactions, + searchAggregatedServiceMetrics, logger, start, end, diff --git a/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_aggregated_transaction_detailed_statistics.ts b/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_aggregated_transaction_detailed_statistics.ts new file mode 100644 index 0000000000000..e1e1c84598eff --- /dev/null +++ b/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_aggregated_transaction_detailed_statistics.ts @@ -0,0 +1,244 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { keyBy } from 'lodash'; +import { ProcessorEvent } from '@kbn/observability-plugin/common'; +import { kqlQuery, rangeQuery } from '@kbn/observability-plugin/server'; +import { + SERVICE_NAME, + TRANSACTION_TYPE, + TRANSACTION_DURATION_SUMMARY, + TRANSACTION_FAILURE_COUNT, + TRANSACTION_SUCCESS_COUNT, +} from '../../../../common/elasticsearch_fieldnames'; +import { withApmSpan } from '../../../utils/with_apm_span'; +import { + TRANSACTION_PAGE_LOAD, + TRANSACTION_REQUEST, +} from '../../../../common/transaction_types'; +import { environmentQuery } from '../../../../common/utils/environment_query'; +import { getOffsetInMs } from '../../../../common/utils/get_offset_in_ms'; +import { calculateThroughputWithRange } from '../../../lib/helpers/calculate_throughput'; +import { getBucketSizeForAggregatedTransactions } from '../../../lib/helpers/get_bucket_size_for_aggregated_transactions'; +import { Setup } from '../../../lib/helpers/setup_request'; +import { calculateFailedTransactionRateFromServiceMetrics } from '../../../lib/helpers/transaction_error_rate'; +import { RandomSampler } from '../../../lib/helpers/get_random_sampler'; +import { getDocumentTypeFilterForServiceMetrics } from '../../../lib/helpers/service_metrics'; + +export async function getServiceAggregatedTransactionDetailedStats({ + serviceNames, + environment, + kuery, + setup, + searchAggregatedServiceMetrics, + offset, + start, + end, + randomSampler, +}: { + serviceNames: string[]; + environment: string; + kuery: string; + setup: Setup; + searchAggregatedServiceMetrics: boolean; + offset?: string; + start: number; + end: number; + randomSampler: RandomSampler; +}) { + const { apmEventClient } = setup; + const { offsetInMs, startWithOffset, endWithOffset } = getOffsetInMs({ + start, + end, + offset, + }); + + const metrics = { + avg_duration: { + avg: { + field: TRANSACTION_DURATION_SUMMARY, + }, + }, + total_doc: { + value_count: { + field: TRANSACTION_DURATION_SUMMARY, + }, + }, + failure_count: { + sum: { + field: TRANSACTION_FAILURE_COUNT, + }, + }, + success_count: { + sum: { + field: TRANSACTION_SUCCESS_COUNT, + }, + }, + }; + + const response = await apmEventClient.search( + 'get_service_aggregated_transaction_detail_stats', + { + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { terms: { [SERVICE_NAME]: serviceNames } }, + ...getDocumentTypeFilterForServiceMetrics(), + ...rangeQuery(startWithOffset, endWithOffset), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], + }, + }, + aggs: { + sample: { + random_sampler: randomSampler, + aggs: { + services: { + terms: { + field: SERVICE_NAME, + size: serviceNames.length, + }, + aggs: { + transactionType: { + terms: { + field: TRANSACTION_TYPE, + }, + aggs: { + ...metrics, + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: + getBucketSizeForAggregatedTransactions({ + start: startWithOffset, + end: endWithOffset, + numBuckets: 20, + searchAggregatedServiceMetrics, + }).intervalString, + min_doc_count: 0, + extended_bounds: { + min: startWithOffset, + max: endWithOffset, + }, + }, + aggs: metrics, + }, + bucket_sort: { + bucket_sort: { + sort: [ + { + total_doc: { + order: 'desc', + }, + }, + ], + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + ); + + return keyBy( + response.aggregations?.sample.services.buckets.map((bucket) => { + const topTransactionTypeBucket = + bucket.transactionType.buckets.find( + ({ key }) => + key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD + ) ?? bucket.transactionType.buckets[0]; + + return { + serviceName: bucket.key as string, + latency: topTransactionTypeBucket.timeseries.buckets.map( + (dateBucket) => ({ + x: dateBucket.key + offsetInMs, + y: dateBucket.avg_duration.value, + }) + ), + transactionErrorRate: topTransactionTypeBucket.timeseries.buckets.map( + (dateBucket) => ({ + x: dateBucket.key + offsetInMs, + y: calculateFailedTransactionRateFromServiceMetrics({ + failedTransactions: dateBucket.failure_count.value, + successfulTransactions: dateBucket.success_count.value, + }), + }) + ), + throughput: topTransactionTypeBucket.timeseries.buckets.map( + (dateBucket) => ({ + x: dateBucket.key + offsetInMs, + y: calculateThroughputWithRange({ + start, + end, + value: dateBucket.total_doc.value, + }), + }) + ), + }; + }) ?? [], + 'serviceName' + ); +} + +export async function getServiceAggregatedDetailedStatsPeriods({ + serviceNames, + environment, + kuery, + setup, + searchAggregatedServiceMetrics, + offset, + start, + end, + randomSampler, +}: { + serviceNames: string[]; + environment: string; + kuery: string; + setup: Setup; + searchAggregatedServiceMetrics: boolean; + offset?: string; + start: number; + end: number; + randomSampler: RandomSampler; +}) { + return withApmSpan('get_service_aggregated_detailed_stats', async () => { + const commonProps = { + serviceNames, + environment, + kuery, + setup, + searchAggregatedServiceMetrics, + start, + end, + randomSampler, + }; + + const [currentPeriod, previousPeriod] = await Promise.all([ + getServiceAggregatedTransactionDetailedStats(commonProps), + offset + ? getServiceAggregatedTransactionDetailedStats({ + ...commonProps, + offset, + }) + : Promise.resolve({}), + ]); + + return { currentPeriod, previousPeriod }; + }); +} diff --git a/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts b/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts index 600d3c71f7e08..a19d7e5984cf8 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/get_service_transaction_detailed_statistics.ts @@ -11,6 +11,7 @@ import { SERVICE_NAME, TRANSACTION_TYPE, } from '../../../../common/elasticsearch_fieldnames'; +import { withApmSpan } from '../../../utils/with_apm_span'; import { TRANSACTION_PAGE_LOAD, TRANSACTION_REQUEST, @@ -31,7 +32,7 @@ import { } from '../../../lib/helpers/transaction_error_rate'; import { RandomSampler } from '../../../lib/helpers/get_random_sampler'; -export async function getServiceTransactionDetailedStatistics({ +export async function getServiceTransactionDetailedStats({ serviceNames, environment, kuery, @@ -175,3 +176,50 @@ export async function getServiceTransactionDetailedStatistics({ 'serviceName' ); } + +export async function getServiceDetailedStatsPeriods({ + serviceNames, + environment, + kuery, + setup, + searchAggregatedTransactions, + offset, + start, + end, + randomSampler, +}: { + serviceNames: string[]; + environment: string; + kuery: string; + setup: Setup; + searchAggregatedTransactions: boolean; + offset?: string; + start: number; + end: number; + randomSampler: RandomSampler; +}) { + return withApmSpan('get_service_detailed_statistics', async () => { + const commonProps = { + serviceNames, + environment, + kuery, + setup, + searchAggregatedTransactions, + start, + end, + randomSampler, + }; + + const [currentPeriod, previousPeriod] = await Promise.all([ + getServiceTransactionDetailedStats(commonProps), + offset + ? getServiceTransactionDetailedStats({ + ...commonProps, + offset, + }) + : Promise.resolve({}), + ]); + + return { currentPeriod, previousPeriod }; + }); +} diff --git a/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/index.ts b/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/index.ts index d75bf9d970c9c..b142b3484d559 100644 --- a/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/index.ts +++ b/x-pack/plugins/apm/server/routes/services/get_services_detailed_statistics/index.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup } from '../../../lib/helpers/setup_request'; -import { getServiceTransactionDetailedStatistics } from './get_service_transaction_detailed_statistics'; +import { getServiceDetailedStatsPeriods } from './get_service_transaction_detailed_statistics'; +import { getServiceAggregatedDetailedStatsPeriods } from './get_service_aggregated_transaction_detailed_statistics'; import { RandomSampler } from '../../../lib/helpers/get_random_sampler'; export async function getServicesDetailedStatistics({ @@ -16,6 +16,7 @@ export async function getServicesDetailedStatistics({ kuery, setup, searchAggregatedTransactions, + searchAggregatedServiceMetrics, offset, start, end, @@ -26,30 +27,29 @@ export async function getServicesDetailedStatistics({ kuery: string; setup: Setup; searchAggregatedTransactions: boolean; + searchAggregatedServiceMetrics: boolean; offset?: string; start: number; end: number; randomSampler: RandomSampler; }) { - return withApmSpan('get_service_detailed_statistics', async () => { - const commonProps = { - serviceNames, - environment, - kuery, - setup, - searchAggregatedTransactions, - start, - end, - randomSampler, - }; - - const [currentPeriod, previousPeriod] = await Promise.all([ - getServiceTransactionDetailedStatistics(commonProps), - offset - ? getServiceTransactionDetailedStatistics({ ...commonProps, offset }) - : Promise.resolve({}), - ]); - - return { currentPeriod, previousPeriod }; - }); + const commonProps = { + serviceNames, + environment, + kuery, + setup, + start, + end, + randomSampler, + offset, + }; + return searchAggregatedServiceMetrics + ? getServiceAggregatedDetailedStatsPeriods({ + ...commonProps, + searchAggregatedServiceMetrics, + }) + : getServiceDetailedStatsPeriods({ + ...commonProps, + searchAggregatedTransactions, + }); } diff --git a/x-pack/plugins/apm/server/routes/services/queries.test.ts b/x-pack/plugins/apm/server/routes/services/queries.test.ts index d02a188d632a3..ff9f79867c2d7 100644 --- a/x-pack/plugins/apm/server/routes/services/queries.test.ts +++ b/x-pack/plugins/apm/server/routes/services/queries.test.ts @@ -54,6 +54,7 @@ describe('services queries', () => { getServicesItems({ setup, searchAggregatedTransactions: false, + searchAggregatedServiceMetrics: false, logger: {} as any, environment: ENVIRONMENT_ALL.value, kuery: '', diff --git a/x-pack/plugins/apm/server/routes/services/route.ts b/x-pack/plugins/apm/server/routes/services/route.ts index 4b76e53877d38..0c77c5b0be86f 100644 --- a/x-pack/plugins/apm/server/routes/services/route.ts +++ b/x-pack/plugins/apm/server/routes/services/route.ts @@ -19,6 +19,7 @@ import { Annotation } from '@kbn/observability-plugin/common/annotations'; import { apmServiceGroupMaxNumberOfServices } from '@kbn/observability-plugin/common'; import { latencyAggregationTypeRt } from '../../../common/latency_aggregation_types'; import { getSearchAggregatedTransactions } from '../../lib/helpers/transactions'; +import { getServiceInventorySearchSource } from '../../lib/helpers/get_service_inventory_search_source'; import { setupRequest } from '../../lib/helpers/setup_request'; import { getServiceAnnotations } from './annotations'; import { getServices } from './get_services'; @@ -129,18 +130,23 @@ const servicesRoute = createApmServerRoute({ : Promise.resolve(null), getRandomSampler({ security, request, probability }), ]); - const searchAggregatedTransactions = await getSearchAggregatedTransactions({ - ...setup, - kuery, - start, - end, - }); + + const { apmEventClient, config } = setup; + const { searchAggregatedTransactions, searchAggregatedServiceMetrics } = + await getServiceInventorySearchSource({ + config, + apmEventClient, + kuery, + start, + end, + }); return getServices({ environment, kuery, setup, searchAggregatedTransactions, + searchAggregatedServiceMetrics, logger, start, end, @@ -213,12 +219,15 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({ getRandomSampler({ security, request, probability }), ]); - const searchAggregatedTransactions = await getSearchAggregatedTransactions({ - ...setup, - start, - end, - kuery, - }); + const { apmEventClient, config } = setup; + const { searchAggregatedTransactions, searchAggregatedServiceMetrics } = + await getServiceInventorySearchSource({ + config, + apmEventClient, + kuery, + start, + end, + }); if (!serviceNames.length) { throw Boom.badRequest(`serviceNames cannot be empty`); @@ -229,6 +238,7 @@ const servicesDetailedStatisticsRoute = createApmServerRoute({ kuery, setup, searchAggregatedTransactions, + searchAggregatedServiceMetrics, offset, serviceNames, start, From 5e3422facaca7d494a4b058edfe36ab28fac3251 Mon Sep 17 00:00:00 2001 From: Joseph McElroy Date: Tue, 20 Sep 2022 16:23:10 +0100 Subject: [PATCH 10/76] hide analytics plugin from nav (#141096) --- x-pack/plugins/enterprise_search/public/plugin.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugins/enterprise_search/public/plugin.ts b/x-pack/plugins/enterprise_search/public/plugin.ts index 6dff94d411597..ac7581814242e 100644 --- a/x-pack/plugins/enterprise_search/public/plugin.ts +++ b/x-pack/plugins/enterprise_search/public/plugin.ts @@ -15,6 +15,7 @@ import { Plugin, PluginInitializerContext, DEFAULT_APP_CATEGORIES, + AppNavLinkStatus, } from '@kbn/core/public'; import { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { HomePublicPluginSetup } from '@kbn/home-plugin/public'; @@ -125,6 +126,9 @@ export class EnterpriseSearchPlugin implements Plugin { title: ANALYTICS_PLUGIN.NAME, euiIconType: ENTERPRISE_SEARCH_OVERVIEW_PLUGIN.LOGO, searchable: bahavioralAnalyticsEnabled, + navLinkStatus: bahavioralAnalyticsEnabled + ? AppNavLinkStatus.default + : AppNavLinkStatus.hidden, appRoute: ANALYTICS_PLUGIN.URL, category: DEFAULT_APP_CATEGORIES.enterpriseSearch, mount: async (params: AppMountParameters) => { From 3e73558f1374b5c0f6998caca7095bc1b0c61086 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 20 Sep 2022 09:25:22 -0600 Subject: [PATCH 11/76] [Maps] adhoc data view support (#140858) * [Maps] adhoc data view support * enable adhoc data views from discover to maps * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' * more discover cleanup * [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' * do not add adhoc data views to references * migrated DataViewSpec persisted state * add dataViewSpec to locator state Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../components/layout/discover_layout.tsx | 1 - .../sidebar/discover_field.test.tsx | 1 - .../components/sidebar/discover_field.tsx | 3 - .../sidebar/discover_field_visualize.tsx | 21 +------ .../sidebar/discover_sidebar.test.tsx | 1 - .../components/sidebar/discover_sidebar.tsx | 7 +-- .../discover_sidebar_responsive.test.tsx | 1 - .../sidebar/discover_sidebar_responsive.tsx | 1 - .../indexpattern_datasource/datapanel.tsx | 4 +- .../migrate_data_view_persisted_state.test.ts | 29 +++++++++ .../migrate_data_view_persisted_state.ts | 40 ++++++++++++ .../maps/common/migrations/references.test.ts | 62 +++++++++++++++++++ .../maps/common/migrations/references.ts | 32 +++++++++- x-pack/plugins/maps/public/locators.ts | 12 +++- .../routes/map_page/map_app/map_app.tsx | 19 ++++++ .../routes/map_page/saved_map/saved_map.ts | 39 +++++++++++- .../public/routes/map_page/saved_map/types.ts | 3 +- .../visualize_geo_field_action.ts | 1 + .../server/embeddable/setup_embeddable.ts | 12 ++-- x-pack/plugins/maps/server/plugin.ts | 8 ++- .../maps/server/saved_objects/index.ts | 6 +- .../saved_objects/setup_saved_objects.ts | 39 ++++++++++-- 22 files changed, 290 insertions(+), 52 deletions(-) create mode 100644 x-pack/plugins/maps/common/migrations/migrate_data_view_persisted_state.test.ts create mode 100644 x-pack/plugins/maps/common/migrations/migrate_data_view_persisted_state.ts diff --git a/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx index fbd58853d797c..fe6ac392c35a8 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx @@ -255,7 +255,6 @@ export function DiscoverLayout({ viewMode={viewMode} onDataViewCreated={onDataViewCreated} availableFields$={savedSearchData$.availableFields$} - persistDataView={persistDataView} /> diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_field.test.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_field.test.tsx index 5d1414ad488a3..ffb2ec7585ed5 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_field.test.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_field.test.tsx @@ -88,7 +88,6 @@ async function getComponent({ onRemoveField: jest.fn(), showFieldStats, selected, - persistDataView: jest.fn(), state: { query: { query: '', language: 'lucene' }, filters: [], diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_field.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_field.tsx index 8fb116f88bdc5..10138dec8b4cb 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_field.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_field.tsx @@ -266,7 +266,6 @@ export interface DiscoverFieldProps { * Optionally show or hide field stats in the popover */ showFieldStats?: boolean; - persistDataView: (dataView: DataView) => Promise; /** * Discover App State @@ -293,7 +292,6 @@ function DiscoverFieldComponent({ onEditField, onDeleteField, showFieldStats, - persistDataView, state, contextualFields, }: DiscoverFieldProps) { @@ -520,7 +518,6 @@ function DiscoverFieldComponent({ multiFields={rawMultiFields} trackUiMetric={trackUiMetric} contextualFields={contextualFields} - persistDataView={persistDataView} /> ); diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_field_visualize.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_field_visualize.tsx index dccd664834b53..f60838d375b68 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_field_visualize.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_field_visualize.tsx @@ -9,12 +9,7 @@ import React, { useEffect, useState } from 'react'; import { METRIC_TYPE, UiCounterMetricType } from '@kbn/analytics'; import type { DataView, DataViewField } from '@kbn/data-views-plugin/public'; -import { VISUALIZE_GEO_FIELD_TRIGGER } from '@kbn/ui-actions-plugin/public'; -import { - getTriggerConstant, - triggerVisualizeActions, - VisualizeInformation, -} from './lib/visualize_trigger_utils'; +import { triggerVisualizeActions, VisualizeInformation } from './lib/visualize_trigger_utils'; import { getVisualizeInformation } from './lib/visualize_trigger_utils'; import { DiscoverFieldVisualizeInner } from './discover_field_visualize_inner'; @@ -24,11 +19,10 @@ interface Props { multiFields?: DataViewField[]; contextualFields: string[]; trackUiMetric?: (metricType: UiCounterMetricType, eventName: string | string[]) => void; - persistDataView: (dataView: DataView) => Promise; } export const DiscoverFieldVisualize: React.FC = React.memo( - ({ field, dataView, contextualFields, trackUiMetric, multiFields, persistDataView }) => { + ({ field, dataView, contextualFields, trackUiMetric, multiFields }) => { const [visualizeInfo, setVisualizeInfo] = useState(); useEffect(() => { @@ -47,20 +41,11 @@ export const DiscoverFieldVisualize: React.FC = React.memo( // regular link click. let the uiActions code handle the navigation and show popup if needed event.preventDefault(); - const trigger = getTriggerConstant(field.type); const triggerVisualization = (updatedDataView: DataView) => { trackUiMetric?.(METRIC_TYPE.CLICK, 'visualize_link_click'); triggerVisualizeActions(visualizeInfo.field, contextualFields, updatedDataView); }; - - if (trigger === VISUALIZE_GEO_FIELD_TRIGGER) { - const updatedDataView = await persistDataView(dataView); - if (updatedDataView) { - triggerVisualization(updatedDataView); - } - } else { - triggerVisualization(dataView); - } + triggerVisualization(dataView); }; return ( diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx index 8ee382b76832e..22d33215b1eaf 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.test.tsx @@ -78,7 +78,6 @@ function getCompProps(): DiscoverSidebarProps { onDataViewCreated: jest.fn(), availableFields$, useNewFieldsApi: true, - persistDataView: jest.fn(), }; } diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.tsx index 7ff64a1070c94..d7e227df94b8c 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar.tsx @@ -26,7 +26,7 @@ import useShallowCompareEffect from 'react-use/lib/useShallowCompareEffect'; import { isEqual } from 'lodash'; import { FormattedMessage } from '@kbn/i18n-react'; import { DataViewPicker } from '@kbn/unified-search-plugin/public'; -import { DataViewField, getFieldSubtypeMulti, type DataView } from '@kbn/data-views-plugin/public'; +import { DataViewField, getFieldSubtypeMulti } from '@kbn/data-views-plugin/public'; import { useDiscoverServices } from '../../../../hooks/use_discover_services'; import { DiscoverField } from './discover_field'; import { DiscoverFieldSearch } from './discover_field_search'; @@ -93,7 +93,6 @@ export interface DiscoverSidebarProps extends Omit Promise; } export function DiscoverSidebarComponent({ @@ -118,7 +117,6 @@ export function DiscoverSidebarComponent({ createNewDataView, showDataViewPicker, state, - persistDataView, }: DiscoverSidebarProps) { const { uiSettings, dataViewFieldEditor } = useDiscoverServices(); const [fields, setFields] = useState(null); @@ -415,7 +413,6 @@ export function DiscoverSidebarComponent({ onEditField={editField} onDeleteField={deleteField} showFieldStats={showFieldStats} - persistDataView={persistDataView} state={state} contextualFields={columns} /> @@ -478,7 +475,6 @@ export function DiscoverSidebarComponent({ onEditField={editField} onDeleteField={deleteField} showFieldStats={showFieldStats} - persistDataView={persistDataView} state={state} contextualFields={columns} /> @@ -510,7 +506,6 @@ export function DiscoverSidebarComponent({ onEditField={editField} onDeleteField={deleteField} showFieldStats={showFieldStats} - persistDataView={persistDataView} state={state} contextualFields={columns} /> diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.test.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.test.tsx index 201033050cc88..3d09302544755 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.test.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.test.tsx @@ -175,7 +175,6 @@ function getCompProps(): DiscoverSidebarResponsiveProps { viewMode: VIEW_MODE.DOCUMENT_LEVEL, onDataViewCreated: jest.fn(), useNewFieldsApi: true, - persistDataView: jest.fn(), }; } diff --git a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx index 07f4759c42b42..de3b30810859f 100644 --- a/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx +++ b/src/plugins/discover/public/application/main/components/sidebar/discover_sidebar_responsive.tsx @@ -106,7 +106,6 @@ export interface DiscoverSidebarResponsiveProps { * list of available fields fetched from ES */ availableFields$: AvailableFields$; - persistDataView: (dataView: DataView) => Promise; } /** diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx index 9c0e87e4a1464..9acec09776207 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx @@ -312,12 +312,12 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ const visualizeGeoFieldTrigger = uiActions.getTrigger(VISUALIZE_GEO_FIELD_TRIGGER); const allFields = useMemo( () => - visualizeGeoFieldTrigger && currentIndexPattern.isPersisted + visualizeGeoFieldTrigger ? currentIndexPattern.fields : currentIndexPattern.fields.filter( ({ type }) => type !== 'geo_point' && type !== 'geo_shape' ), - [currentIndexPattern.fields, currentIndexPattern.isPersisted, visualizeGeoFieldTrigger] + [currentIndexPattern.fields, visualizeGeoFieldTrigger] ); const clearLocalState = () => setLocalState((s) => ({ ...s, nameFilter: '', typeFilter: [] })); const availableFieldTypes = uniq([ diff --git a/x-pack/plugins/maps/common/migrations/migrate_data_view_persisted_state.test.ts b/x-pack/plugins/maps/common/migrations/migrate_data_view_persisted_state.test.ts new file mode 100644 index 0000000000000..e46ad2e77304a --- /dev/null +++ b/x-pack/plugins/maps/common/migrations/migrate_data_view_persisted_state.test.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Serializable } from '@kbn/utility-types'; +import { migrateDataViewsPersistedState } from './migrate_data_view_persisted_state'; + +const migrationMock = (spec: Serializable): Serializable => { + return { + ...(spec as unknown as Record), + newProp: 'somethingThatChanged', + } as unknown as Serializable; +}; + +test('should apply data view migrations to adhoc data view specs', () => { + const attributes = { + title: 'My map', + mapStateJSON: JSON.stringify({ + adHocDataViews: [{ id: 'myAdHocDataView' }], + }), + }; + const { mapStateJSON } = migrateDataViewsPersistedState({ attributes }, migrationMock); + expect(JSON.parse(mapStateJSON!)).toEqual({ + adHocDataViews: [{ id: 'myAdHocDataView', newProp: 'somethingThatChanged' }], + }); +}); diff --git a/x-pack/plugins/maps/common/migrations/migrate_data_view_persisted_state.ts b/x-pack/plugins/maps/common/migrations/migrate_data_view_persisted_state.ts new file mode 100644 index 0000000000000..413afbdc245b9 --- /dev/null +++ b/x-pack/plugins/maps/common/migrations/migrate_data_view_persisted_state.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Serializable } from '@kbn/utility-types'; +import type { DataViewSpec } from '@kbn/data-plugin/common'; +import { MigrateFunction } from '@kbn/kibana-utils-plugin/common'; +import { MapSavedObjectAttributes } from '../map_saved_object_type'; + +export function migrateDataViewsPersistedState( + { + attributes, + }: { + attributes: MapSavedObjectAttributes; + }, + migration: MigrateFunction +): MapSavedObjectAttributes { + let mapState: { adHocDataViews?: DataViewSpec[] } = { adHocDataViews: [] }; + if (attributes.mapStateJSON) { + try { + mapState = JSON.parse(attributes.mapStateJSON); + } catch (e) { + throw new Error('Unable to parse attribute mapStateJSON'); + } + + if (mapState.adHocDataViews && mapState.adHocDataViews.length > 0) { + mapState.adHocDataViews = mapState.adHocDataViews.map((spec) => { + return migration(spec) as unknown as DataViewSpec; + }); + } + } + + return { + ...attributes, + mapStateJSON: JSON.stringify(mapState), + }; +} diff --git a/x-pack/plugins/maps/common/migrations/references.test.ts b/x-pack/plugins/maps/common/migrations/references.test.ts index 5b749022bb62b..e7f62bd866fbb 100644 --- a/x-pack/plugins/maps/common/migrations/references.test.ts +++ b/x-pack/plugins/maps/common/migrations/references.test.ts @@ -42,6 +42,21 @@ describe('extractReferences', () => { }); }); + test('Should handle mapStateJSON without adHocDataViews', () => { + const mapStateJSON = JSON.stringify({}); + const attributes = { + title: 'my map', + mapStateJSON, + }; + expect(extractReferences({ attributes })).toEqual({ + attributes: { + title: 'my map', + mapStateJSON, + }, + references: [], + }); + }); + test('Should extract index-pattern reference from ES search source descriptor', () => { const attributes = { title: 'my map', @@ -62,6 +77,30 @@ describe('extractReferences', () => { }); }); + test('Should ignore adhoc data view reference from search source', () => { + const mapStateJSON = JSON.stringify({ + adHocDataViews: [ + { + id: 'c698b940-e149-11e8-a35a-370a8516603a', + }, + ], + }); + + const attributes = { + title: 'my map', + layerListJSON: layerListJSON.esSearchSource.withIndexPatternId, + mapStateJSON, + }; + expect(extractReferences({ attributes })).toEqual({ + attributes: { + title: 'my map', + layerListJSON: layerListJSON.esSearchSource.withIndexPatternId, + mapStateJSON, + }, + references: [], + }); + }); + test('Should extract index-pattern reference from ES geo grid source descriptor', () => { const attributes = { title: 'my map', @@ -121,6 +160,29 @@ describe('extractReferences', () => { ], }); }); + + test('Should ignore adhoc data view reference from joins', () => { + const mapStateJSON = JSON.stringify({ + adHocDataViews: [ + { + id: 'e20b2a30-f735-11e8-8ce0-9723965e01e3', + }, + ], + }); + const attributes = { + title: 'my map', + layerListJSON: layerListJSON.join.withIndexPatternId, + mapStateJSON, + }; + expect(extractReferences({ attributes })).toEqual({ + attributes: { + title: 'my map', + layerListJSON: layerListJSON.join.withIndexPatternId, + mapStateJSON, + }, + references: [], + }); + }); }); describe('injectReferences', () => { diff --git a/x-pack/plugins/maps/common/migrations/references.ts b/x-pack/plugins/maps/common/migrations/references.ts index 944789445fa06..fc94236583470 100644 --- a/x-pack/plugins/maps/common/migrations/references.ts +++ b/x-pack/plugins/maps/common/migrations/references.ts @@ -7,6 +7,7 @@ // Can not use public Layer classes to extract references since this logic must run in both client and server. +import type { DataViewSpec } from '@kbn/data-plugin/common'; import { SavedObjectReference } from '@kbn/core/types'; import { MapSavedObjectAttributes } from '../map_saved_object_type'; import { LayerDescriptor, VectorLayerDescriptor } from '../descriptor_types'; @@ -27,6 +28,22 @@ export function extractReferences({ return { attributes, references }; } + const adhocDataViewIds: string[] = []; + if (attributes.mapStateJSON) { + try { + const mapState = JSON.parse(attributes.mapStateJSON); + if (mapState.adHocDataViews && mapState.adHocDataViews.length > 0) { + (mapState.adHocDataViews as DataViewSpec[]).forEach((spec) => { + if (spec.id) { + adhocDataViewIds.push(spec.id); + } + }); + } + } catch (e) { + throw new Error('Unable to parse attribute mapStateJSON'); + } + } + const extractedReferences: SavedObjectReference[] = []; let layerList: LayerDescriptor[] = []; @@ -38,7 +55,13 @@ export function extractReferences({ layerList.forEach((layer, layerIndex) => { // Extract index-pattern references from source descriptor - if (layer.sourceDescriptor && 'indexPatternId' in layer.sourceDescriptor) { + if ( + layer.sourceDescriptor && + 'indexPatternId' in layer.sourceDescriptor && + !adhocDataViewIds.includes( + (layer.sourceDescriptor as IndexPatternReferenceDescriptor).indexPatternId! + ) + ) { const sourceDescriptor = layer.sourceDescriptor as IndexPatternReferenceDescriptor; const refName = `layer_${layerIndex}_source_index_pattern`; extractedReferences.push({ @@ -55,7 +78,12 @@ export function extractReferences({ const vectorLayer = layer as VectorLayerDescriptor; const joins = vectorLayer.joins ? vectorLayer.joins : []; joins.forEach((join, joinIndex) => { - if ('indexPatternId' in join.right) { + if ( + 'indexPatternId' in join.right && + !adhocDataViewIds.includes( + (join.right as IndexPatternReferenceDescriptor).indexPatternId! + ) + ) { const sourceDescriptor = join.right as IndexPatternReferenceDescriptor; const refName = `layer_${layerIndex}_join_${joinIndex}_index_pattern`; extractedReferences.push({ diff --git a/x-pack/plugins/maps/public/locators.ts b/x-pack/plugins/maps/public/locators.ts index 92749605efbb1..c4d7bb6f3d6bb 100644 --- a/x-pack/plugins/maps/public/locators.ts +++ b/x-pack/plugins/maps/public/locators.ts @@ -8,6 +8,7 @@ /* eslint-disable max-classes-per-file */ import rison from 'rison-node'; +import type { DataViewSpec } from '@kbn/data-views-plugin/public'; import type { SerializableRecord } from '@kbn/utility-types'; import { type Filter, isFilterPinned, type TimeRange, type Query } from '@kbn/es-query'; import type { GlobalQueryStateFromUrl, RefreshInterval } from '@kbn/data-plugin/public'; @@ -55,6 +56,11 @@ export interface MapsAppLocatorParams extends SerializableRecord { * whether to hash the data in the url to avoid url length issues. */ hash?: boolean; + + /** + * Optionally pass adhoc data view spec. + */ + dataViewSpec?: DataViewSpec; } export const MAPS_APP_LOCATOR = 'MAPS_APP_LOCATOR' as const; @@ -104,7 +110,11 @@ export class MapsAppLocatorDefinition implements LocatorDefinition { }; async _initMap() { + // Handle redirect with adhoc data view spec provided via history location state (MAPS_APP_LOCATOR) + const historyLocationState = this.props.history.location?.state as + | { + dataViewSpec: DataViewSpec; + } + | undefined; + if (historyLocationState?.dataViewSpec?.id) { + const dataViewService = getIndexPatternService(); + try { + const dataView = await dataViewService.get(historyLocationState.dataViewSpec.id); + if (!dataView.isPersisted()) { + await dataViewService.create(historyLocationState.dataViewSpec); + } + } catch (error) { + // ignore errors, not a critical error for viewing map - layer(s) using data view will surface error + } + } + try { await this.props.savedMap.whenReady(); } catch (err) { diff --git a/x-pack/plugins/maps/public/routes/map_page/saved_map/saved_map.ts b/x-pack/plugins/maps/public/routes/map_page/saved_map/saved_map.ts index 181fe6c56e1ef..e7adcd0795efb 100644 --- a/x-pack/plugins/maps/public/routes/map_page/saved_map/saved_map.ts +++ b/x-pack/plugins/maps/public/routes/map_page/saved_map/saved_map.ts @@ -19,6 +19,7 @@ import { getMapZoom, getMapCenter, getLayerListRaw, + getLayerList, getQuery, getFilters, getMapSettings, @@ -37,6 +38,7 @@ import { getMapAttributeService, SharingSavedObjectProps } from '../../../map_at import { MapByReferenceInput, MapEmbeddableInput } from '../../../embeddable/types'; import { getCoreChrome, + getIndexPatternService, getToasts, getIsAllowByValueEmbeddables, getSavedObjectsTagging, @@ -53,6 +55,7 @@ import { whenLicenseInitialized } from '../../../licensed_features'; import { SerializedMapState, SerializedUiState } from './types'; import { setAutoOpenLayerWizardId } from '../../../actions/ui_actions'; import { LayerStatsCollector, MapSettingsCollector } from '../../../../common/telemetry'; +import { getIndexPatternsFromIds } from '../../../index_pattern_util'; function setMapSettingsFromEncodedState(settings: Partial) { const decodedCustomIcons = settings.customIcons @@ -141,6 +144,21 @@ export class SavedMap { this._reportUsage(); + if (this._attributes?.mapStateJSON) { + try { + const mapState = JSON.parse(this._attributes.mapStateJSON) as SerializedMapState; + if (mapState.adHocDataViews && mapState.adHocDataViews.length > 0) { + const dataViewService = getIndexPatternService(); + const promises = mapState.adHocDataViews.map((spec) => { + return dataViewService.create(spec); + }); + await Promise.all(promises); + } + } catch (e) { + // ignore malformed mapStateJSON, not a critical error for viewing map - map will just use defaults + } + } + if (this._mapEmbeddableInput && this._mapEmbeddableInput.mapSettings !== undefined) { this._store.dispatch(setMapSettingsFromEncodedState(this._mapEmbeddableInput.mapSettings)); } else if (this._attributes?.mapStateJSON) { @@ -433,7 +451,7 @@ export class SavedMap { if (newTags) { this._tags = newTags; } - this._syncAttributesWithStore(); + await this._syncAttributesWithStore(); let updatedMapEmbeddableInput: MapEmbeddableInput; try { @@ -517,7 +535,7 @@ export class SavedMap { return; } - private _syncAttributesWithStore() { + private async _syncAttributesWithStore() { const state: MapStoreState = this._store.getState(); const layerList = getLayerListRaw(state); const layerListConfigOnly = copyPersistentState(layerList); @@ -526,6 +544,7 @@ export class SavedMap { const mapSettings = getMapSettings(state); this._attributes!.mapStateJSON = JSON.stringify({ + adHocDataViews: await this._getAdHocDataViews(), zoom: getMapZoom(state), center: getMapCenter(state), timeFilters: getTimeFilters(state), @@ -549,4 +568,20 @@ export class SavedMap { openTOCDetails: getOpenTOCDetails(state), } as SerializedUiState); } + + private async _getAdHocDataViews() { + const dataViewIds: string[] = []; + getLayerList(this._store.getState()).forEach((layer) => { + dataViewIds.push(...layer.getIndexPatternIds()); + }); + + const dataViews = await getIndexPatternsFromIds(_.uniq(dataViewIds)); + return dataViews + .filter((dataView) => { + return !dataView.isPersisted(); + }) + .map((dataView) => { + return dataView.toSpec(false); + }); + } } diff --git a/x-pack/plugins/maps/public/routes/map_page/saved_map/types.ts b/x-pack/plugins/maps/public/routes/map_page/saved_map/types.ts index b20bcad980e72..28568e8610da0 100644 --- a/x-pack/plugins/maps/public/routes/map_page/saved_map/types.ts +++ b/x-pack/plugins/maps/public/routes/map_page/saved_map/types.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { Query } from '@kbn/data-plugin/common'; +import type { DataViewSpec, Query } from '@kbn/data-plugin/common'; import { Filter } from '@kbn/es-query'; import type { TimeRange } from '@kbn/es-query'; import { MapCenter, MapSettings } from '../../../../common/descriptor_types'; @@ -17,6 +17,7 @@ export interface RefreshConfig { // parsed contents of mapStateJSON export interface SerializedMapState { + adHocDataViews?: DataViewSpec[]; zoom: number; center: MapCenter; timeFilters?: TimeRange; diff --git a/x-pack/plugins/maps/public/trigger_actions/visualize_geo_field_action.ts b/x-pack/plugins/maps/public/trigger_actions/visualize_geo_field_action.ts index f073c7335eb09..55983a54a61a3 100644 --- a/x-pack/plugins/maps/public/trigger_actions/visualize_geo_field_action.ts +++ b/x-pack/plugins/maps/public/trigger_actions/visualize_geo_field_action.ts @@ -90,6 +90,7 @@ const getMapsLink = async (context: VisualizeFieldContext) => { query: getData().query.queryString.getQuery() as Query, initialLayers: initialLayers as unknown as LayerDescriptor[] & SerializableRecord, timeRange: getData().query.timefilter.timefilter.getTime(), + dataViewSpec: context.dataViewSpec, }); return location; diff --git a/x-pack/plugins/maps/server/embeddable/setup_embeddable.ts b/x-pack/plugins/maps/server/embeddable/setup_embeddable.ts index 68ecc70df0e9e..eefc67ac9318e 100644 --- a/x-pack/plugins/maps/server/embeddable/setup_embeddable.ts +++ b/x-pack/plugins/maps/server/embeddable/setup_embeddable.ts @@ -13,18 +13,22 @@ import { import { MAP_SAVED_OBJECT_TYPE } from '../../common/constants'; import { extract, inject } from '../../common/embeddable'; import { embeddableMigrations } from './embeddable_migrations'; -import { getPersistedStateMigrations } from '../saved_objects'; +import { getMapsFilterMigrations, getMapsDataViewMigrations } from '../saved_objects'; export function setupEmbeddable( embeddable: EmbeddableSetup, - getFilterMigrations: () => MigrateFunctionsObject + getFilterMigrations: () => MigrateFunctionsObject, + getDataViewMigrations: () => MigrateFunctionsObject ) { embeddable.registerEmbeddableFactory({ id: MAP_SAVED_OBJECT_TYPE, migrations: () => { return mergeMigrationFunctionMaps( - embeddableMigrations, - getPersistedStateMigrations(getFilterMigrations()) + mergeMigrationFunctionMaps( + embeddableMigrations, + getMapsFilterMigrations(getFilterMigrations()) + ), + getMapsDataViewMigrations(getDataViewMigrations()) ); }, inject, diff --git a/x-pack/plugins/maps/server/plugin.ts b/x-pack/plugins/maps/server/plugin.ts index 8101018c6db6f..838b6c3cc7ef5 100644 --- a/x-pack/plugins/maps/server/plugin.ts +++ b/x-pack/plugins/maps/server/plugin.ts @@ -15,6 +15,7 @@ import { DEFAULT_APP_CATEGORIES, } from '@kbn/core/server'; import { HomeServerPluginSetup } from '@kbn/home-plugin/server'; +import { DataViewPersistableStateService } from '@kbn/data-views-plugin/common'; import type { EMSSettings } from '@kbn/maps-ems-plugin/server'; // @ts-expect-error import { getEcommerceSavedObjects } from './sample_data/ecommerce_saved_objects'; @@ -148,6 +149,9 @@ export class MapsPlugin implements Plugin { const getFilterMigrations = plugins.data.query.filterManager.getAllMigrations.bind( plugins.data.query.filterManager ); + const getDataViewMigrations = DataViewPersistableStateService.getAllMigrations.bind( + DataViewPersistableStateService + ); const { usageCollection, home, features, customIntegrations } = plugins; const config$ = this._initializerContext.config.create(); @@ -195,10 +199,10 @@ export class MapsPlugin implements Plugin { }, }); - setupSavedObjects(core, getFilterMigrations); + setupSavedObjects(core, getFilterMigrations, getDataViewMigrations); registerMapsUsageCollector(usageCollection); - setupEmbeddable(plugins.embeddable, getFilterMigrations); + setupEmbeddable(plugins.embeddable, getFilterMigrations, getDataViewMigrations); return { config: config$, diff --git a/x-pack/plugins/maps/server/saved_objects/index.ts b/x-pack/plugins/maps/server/saved_objects/index.ts index 1b0b129a29299..2cf9a1144ac27 100644 --- a/x-pack/plugins/maps/server/saved_objects/index.ts +++ b/x-pack/plugins/maps/server/saved_objects/index.ts @@ -5,4 +5,8 @@ * 2.0. */ -export { getPersistedStateMigrations, setupSavedObjects } from './setup_saved_objects'; +export { + getMapsDataViewMigrations, + getMapsFilterMigrations, + setupSavedObjects, +} from './setup_saved_objects'; diff --git a/x-pack/plugins/maps/server/saved_objects/setup_saved_objects.ts b/x-pack/plugins/maps/server/saved_objects/setup_saved_objects.ts index caa99d98662c1..c75f1f322e25c 100644 --- a/x-pack/plugins/maps/server/saved_objects/setup_saved_objects.ts +++ b/x-pack/plugins/maps/server/saved_objects/setup_saved_objects.ts @@ -12,12 +12,14 @@ import { MigrateFunctionsObject } from '@kbn/kibana-utils-plugin/common'; import { mergeSavedObjectMigrationMaps } from '@kbn/core/server'; import { APP_ICON, getFullPath } from '../../common/constants'; import { migrateDataPersistedState } from '../../common/migrations/migrate_data_persisted_state'; +import { migrateDataViewsPersistedState } from '../../common/migrations/migrate_data_view_persisted_state'; import type { MapSavedObjectAttributes } from '../../common/map_saved_object_type'; import { savedObjectMigrations } from './saved_object_migrations'; export function setupSavedObjects( core: CoreSetup, - getFilterMigrations: () => MigrateFunctionsObject + getFilterMigrations: () => MigrateFunctionsObject, + getDataViewMigrations: () => MigrateFunctionsObject ) { core.savedObjects.registerType({ name: 'map', @@ -51,8 +53,11 @@ export function setupSavedObjects( }, migrations: () => { return mergeSavedObjectMigrationMaps( - savedObjectMigrations, - getPersistedStateMigrations(getFilterMigrations()) as unknown as SavedObjectMigrationMap + mergeSavedObjectMigrationMaps( + savedObjectMigrations, + getMapsFilterMigrations(getFilterMigrations()) as unknown as SavedObjectMigrationMap + ), + getMapsDataViewMigrations(getDataViewMigrations()) ); }, }); @@ -76,9 +81,9 @@ export function setupSavedObjects( } /** - * This creates a migration map that applies external plugin migrations to persisted state stored in Maps + * This creates a migration map that applies external data plugin migrations to persisted filter state stored in Maps */ -export const getPersistedStateMigrations = ( +export const getMapsFilterMigrations = ( filterMigrations: MigrateFunctionsObject ): MigrateFunctionsObject => mapValues( @@ -98,3 +103,27 @@ export const getPersistedStateMigrations = ( } } ); + +/** + * This creates a migration map that applies external data view plugin migrations to persisted data view state stored in Maps + */ +export const getMapsDataViewMigrations = ( + migrations: MigrateFunctionsObject +): MigrateFunctionsObject => + mapValues( + migrations, + (migration) => (doc: SavedObjectUnsanitizedDoc) => { + try { + const attributes = migrateDataViewsPersistedState(doc, migration); + + return { + ...doc, + attributes, + }; + } catch (e) { + // Do not fail migration + // Maps application can display error when saved object is viewed + return doc; + } + } + ); From 45649ced6ae2c71662ab27444042ae86fa61f7dd Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Tue, 20 Sep 2022 17:31:27 +0200 Subject: [PATCH 12/76] [APM] Agent configuration - EuiMarkdownFormat when rendering setting description (#141086) * Used ReactMarkdown component to render setting description * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../settings_page/setting_form_row.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/apm/public/components/app/settings/agent_configurations/agent_configuration_create_edit/settings_page/setting_form_row.tsx b/x-pack/plugins/apm/public/components/app/settings/agent_configurations/agent_configuration_create_edit/settings_page/setting_form_row.tsx index ccc483182d772..37bf55ec1f456 100644 --- a/x-pack/plugins/apm/public/components/app/settings/agent_configurations/agent_configuration_create_edit/settings_page/setting_form_row.tsx +++ b/x-pack/plugins/apm/public/components/app/settings/agent_configurations/agent_configuration_create_edit/settings_page/setting_form_row.tsx @@ -18,6 +18,7 @@ import { EuiIconTip, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { EuiMarkdownFormat } from '@elastic/eui'; import { SettingDefinition } from '../../../../../../../common/agent_configuration/setting_definitions/types'; import { validateSetting } from '../../../../../../../common/agent_configuration/setting_definitions'; import { @@ -163,7 +164,7 @@ export function SettingFormRow({ } description={ <> - {setting.description} + {setting.description} {setting.defaultValue && ( <> From 706d3defdd9bd99788219b601694c9882133f0ad Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 20 Sep 2022 17:37:26 +0200 Subject: [PATCH 13/76] [ML] Explain Log Rate Spikes: Fix grouping edge cases. (#140891) - Change point groups might miss individual change points that were not returned by the `frequent_items` agg as part of groups. This PR now adds each missing one as an individual additional group. - Only return groups if there's at least one group with more than one item, otherwise fall back to basic table with significant terms. - Changes the UI behaviour to show the regular table by default and the grouping switch set to off. - Adds `p-value` column to grouped table and defaults to sorting by that column similar to table with indidivual items. --- x-pack/packages/ml/agg_utils/src/types.ts | 1 + .../explain_log_rate_spikes_analysis.tsx | 12 +- .../spike_analysis_table.tsx | 38 +++-- .../spike_analysis_table_expanded_row.tsx | 28 ++-- .../spike_analysis_table_groups.tsx | 57 +++++-- .../server/routes/explain_log_rate_spikes.ts | 146 ++++++++++++++++-- .../routes/queries/fetch_frequent_items.ts | 59 ++++--- .../get_simple_hierarchical_tree.test.ts | 101 ++++++++++++ .../queries/get_simple_hierarchical_tree.ts | 50 ++++-- .../translations/translations/fr-FR.json | 16 +- .../translations/translations/ja-JP.json | 16 +- .../translations/translations/zh-CN.json | 16 +- .../apps/aiops/explain_log_rate_spikes.ts | 48 +++++- .../test/functional/apps/aiops/test_data.ts | 12 +- .../services/aiops/explain_log_rate_spikes.ts | 20 +++ 15 files changed, 503 insertions(+), 117 deletions(-) create mode 100644 x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts diff --git a/x-pack/packages/ml/agg_utils/src/types.ts b/x-pack/packages/ml/agg_utils/src/types.ts index 0128d5641bf24..de993e4b9f321 100644 --- a/x-pack/packages/ml/agg_utils/src/types.ts +++ b/x-pack/packages/ml/agg_utils/src/types.ts @@ -97,4 +97,5 @@ interface ChangePointGroupItem extends FieldValuePair { export interface ChangePointGroup { group: ChangePointGroupItem[]; docCount: number; + pValue: number | null; } diff --git a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx index 98d6053a8b6f6..350ab9f2e0205 100644 --- a/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx +++ b/x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx @@ -74,7 +74,7 @@ export const ExplainLogRateSpikesAnalysis: FC const [currentAnalysisWindowParameters, setCurrentAnalysisWindowParameters] = useState< WindowParameters | undefined >(); - const [groupResults, setGroupResults] = useState(true); + const [groupResults, setGroupResults] = useState(false); const onSwitchToggle = (e: { target: { checked: React.SetStateAction } }) => { setGroupResults(e.target.checked); @@ -106,6 +106,9 @@ export const ExplainLogRateSpikesAnalysis: FC // Start handler clears possibly hovered or pinned // change points on analysis refresh. function startHandler() { + // Reset grouping to false when restarting the analysis. + setGroupResults(false); + if (onPinnedChangePoint) { onPinnedChangePoint(null); } @@ -124,7 +127,7 @@ export const ExplainLogRateSpikesAnalysis: FC }, []); const groupTableItems = useMemo(() => { - const tableItems = data.changePointsGroups.map(({ group, docCount }, index) => { + const tableItems = data.changePointsGroups.map(({ group, docCount, pValue }, index) => { const sortedGroup = group.sort((a, b) => a.fieldName > b.fieldName ? 1 : b.fieldName > a.fieldName ? -1 : 0 ); @@ -143,6 +146,7 @@ export const ExplainLogRateSpikesAnalysis: FC return { id: index, docCount, + pValue, group: dedupedGroup, repeatedValues, }; @@ -162,8 +166,7 @@ export const ExplainLogRateSpikesAnalysis: FC const groupItemCount = groupTableItems.reduce((p, c) => { return p + Object.keys(c.group).length; }, 0); - const foundGroups = - groupTableItems.length === 0 || (groupTableItems.length > 0 && groupItemCount > 0); + const foundGroups = groupTableItems.length > 0 && groupItemCount > 0; return (
@@ -178,6 +181,7 @@ export const ExplainLogRateSpikesAnalysis: FC {showSpikeAnalysisTable && foundGroups && ( = ({ { 'data-test-subj': 'aiopsSpikeAnalysisTableColumnFieldName', field: 'fieldName', - name: i18n.translate( - 'xpack.aiops.correlations.failedTransactions.correlationsTable.fieldNameLabel', - { defaultMessage: 'Field name' } - ), + name: i18n.translate('xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.fieldNameLabel', { + defaultMessage: 'Field name', + }), sortable: true, }, { 'data-test-subj': 'aiopsSpikeAnalysisTableColumnFieldValue', field: 'fieldValue', - name: i18n.translate( - 'xpack.aiops.correlations.failedTransactions.correlationsTable.fieldValueLabel', - { defaultMessage: 'Field value' } - ), + name: i18n.translate('xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.fieldValueLabel', { + defaultMessage: 'Field value', + }), render: (_, { fieldValue }) => String(fieldValue).slice(0, 50), sortable: true, }, @@ -153,7 +152,7 @@ export const SpikeAnalysisTable: FC = ({ = ({ > <> @@ -178,6 +177,15 @@ export const SpikeAnalysisTable: FC = ({ ), sortable: false, }, + { + 'data-test-subj': 'aiopsSpikeAnalysisTableColumnDocCount', + width: NARROW_COLUMN_WIDTH, + field: 'doc_count', + name: i18n.translate('xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.docCountLabel', { + defaultMessage: 'Doc count', + }), + sortable: true, + }, { 'data-test-subj': 'aiopsSpikeAnalysisTableColumnPValue', width: NARROW_COLUMN_WIDTH, @@ -186,7 +194,7 @@ export const SpikeAnalysisTable: FC = ({ = ({ > <> ), - render: (pValue: number) => pValue.toPrecision(3), + render: (pValue: number | null) => pValue?.toPrecision(3) ?? NOT_AVAILABLE, sortable: true, }, { @@ -213,7 +221,7 @@ export const SpikeAnalysisTable: FC = ({ = ({ > <> diff --git a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_expanded_row.tsx b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_expanded_row.tsx index 367e5f32c695b..9eb235c07a1a1 100644 --- a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_expanded_row.tsx +++ b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_expanded_row.tsx @@ -130,19 +130,17 @@ export const SpikeAnalysisTableExpandedRow: FC String(fieldValue).slice(0, 50), sortable: true, }, @@ -154,7 +152,7 @@ export const SpikeAnalysisTableExpandedRow: FC <> @@ -190,7 +188,7 @@ export const SpikeAnalysisTableExpandedRow: FC <> ), - render: (pValue: number) => pValue?.toPrecision(3) ?? NOT_AVAILABLE, + render: (pValue: number | null) => pValue?.toPrecision(3) ?? NOT_AVAILABLE, sortable: true, }, { @@ -217,7 +215,7 @@ export const SpikeAnalysisTableExpandedRow: FC <> diff --git a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx index 7b61911a96384..7a6f5508aea82 100644 --- a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx +++ b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx @@ -13,23 +13,32 @@ import { EuiBasicTable, EuiBasicTableColumn, EuiButtonIcon, + EuiIcon, EuiScreenReaderOnly, EuiSpacer, EuiTableSortingType, + EuiToolTip, RIGHT_ALIGNMENT, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; import type { ChangePoint } from '@kbn/ml-agg-utils'; import { useEuiTheme } from '../../hooks/use_eui_theme'; import { SpikeAnalysisTableExpandedRow } from './spike_analysis_table_expanded_row'; +const NARROW_COLUMN_WIDTH = '120px'; +const EXPAND_COLUMN_WIDTH = '40px'; +const NOT_AVAILABLE = '--'; + const PAGINATION_SIZE_OPTIONS = [5, 10, 20, 50]; -const DEFAULT_SORT_FIELD = 'docCount'; -const DEFAULT_SORT_DIRECTION = 'desc'; +const DEFAULT_SORT_FIELD = 'pValue'; +const DEFAULT_SORT_DIRECTION = 'asc'; + interface GroupTableItem { id: number; docCount: number; + pValue: number | null; group: Record; repeatedValues: Record; } @@ -107,7 +116,7 @@ export const SpikeAnalysisGroupsTable: FC = ({ const columns: Array> = [ { align: RIGHT_ALIGNMENT, - width: '40px', + width: EXPAND_COLUMN_WIDTH, isExpander: true, name: ( @@ -126,10 +135,9 @@ export const SpikeAnalysisGroupsTable: FC = ({ { 'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnGroup', field: 'group', - name: i18n.translate( - 'xpack.aiops.correlations.failedTransactions.correlationsTable.groupLabel', - { defaultMessage: 'Group' } - ), + name: i18n.translate('xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.groupLabel', { + defaultMessage: 'Group', + }), render: (_, { group, repeatedValues }) => { const valuesBadges = []; for (const fieldName in group) { @@ -159,7 +167,11 @@ export const SpikeAnalysisGroupsTable: FC = ({ data-test-subj="aiopsSpikeAnalysisGroupsTableColumnGroupBadge" color="hollow" > - +{Object.keys(repeatedValues).length} more + +{Object.keys(repeatedValues).length}{' '} + @@ -170,10 +182,37 @@ export const SpikeAnalysisGroupsTable: FC = ({ sortable: false, textOnly: true, }, + { + 'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnPValue', + width: NARROW_COLUMN_WIDTH, + field: 'pValue', + name: ( + + <> + + + + + ), + render: (pValue: number | null) => pValue?.toPrecision(3) ?? NOT_AVAILABLE, + sortable: true, + }, { 'data-test-subj': 'aiopsSpikeAnalysisGroupsTableColumnDocCount', field: 'docCount', - name: i18n.translate('xpack.aiops.correlations.correlationsTable.docCountLabel', { + name: i18n.translate('xpack.aiops.correlations.spikeAnalysisTableGroups.docCountLabel', { defaultMessage: 'Doc count', }), sortable: true, diff --git a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts index f0fadf9476e74..a6fecc0e1a870 100644 --- a/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts +++ b/x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts @@ -35,8 +35,14 @@ import type { AiopsLicense } from '../types'; import { fetchChangePointPValues } from './queries/fetch_change_point_p_values'; import { fetchFieldCandidates } from './queries/fetch_field_candidates'; -import { fetchFrequentItems } from './queries/fetch_frequent_items'; import { + dropDuplicates, + fetchFrequentItems, + groupDuplicates, +} from './queries/fetch_frequent_items'; +import type { ItemsetResult } from './queries/fetch_frequent_items'; +import { + getFieldValuePairCounts, getSimpleHierarchicalTree, getSimpleHierarchicalTreeLeaves, markDuplicates, @@ -211,31 +217,147 @@ export const defineExplainLogRateSpikesRoute = ( } if (groupingEnabled) { + // To optimize the `frequent_items` query, we identify duplicate change points by count attributes. + // Note this is a compromise and not 100% accurate because there could be change points that + // have the exact same counts but still don't co-occur. + const duplicateIdentifier: Array = [ + 'doc_count', + 'bg_count', + 'total_doc_count', + 'total_bg_count', + ]; + + // These are the deduplicated change points we pass to the `frequent_items` aggregation. + const deduplicatedChangePoints = dropDuplicates(changePoints, duplicateIdentifier); + + // We use the grouped change points to later repopulate + // the `frequent_items` result with the missing duplicates. + const groupedChangePoints = groupDuplicates(changePoints, duplicateIdentifier).filter( + (g) => g.group.length > 1 + ); + const { fields, df } = await fetchFrequentItems( client, request.body.index, JSON.parse(request.body.searchQuery) as estypes.QueryDslQueryContainer, - changePoints, + deduplicatedChangePoints, request.body.timeFieldName, request.body.deviationMin, request.body.deviationMax ); - // Filter itemsets by significant change point field value pairs - const filteredDf = df.filter((fi) => { - const { set: currentItems } = fi; + // The way the `frequent_items` aggregations works could return item sets that include + // field/value pairs that are not part of the original list of significant change points. + // This cleans up groups and removes those unrelated field/value pairs. + const filteredDf = df + .map((fi) => { + fi.set = Object.entries(fi.set).reduce( + (set, [field, value]) => { + if ( + changePoints.some((cp) => cp.fieldName === field && cp.fieldValue === value) + ) { + set[field] = value; + } + return set; + }, + {} + ); + fi.size = Object.keys(fi.set).length; + return fi; + }) + .filter((fi) => fi.size > 1); + + // `frequent_items` returns lot of different small groups of field/value pairs that co-occur. + // The following steps analyse these small groups, identify overlap between these groups, + // and then summarize them in larger groups where possible. + + // Get a tree structure based on `frequent_items`. + const { root } = getSimpleHierarchicalTree(filteredDf, true, false, fields); + + // Each leave of the tree will be a summarized group of co-occuring field/value pairs. + const treeLeaves = getSimpleHierarchicalTreeLeaves(root, []); + + // To be able to display a more cleaned up results table in the UI, we identify field/value pairs + // that occur in multiple groups. This will allow us to highlight field/value pairs that are + // unique to a group in a better way. This step will also re-add duplicates we identified in the + // beginning and didn't pass on to the `frequent_items` agg. + const fieldValuePairCounts = getFieldValuePairCounts(treeLeaves); + const changePointGroups = markDuplicates(treeLeaves, fieldValuePairCounts).map((g) => { + const group = [...g.group]; + + for (const groupItem of g.group) { + const { duplicate } = groupItem; + const duplicates = groupedChangePoints.find((d) => + d.group.some( + (dg) => + dg.fieldName === groupItem.fieldName && dg.fieldValue === groupItem.fieldValue + ) + ); + + if (duplicates !== undefined) { + group.push( + ...duplicates.group.map((d) => { + return { + fieldName: d.fieldName, + fieldValue: d.fieldValue, + duplicate, + }; + }) + ); + } + } + + return { + ...g, + group, + }; + }); - return Object.entries(currentItems).every(([key, value]) => { - return changePoints.some((cp) => { - return cp.fieldName === key && cp.fieldValue === value; - }); + // Some field/value pairs might not be part of the `frequent_items` result set, for example + // because they don't co-occur with other field/value pairs or because of the limits we set on the query. + // In this next part we identify those missing pairs and add them as individual groups. + const missingChangePoints = deduplicatedChangePoints.filter((cp) => { + return !changePointGroups.some((cpg) => { + return cpg.group.some( + (d) => d.fieldName === cp.fieldName && d.fieldValue === cp.fieldValue + ); }); }); - const { root } = getSimpleHierarchicalTree(filteredDf, true, false, fields); - const changePointsGroups = getSimpleHierarchicalTreeLeaves(root, []); + changePointGroups.push( + ...missingChangePoints.map((cp) => { + const duplicates = groupedChangePoints.find((d) => + d.group.some( + (dg) => dg.fieldName === cp.fieldName && dg.fieldValue === cp.fieldValue + ) + ); + if (duplicates !== undefined) { + return { + group: duplicates.group.map((d) => ({ + fieldName: d.fieldName, + fieldValue: d.fieldValue, + duplicate: false, + })), + docCount: cp.doc_count, + pValue: cp.pValue, + }; + } else { + return { + group: [{ fieldName: cp.fieldName, fieldValue: cp.fieldValue, duplicate: false }], + docCount: cp.doc_count, + pValue: cp.pValue, + }; + } + }) + ); + + // Finally, we'll find out if there's at least one group with at least two items, + // only then will we return the groups to the clients and make the grouping option available. + const maxItems = Math.max(...changePointGroups.map((g) => g.group.length)); - push(addChangePointsGroupAction(markDuplicates(changePointsGroups))); + if (maxItems > 1) { + push(addChangePointsGroupAction(changePointGroups)); + } } const histogramFields: [NumericHistogramField] = [ diff --git a/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts b/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts index 02d20ba18795c..055c22397064f 100644 --- a/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts +++ b/x-pack/plugins/aiops/server/routes/queries/fetch_frequent_items.ts @@ -10,7 +10,7 @@ import { uniq, uniqWith, pick, isEqual } from 'lodash'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import type { ChangePoint } from '@kbn/ml-agg-utils'; +import type { ChangePoint, FieldValuePair } from '@kbn/ml-agg-utils'; interface FrequentItemsAggregation extends estypes.AggregationsSamplerAggregation { fi: { @@ -18,8 +18,32 @@ interface FrequentItemsAggregation extends estypes.AggregationsSamplerAggregatio }; } -function dropDuplicates(cp: ChangePoint[], uniqueFields: string[]) { - return uniqWith(cp, (a, b) => isEqual(pick(a, uniqueFields), pick(b, uniqueFields))); +export function dropDuplicates(cps: ChangePoint[], uniqueFields: Array) { + return uniqWith(cps, (a, b) => isEqual(pick(a, uniqueFields), pick(b, uniqueFields))); +} + +interface ChangePointDuplicateGroup { + keys: Pick; + group: ChangePoint[]; +} +export function groupDuplicates(cps: ChangePoint[], uniqueFields: Array) { + const groups: ChangePointDuplicateGroup[] = []; + + for (const cp of cps) { + const compareAttributes = pick(cp, uniqueFields); + + const groupIndex = groups.findIndex((g) => isEqual(g.keys, compareAttributes)); + if (groupIndex === -1) { + groups.push({ + keys: compareAttributes, + group: [cp], + }); + } else { + groups[groupIndex].group.push(cp); + } + } + + return groups; } export async function fetchFrequentItems( @@ -31,17 +55,8 @@ export async function fetchFrequentItems( deviationMin: number, deviationMax: number ) { - // first remove duplicates in sig terms - note this is not strictly perfect as there could - // be conincidentally equal counts, but in general is ok... - const terms = dropDuplicates(changePoints, [ - 'doc_count', - 'bg_count', - 'total_doc_count', - 'total_bg_count', - ]); - // get unique fields that are left - const fields = [...new Set(terms.map((t) => t.fieldName))]; + const fields = [...new Set(changePoints.map((t) => t.fieldName))]; // TODO add query params const query = { @@ -58,7 +73,7 @@ export async function fetchFrequentItems( }, }, ], - should: terms.map((t) => { + should: changePoints.map((t) => { return { term: { [t.fieldName]: t.fieldValue } }; }), }, @@ -68,7 +83,7 @@ export async function fetchFrequentItems( field, })); - const totalDocCount = terms[0].total_doc_count; + const totalDocCount = changePoints[0].total_doc_count; const minDocCount = 50000; let sampleProbability = 1; @@ -88,7 +103,7 @@ export async function fetchFrequentItems( frequent_items: { minimum_set_size: 2, size: 200, - minimum_support: 0.1, + minimum_support: 0.01, fields: aggFields, }, }, @@ -153,7 +168,7 @@ export async function fetchFrequentItems( return; } - result.size = Object.keys(result).length; + result.size = Object.keys(result.set).length; result.maxPValue = maxPValue; result.doc_count = fis.doc_count; result.support = fis.support; @@ -162,15 +177,21 @@ export async function fetchFrequentItems( results.push(result); }); + results.sort((a, b) => { + return b.doc_count - a.doc_count; + }); + + const uniqueFields = uniq(results.flatMap((r) => Object.keys(r.set))); + return { - fields: uniq(results.flatMap((r) => Object.keys(r.set))), + fields: uniqueFields, df: results, totalDocCount: totalDocCountFi, }; } export interface ItemsetResult { - set: Record; + set: Record; size: number; maxPValue: number; doc_count: number; diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts new file mode 100644 index 0000000000000..1ff92181dee96 --- /dev/null +++ b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.test.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ChangePointGroup } from '@kbn/ml-agg-utils'; + +import { getFieldValuePairCounts, markDuplicates } from './get_simple_hierarchical_tree'; + +const changePointGroups: ChangePointGroup[] = [ + { + group: [ + { + fieldName: 'custom_field.keyword', + fieldValue: 'deviation', + }, + { + fieldName: 'airline', + fieldValue: 'UAL', + }, + ], + docCount: 101, + pValue: 0.01, + }, + { + group: [ + { + fieldName: 'custom_field.keyword', + fieldValue: 'deviation', + }, + { + fieldName: 'airline', + fieldValue: 'AAL', + }, + ], + docCount: 49, + pValue: 0.001, + }, +]; + +describe('get_simple_hierarchical_tree', () => { + describe('getFieldValuePairCounts', () => { + it('returns a nested record with field/value pair counts', () => { + const fieldValuePairCounts = getFieldValuePairCounts(changePointGroups); + + expect(fieldValuePairCounts).toEqual({ + airline: { + AAL: 1, + UAL: 1, + }, + 'custom_field.keyword': { + deviation: 2, + }, + }); + }); + }); + + describe('markDuplicates', () => { + it('marks duplicates based on change point groups', () => { + const fieldValuePairCounts = getFieldValuePairCounts(changePointGroups); + const markedDuplicates = markDuplicates(changePointGroups, fieldValuePairCounts); + + expect(markedDuplicates).toEqual([ + { + group: [ + { + fieldName: 'custom_field.keyword', + fieldValue: 'deviation', + duplicate: true, + }, + { + fieldName: 'airline', + fieldValue: 'UAL', + duplicate: false, + }, + ], + docCount: 101, + pValue: 0.01, + }, + { + group: [ + { + fieldName: 'custom_field.keyword', + fieldValue: 'deviation', + duplicate: true, + }, + { + fieldName: 'airline', + fieldValue: 'AAL', + duplicate: false, + }, + ], + docCount: 49, + pValue: 0.001, + }, + ]); + }); + }); +}); diff --git a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts index 7d00cc4fc3e0f..bebbb1302376d 100644 --- a/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts +++ b/x-pack/plugins/aiops/server/routes/queries/get_simple_hierarchical_tree.ts @@ -12,13 +12,13 @@ import type { ChangePointGroup, FieldValuePair } from '@kbn/ml-agg-utils'; import type { ItemsetResult } from './fetch_frequent_items'; function getValueCounts(df: ItemsetResult[], field: string) { - return df.reduce((p, c) => { + return df.reduce>((p, c) => { if (c.set[field] === undefined) { return p; } p[c.set[field]] = p[c.set[field]] ? p[c.set[field]] + 1 : 1; return p; - }, {} as Record); + }, {}); } function getValuesDescending(df: ItemsetResult[], field: string): string[] { @@ -34,6 +34,7 @@ interface NewNode { name: string; set: FieldValuePair[]; docCount: number; + pValue: number | null; children: NewNode[]; icon: string; iconStyle: string; @@ -51,6 +52,7 @@ function NewNodeFactory(name: string): NewNode { name, set: [], docCount: 0, + pValue: 0, children, icon: 'default', iconStyle: 'default', @@ -87,8 +89,8 @@ function dfDepthFirstSearch( displayOther: boolean ) { const filteredItemSets = iss.filter((is) => { - for (const [key, values] of Object.entries(is.set)) { - if (key === field && values.includes(value)) { + for (const [key, setValue] of Object.entries(is.set)) { + if (key === field && setValue === value) { return true; } } @@ -100,6 +102,7 @@ function dfDepthFirstSearch( } const docCount = Math.max(...filteredItemSets.map((fis) => fis.doc_count)); + const pValue = Math.max(...filteredItemSets.map((fis) => fis.maxPValue)); const totalDocCount = Math.max(...filteredItemSets.map((fis) => fis.total_doc_count)); let label = `${parentLabel} ${value}`; @@ -110,6 +113,7 @@ function dfDepthFirstSearch( displayParent.name += ` ${value}`; displayParent.set.push({ fieldName: field, fieldValue: value }); displayParent.docCount = docCount; + displayParent.pValue = pValue; displayNode = displayParent; } else { displayNode = NewNodeFactory(`${docCount}/${totalDocCount}${label}`); @@ -117,6 +121,7 @@ function dfDepthFirstSearch( displayNode.set = [...displayParent.set]; displayNode.set.push({ fieldName: field, fieldValue: value }); displayNode.docCount = docCount; + displayNode.pValue = pValue; displayParent.addNode(displayNode); } @@ -144,6 +149,7 @@ function dfDepthFirstSearch( nextDisplayNode.iconStyle = 'warning'; nextDisplayNode.set = displayNode.set; nextDisplayNode.docCount = docCount; + nextDisplayNode.pValue = pValue; displayNode.addNode(nextDisplayNode); displayNode = nextDisplayNode; } @@ -226,7 +232,7 @@ export function getSimpleHierarchicalTreeLeaves( ) { // console.log(`${'-'.repeat(level)} ${tree.name} ${tree.children.length}`); if (tree.children.length === 0) { - leaves.push({ group: tree.set, docCount: tree.docCount }); + leaves.push({ group: tree.set, docCount: tree.docCount, pValue: tree.pValue }); } else { for (const child of tree.children) { const newLeaves = getSimpleHierarchicalTreeLeaves(child, [], level + 1); @@ -236,29 +242,43 @@ export function getSimpleHierarchicalTreeLeaves( } } + if (leaves.length === 1 && leaves[0].group.length === 0 && leaves[0].docCount === 0) { + return []; + } + return leaves; } +type FieldValuePairCounts = Record>; /** - * Analyse duplicate field/value pairs in change point groups. + * Get a nested record of field/value pairs with counts */ -export function markDuplicates(cpgs: ChangePointGroup[]): ChangePointGroup[] { - const fieldValuePairCounts: Record = {}; - cpgs.forEach((cpg) => { - cpg.group.forEach((g) => { - const str = `${g.fieldName}$$$$${g.fieldValue}`; - fieldValuePairCounts[str] = fieldValuePairCounts[str] ? fieldValuePairCounts[str] + 1 : 1; +export function getFieldValuePairCounts(cpgs: ChangePointGroup[]): FieldValuePairCounts { + return cpgs.reduce((p, { group }) => { + group.forEach(({ fieldName, fieldValue }) => { + if (p[fieldName] === undefined) { + p[fieldName] = {}; + } + p[fieldName][fieldValue] = p[fieldName][fieldValue] ? p[fieldName][fieldValue] + 1 : 1; }); - }); + return p; + }, {}); +} +/** + * Analyse duplicate field/value pairs in change point groups. + */ +export function markDuplicates( + cpgs: ChangePointGroup[], + fieldValuePairCounts: FieldValuePairCounts +): ChangePointGroup[] { return cpgs.map((cpg) => { return { ...cpg, group: cpg.group.map((g) => { - const str = `${g.fieldName}$$$$${g.fieldValue}`; return { ...g, - duplicate: fieldValuePairCounts[str] > 1, + duplicate: fieldValuePairCounts[g.fieldName][g.fieldValue] > 1, }; }), }; diff --git a/x-pack/plugins/translations/translations/fr-FR.json b/x-pack/plugins/translations/translations/fr-FR.json index c19761a44a557..8020b449bf4e6 100644 --- a/x-pack/plugins/translations/translations/fr-FR.json +++ b/x-pack/plugins/translations/translations/fr-FR.json @@ -6385,14 +6385,14 @@ "xpack.aiops.progressTitle": "Progression : {progress} % — {progressMessage}", "xpack.aiops.searchPanel.totalDocCountLabel": "Total des documents : {strongTotalCount}", "xpack.aiops.searchPanel.totalDocCountNumber": "{totalCount, plural, other {#}}", - "xpack.aiops.correlations.failedTransactions.correlationsTable.fieldNameLabel": "Nom du champ", - "xpack.aiops.correlations.failedTransactions.correlationsTable.fieldValueLabel": "Valeur du champ", - "xpack.aiops.correlations.failedTransactions.correlationsTable.impactLabel": "Impact", - "xpack.aiops.correlations.failedTransactions.correlationsTable.impactLabelColumnTooltip": "Le niveau d'impact du champ sur la différence de taux de messages", - "xpack.aiops.correlations.failedTransactions.correlationsTable.logRateColumnTooltip": "Une représentation visuelle de l'impact du champ sur la différence de taux de messages", - "xpack.aiops.correlations.failedTransactions.correlationsTable.logRateLabel": "Taux du log", - "xpack.aiops.correlations.failedTransactions.correlationsTable.pValueColumnTooltip": "L'importance de changements dans la fréquence des valeurs ; des valeurs plus faibles indiquent un changement plus important.", - "xpack.aiops.correlations.failedTransactions.correlationsTable.pValueLabel": "valeur-p", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.fieldNameLabel": "Nom du champ", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.fieldValueLabel": "Valeur du champ", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.impactLabel": "Impact", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.impactLabelColumnTooltip": "Le niveau d'impact du champ sur la différence de taux de messages", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.logRateColumnTooltip": "Une représentation visuelle de l'impact du champ sur la différence de taux de messages", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.logRateLabel": "Taux du log", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.pValueColumnTooltip": "L'importance de changements dans la fréquence des valeurs ; des valeurs plus faibles indiquent un changement plus important.", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.pValueLabel": "valeur-p", "xpack.aiops.correlations.highImpactText": "Élevé", "xpack.aiops.correlations.lowImpactText": "Bas", "xpack.aiops.correlations.mediumImpactText": "Moyenne", diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 140eb400615e5..7896bdda353aa 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -6379,14 +6379,14 @@ "xpack.aiops.progressTitle": "進行状況：{progress}% — {progressMessage}", "xpack.aiops.searchPanel.totalDocCountLabel": "合計ドキュメント数：{strongTotalCount}", "xpack.aiops.searchPanel.totalDocCountNumber": "{totalCount, plural, other {#}}", - "xpack.aiops.correlations.failedTransactions.correlationsTable.fieldNameLabel": "フィールド名", - "xpack.aiops.correlations.failedTransactions.correlationsTable.fieldValueLabel": "フィールド値", - "xpack.aiops.correlations.failedTransactions.correlationsTable.impactLabel": "インパクト", - "xpack.aiops.correlations.failedTransactions.correlationsTable.impactLabelColumnTooltip": "メッセージレート差異に対するフィールドの影響のレベル", - "xpack.aiops.correlations.failedTransactions.correlationsTable.logRateColumnTooltip": "メッセージレート差異に対するフィールドの影響の視覚的な表示", - "xpack.aiops.correlations.failedTransactions.correlationsTable.logRateLabel": "ログレート", - "xpack.aiops.correlations.failedTransactions.correlationsTable.pValueColumnTooltip": "値の頻度の変化の有意性。値が小さいほど、変化が大きいことを示します。", - "xpack.aiops.correlations.failedTransactions.correlationsTable.pValueLabel": "p値", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.fieldNameLabel": "フィールド名", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.fieldValueLabel": "フィールド値", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.impactLabel": "インパクト", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.impactLabelColumnTooltip": "メッセージレート差異に対するフィールドの影響のレベル", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.logRateColumnTooltip": "メッセージレート差異に対するフィールドの影響の視覚的な表示", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.logRateLabel": "ログレート", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.pValueColumnTooltip": "値の頻度の変化の有意性。値が小さいほど、変化が大きいことを示します。", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.pValueLabel": "p値", "xpack.aiops.correlations.highImpactText": "高", "xpack.aiops.correlations.lowImpactText": "低", "xpack.aiops.correlations.mediumImpactText": "中", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 10aa7ea3f37e9..f4f8bb4cc7e50 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -6386,14 +6386,14 @@ "xpack.aiops.progressTitle": "进度：{progress}% — {progressMessage}", "xpack.aiops.searchPanel.totalDocCountLabel": "文档总数：{strongTotalCount}", "xpack.aiops.searchPanel.totalDocCountNumber": "{totalCount, plural, other {#}}", - "xpack.aiops.correlations.failedTransactions.correlationsTable.fieldNameLabel": "字段名称", - "xpack.aiops.correlations.failedTransactions.correlationsTable.fieldValueLabel": "字段值", - "xpack.aiops.correlations.failedTransactions.correlationsTable.impactLabel": "影响", - "xpack.aiops.correlations.failedTransactions.correlationsTable.impactLabelColumnTooltip": "字段对消息速率差异的影响水平", - "xpack.aiops.correlations.failedTransactions.correlationsTable.logRateColumnTooltip": "字段对消息速率差异的影响的视觉表示形式", - "xpack.aiops.correlations.failedTransactions.correlationsTable.logRateLabel": "日志速率", - "xpack.aiops.correlations.failedTransactions.correlationsTable.pValueColumnTooltip": "值的频率更改的意义；值越小表示变化越大", - "xpack.aiops.correlations.failedTransactions.correlationsTable.pValueLabel": "p-value", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.fieldNameLabel": "字段名称", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.fieldValueLabel": "字段值", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.impactLabel": "影响", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.impactLabelColumnTooltip": "字段对消息速率差异的影响水平", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.logRateColumnTooltip": "字段对消息速率差异的影响的视觉表示形式", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.logRateLabel": "日志速率", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.pValueColumnTooltip": "值的频率更改的意义；值越小表示变化越大", + "xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.pValueLabel": "p-value", "xpack.aiops.correlations.highImpactText": "高", "xpack.aiops.correlations.lowImpactText": "低", "xpack.aiops.correlations.mediumImpactText": "中", diff --git a/x-pack/test/functional/apps/aiops/explain_log_rate_spikes.ts b/x-pack/test/functional/apps/aiops/explain_log_rate_spikes.ts index f459ca8123e42..88977abac4099 100644 --- a/x-pack/test/functional/apps/aiops/explain_log_rate_spikes.ts +++ b/x-pack/test/functional/apps/aiops/explain_log_rate_spikes.ts @@ -12,6 +12,7 @@ import type { TestData } from './types'; import { farequoteDataViewTestData } from './test_data'; export default function ({ getPageObject, getService }: FtrProviderContext) { + const es = getService('es'); const headerPage = getPageObject('header'); const elasticChart = getService('elasticChart'); const esArchiver = getService('esArchiver'); @@ -114,6 +115,12 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { await aiops.explainLogRateSpikes.clickRerunAnalysisButton(true); await aiops.explainLogRateSpikes.assertProgressTitle('Progress: 100% — Done.'); + // The group switch should be disabled by default + await aiops.explainLogRateSpikes.assertSpikeAnalysisGroupSwitchExists(false); + + // Enabled grouping + await aiops.explainLogRateSpikes.clickSpikeAnalysisGroupSwitch(false); + await aiops.explainLogRateSpikesAnalysisGroupsTable.assertSpikeAnalysisTableExists(); const analysisGroupsTable = @@ -131,12 +138,51 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { } // Failing: See https://github.com/elastic/kibana/issues/140848 - describe.skip('explain log rate spikes', function () { + describe('explain log rate spikes', function () { this.tags(['aiops']); before(async () => { await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); + + await es.updateByQuery({ + index: 'ft_farequote', + body: { + script: { + // @ts-expect-error + inline: 'ctx._source.custom_field = "default"', + lang: 'painless', + }, + }, + }); + + for (const i of [...Array(100)]) { + await es.index({ + index: 'ft_farequote', + body: { + '@timestamp': '2016-02-09T16:19:59.000Z', + '@version': i, + airline: 'UAL', + custom_field: 'deviation', + responsetime: 10, + type: 'farequote', + }, + }); + } + + await es.index({ + index: 'ft_farequote', + body: { + '@timestamp': '2016-02-09T16:19:59.000Z', + '@version': 101, + airline: 'UAL', + custom_field: 'deviation', + responsetime: 10, + type: 'farequote', + }, + refresh: 'wait_for', + }); + await ml.testResources.setKibanaTimeZoneToUTC(); await ml.securityUI.loginAsMlPowerUser(); diff --git a/x-pack/test/functional/apps/aiops/test_data.ts b/x-pack/test/functional/apps/aiops/test_data.ts index 332e9ac91121a..861be210ddcb4 100644 --- a/x-pack/test/functional/apps/aiops/test_data.ts +++ b/x-pack/test/functional/apps/aiops/test_data.ts @@ -13,14 +13,20 @@ export const farequoteDataViewTestData: TestData = { sourceIndexOrSavedSearch: 'ft_farequote', brushTargetTimestamp: 1455033600000, expected: { - totalDocCountFormatted: '86,274', - analysisGroupsTable: [{ group: 'airline: AAL', docCount: '297' }], + totalDocCountFormatted: '86,375', + analysisGroupsTable: [ + { docCount: '297', group: 'airline: AAL' }, + { + docCount: '101', + group: 'airline: UALcustom_field.keyword: deviation', + }, + ], analysisTable: [ { fieldName: 'airline', fieldValue: 'AAL', logRate: 'Chart type:bar chart', - pValue: '1.26e-13', + pValue: '5.00e-11', impact: 'High', }, ], diff --git a/x-pack/test/functional/services/aiops/explain_log_rate_spikes.ts b/x-pack/test/functional/services/aiops/explain_log_rate_spikes.ts index 646f39ed43e69..204dad2f3b7e7 100644 --- a/x-pack/test/functional/services/aiops/explain_log_rate_spikes.ts +++ b/x-pack/test/functional/services/aiops/explain_log_rate_spikes.ts @@ -99,6 +99,26 @@ export function ExplainLogRateSpikesProvider({ getService }: FtrProviderContext) }); }, + async assertSpikeAnalysisGroupSwitchExists(checked: boolean) { + await retry.tryForTime(5000, async () => { + await testSubjects.existOrFail( + `aiopsExplainLogRateSpikesGroupSwitch${checked ? ' checked' : ''}` + ); + }); + }, + + async clickSpikeAnalysisGroupSwitch(checked: boolean) { + await testSubjects.clickWhenNotDisabledWithoutRetry( + `aiopsExplainLogRateSpikesGroupSwitch${checked ? ' checked' : ''}` + ); + + await retry.tryForTime(30 * 1000, async () => { + await testSubjects.existOrFail( + `aiopsExplainLogRateSpikesGroupSwitch${!checked ? ' checked' : ''}` + ); + }); + }, + async assertRerunAnalysisButtonExists(shouldRerun: boolean) { await testSubjects.existOrFail( `aiopsRerunAnalysisButton${shouldRerun ? ' shouldRerun' : ''}` From 3b74f5cc1885a64d2c88895be9f64cff669f75f0 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 20 Sep 2022 09:46:17 -0600 Subject: [PATCH 14/76] skip failing test suite (#139626) --- .../security_and_spaces/tests/common/cases/find_cases.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/find_cases.ts b/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/find_cases.ts index 3daa29c02b107..df0fdaeba6973 100644 --- a/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/find_cases.ts +++ b/x-pack/test/cases_api_integration/security_and_spaces/tests/common/cases/find_cases.ts @@ -55,7 +55,8 @@ export default ({ getService }: FtrProviderContext): void => { const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); - describe('find_cases', () => { + // Failing: See https://github.com/elastic/kibana/issues/139626 + describe.skip('find_cases', () => { describe('basic tests', () => { afterEach(async () => { await deleteAllCaseItems(es); From b22db90709c0a26abd47f8143d965cdd1d26d369 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 20 Sep 2022 08:51:50 -0700 Subject: [PATCH 15/76] Update dependency @elastic/charts to v49 (main) (#140890) * Update dependency @elastic/charts to v49 * Align metric and donut tests with new Metric types/theme Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Marco Vettorello Co-authored-by: Nick Partridge --- package.json | 2 +- .../public/components/metric_vis.test.tsx | 3 ++- .../common/charts/__snapshots__/donut_chart.test.tsx.snap | 2 ++ yarn.lock | 8 ++++---- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index d895c8d00e819..1ee38e61ba5af 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,7 @@ "@dnd-kit/utilities": "^2.0.0", "@elastic/apm-rum": "^5.12.0", "@elastic/apm-rum-react": "^1.4.2", - "@elastic/charts": "48.0.1", + "@elastic/charts": "49.0.0", "@elastic/datemath": "5.0.3", "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@8.3.0-canary.1", "@elastic/ems-client": "8.3.3", diff --git a/src/plugins/chart_expressions/expression_metric/public/components/metric_vis.test.tsx b/src/plugins/chart_expressions/expression_metric/public/components/metric_vis.test.tsx index edd1acc314e8e..ab5b340012125 100644 --- a/src/plugins/chart_expressions/expression_metric/public/components/metric_vis.test.tsx +++ b/src/plugins/chart_expressions/expression_metric/public/components/metric_vis.test.tsx @@ -14,6 +14,7 @@ import { LayoutDirection, Metric, MetricElementEvent, + MetricWNumber, MetricWProgress, Settings, } from '@elastic/charts'; @@ -1124,7 +1125,7 @@ describe('MetricVisComponent', function () { value: primaryMetric, valueFormatter, extra, - } = component.find(Metric).props().data?.[0][0]!; + } = component.find(Metric).props().data?.[0][0]! as MetricWNumber; return { primary: valueFormatter(primaryMetric), secondary: extra?.props.children[1] }; }; diff --git a/x-pack/plugins/synthetics/public/legacy_uptime/components/common/charts/__snapshots__/donut_chart.test.tsx.snap b/x-pack/plugins/synthetics/public/legacy_uptime/components/common/charts/__snapshots__/donut_chart.test.tsx.snap index 9e7b46568882b..b4486e65c5396 100644 --- a/x-pack/plugins/synthetics/public/legacy_uptime/components/common/charts/__snapshots__/donut_chart.test.tsx.snap +++ b/x-pack/plugins/synthetics/public/legacy_uptime/components/common/charts/__snapshots__/donut_chart.test.tsx.snap @@ -403,6 +403,8 @@ exports[`DonutChart component passes correct props without errors for valid prop "metric": Object { "background": "#FFFFFF", "barBackground": "#EDF0F5", + "border": "#EDF0F5", + "minHeight": 64, "nonFiniteText": "N/A", "text": Object { "darkColor": "#343741", diff --git a/yarn.lock b/yarn.lock index cc73531824814..012d925b7557e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1455,10 +1455,10 @@ dependencies: object-hash "^1.3.0" -"@elastic/charts@48.0.1": - version "48.0.1" - resolved "https://registry.yarnpkg.com/@elastic/charts/-/charts-48.0.1.tgz#52391ab37bcc3188748e5ac4ffe5275ce8cd93cd" - integrity sha512-cmu65UTwbQSj4Gk1gdhlORfbqrCeel9FMpz/fVODOpsRWh/4dwxZhN/x9Ob1e4W5jZDfd6OV47Gc9ZfQVzxKrQ== +"@elastic/charts@49.0.0": + version "49.0.0" + resolved "https://registry.yarnpkg.com/@elastic/charts/-/charts-49.0.0.tgz#6cda2bdb8c92691c12843ba44014fad5a60d61b0" + integrity sha512-LXurKWGyPWXgMUJ6l21tSI4y9N3lJh3yJkDGCjvW9M5P3CXICR1V9ZizMMZlj9p1OXULsccrInsiUBFMU0Ktog== dependencies: "@popperjs/core" "^2.4.0" bezier-easing "^2.1.0" From e8b3a62638f4be73ca09c184804149e95373ea69 Mon Sep 17 00:00:00 2001 From: Byron Hulcher Date: Tue, 20 Sep 2022 11:52:15 -0400 Subject: [PATCH 16/76] [Enterprise Search] Update copy for New Index view (#141080) --- .../method_connector/method_connector.tsx | 27 ++++++++++--------- .../components/new_index/new_index.tsx | 19 ++++++++----- .../applications/shared/constants/labels.ts | 7 +++++ 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/new_index/method_connector/method_connector.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/new_index/method_connector/method_connector.tsx index 47b5d565e62d4..cc97705d3f043 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/new_index/method_connector/method_connector.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/new_index/method_connector/method_connector.tsx @@ -92,7 +92,19 @@ export const MethodConnector: React.FC<{ isNative: boolean }> = ({ isNative }) =

+ {i18n.translate( + 'xpack.enterpriseSearch.content.newIndex.steps.buildConnector.title', + { + defaultMessage: 'Build a connector', + } + )} + + ), + }} />

@@ -101,22 +113,13 @@ export const MethodConnector: React.FC<{ isNative: boolean }> = ({ isNative }) = title: i18n.translate( 'xpack.enterpriseSearch.content.newIndex.methodConnector.steps.nativeConnector.title', { - defaultMessage: 'Use a pre-built connector to populate your index', + defaultMessage: 'Configure a connector', } ), titleSize: 'xs', } : { - children: isNative ? ( - -

- -

- - ) : ( + children: (

+ {TECHNICAL_PREVIEW_LABEL} + +); + const METHOD_BUTTON_GROUP_OPTIONS: ButtonGroupOption[] = [ { description: i18n.translate( 'xpack.enterpriseSearch.content.newIndex.buttonGroup.crawler.description', { - defaultMessage: 'Discover, extract, index, and sync of all your website content', + defaultMessage: 'Discover, extract, index, and sync all of your website content', } ), footer: i18n.translate('xpack.enterpriseSearch.content.newIndex.buttonGroup.crawler.footer', { @@ -57,10 +64,12 @@ const METHOD_BUTTON_GROUP_OPTIONS: ButtonGroupOption[] = [ }), }, { + badge: technicalPreviewBadge, description: i18n.translate( 'xpack.enterpriseSearch.content.newIndex.buttonGroup.nativeConnector.description', { - defaultMessage: 'Use our built-in connectors to connect to your data sources.', + defaultMessage: + 'Configure a connector to extract, index, and sync all of your content from supported data sources ', } ), footer: i18n.translate( @@ -95,11 +104,7 @@ const METHOD_BUTTON_GROUP_OPTIONS: ButtonGroupOption[] = [ }), }, { - badge: ( - - Technical Preview - - ), + badge: technicalPreviewBadge, description: i18n.translate( 'xpack.enterpriseSearch.content.newIndex.buttonGroup.connector.description', { diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/constants/labels.ts b/x-pack/plugins/enterprise_search/public/applications/shared/constants/labels.ts index 88cec898afcb0..97845b0a8e109 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/constants/labels.ts +++ b/x-pack/plugins/enterprise_search/public/applications/shared/constants/labels.ts @@ -26,3 +26,10 @@ export const TYPE_LABEL = i18n.translate('xpack.enterpriseSearch.typeLabel', { export const EMAIL_LABEL = i18n.translate('xpack.enterpriseSearch.emailLabel', { defaultMessage: 'Email', }); + +export const TECHNICAL_PREVIEW_LABEL = i18n.translate( + 'xpack.enterpriseSearch.technicalPreviewLabel', + { + defaultMessage: 'Technical Preview', // title case specifically requested + } +); From 9188db201526b657227c353313f692e39e8a159c Mon Sep 17 00:00:00 2001 From: Jordan <51442161+JordanSh@users.noreply.github.com> Date: Tue, 20 Sep 2022 18:54:01 +0300 Subject: [PATCH 17/76] [Cloud Posture] Inlining group by selector #141094 --- .../latest_findings_container.tsx | 10 +++++-- .../findings_by_resource_container.tsx | 30 +++++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings/latest_findings_container.tsx b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings/latest_findings_container.tsx index 040b383707a97..9ef90e0ddecbb 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings/latest_findings_container.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings/latest_findings_container.tsx @@ -6,7 +6,7 @@ */ import React, { useMemo } from 'react'; import { FormattedMessage } from '@kbn/i18n-react'; -import { EuiBottomBar, EuiSpacer, EuiText } from '@elastic/eui'; +import { EuiBottomBar, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiText } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import type { Evaluation } from '../../../../common/types'; import { CloudPosturePageTitle } from '../../../components/cloud_posture_page_title'; @@ -103,11 +103,15 @@ export const LatestFindingsContainer = ({ dataView }: FindingsBaseProps) => { }} loading={findingsGroupByNone.isFetching} /> - + + + + + {!error && } + {error && } {!error && ( <> - {findingsGroupByNone.isSuccess && !!findingsGroupByNone.data.page.length && ( { }} loading={findingsGroupByResource.isFetching} /> - - + + + + } /> - } - /> - + + + {!error && } + {error && } {!error && ( <> - {findingsGroupByResource.isSuccess && !!findingsGroupByResource.data.page.length && ( Date: Tue, 20 Sep 2022 09:56:15 -0600 Subject: [PATCH 18/76] skip failing test suite (#141093) --- .../apps/triggers_actions_ui/alerts_list.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts_list.ts b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts_list.ts index 123fee9476060..7262ff280faea 100644 --- a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts_list.ts +++ b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts_list.ts @@ -30,7 +30,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { await testSubjects.click('rulesTab'); } - describe('rules list', function () { + // Failing: See https://github.com/elastic/kibana/issues/141093 + describe.skip('rules list', function () { const assertRulesLength = async (length: number) => { return await retry.try(async () => { const rules = await pageObjects.triggersActionsUI.getAlertsList(); From 43e2c85a090a0d206e73d4a9d787bfa8015449a2 Mon Sep 17 00:00:00 2001 From: Brian McGue Date: Tue, 20 Sep 2022 08:59:55 -0700 Subject: [PATCH 19/76] ML Inference Pipeline backend cleanup (#141015) * Deduplicate ml_inference pipeline route structure * Create utils functions for ml inference pipelines * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../lib/indices/delete_ml_inference_pipeline.ts | 4 +++- .../fetch_ml_inference_pipeline_processors.ts | 3 ++- .../pipelines/create_pipeline_definitions.test.ts | 4 +++- .../lib/pipelines/create_pipeline_definitions.ts | 10 +++++++--- .../server/routes/enterprise_search/indices.test.ts | 12 ++++++------ .../server/routes/enterprise_search/indices.ts | 9 +++++---- .../utils/create_ml_inference_pipeline.test.ts | 8 ++++++-- .../server/utils/create_ml_inference_pipeline.ts | 9 +++++++-- .../server/utils/ml_inference_pipeline_utils.ts | 12 ++++++++++++ 9 files changed, 51 insertions(+), 20 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/server/utils/ml_inference_pipeline_utils.ts diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/delete_ml_inference_pipeline.ts b/x-pack/plugins/enterprise_search/server/lib/indices/delete_ml_inference_pipeline.ts index 282487e62f023..04a032e3be102 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/delete_ml_inference_pipeline.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/delete_ml_inference_pipeline.ts @@ -8,6 +8,8 @@ import { IngestPutPipelineRequest } from '@elastic/elasticsearch/lib/api/types'; import { ElasticsearchClient } from '@kbn/core/server'; +import { getInferencePipelineNameFromIndexName } from '../../utils/ml_inference_pipeline_utils'; + /** * Response for deleting sub-pipeline from @ml-inference pipeline. * If sub-pipeline was deleted successfully, 'deleted' field contains its name. @@ -24,7 +26,7 @@ export const deleteMlInferencePipeline = async ( client: ElasticsearchClient ) => { const response: DeleteMlInferencePipelineResponse = {}; - const parentPipelineId = `${indexName}@ml-inference`; + const parentPipelineId = getInferencePipelineNameFromIndexName(indexName); // find parent pipeline try { diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.ts b/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.ts index 2a4a90dcfacfd..48e626ff1cecd 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/fetch_ml_inference_pipeline_processors.ts @@ -8,13 +8,14 @@ import { ElasticsearchClient } from '@kbn/core/server'; import { InferencePipeline } from '../../../common/types/pipelines'; +import { getInferencePipelineNameFromIndexName } from '../../utils/ml_inference_pipeline_utils'; export const fetchMlInferencePipelineProcessorNames = async ( client: ElasticsearchClient, indexName: string ): Promise => { try { - const mlInferencePipelineName = `${indexName}@ml-inference`; + const mlInferencePipelineName = getInferencePipelineNameFromIndexName(indexName); const { [mlInferencePipelineName]: { processors: mlInferencePipelineProcessors = [] }, } = await client.ingest.getPipeline({ diff --git a/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.test.ts b/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.test.ts index 974d00fb6ae1d..6f9a5f0fdc5fb 100644 --- a/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.test.ts @@ -7,6 +7,8 @@ import { ElasticsearchClient } from '@kbn/core/server'; +import { getInferencePipelineNameFromIndexName } from '../../utils/ml_inference_pipeline_utils'; + import { createIndexPipelineDefinitions } from './create_pipeline_definitions'; import { formatMlPipelineBody } from './create_pipeline_definitions'; @@ -20,7 +22,7 @@ describe('createIndexPipelineDefinitions util function', () => { }; const expectedResult = { - created: [indexName, `${indexName}@custom`, `${indexName}@ml-inference`], + created: [indexName, `${indexName}@custom`, getInferencePipelineNameFromIndexName(indexName)], }; beforeEach(() => { diff --git a/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.ts b/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.ts index 2099f6bfc070f..4389ab8659af7 100644 --- a/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.ts +++ b/x-pack/plugins/enterprise_search/server/lib/pipelines/create_pipeline_definitions.ts @@ -8,6 +8,8 @@ import { IngestPipeline } from '@elastic/elasticsearch/lib/api/types'; import { ElasticsearchClient } from '@kbn/core/server'; +import { getInferencePipelineNameFromIndexName } from '../../utils/ml_inference_pipeline_utils'; + export interface CreatedPipelines { created: string[]; } @@ -35,7 +37,7 @@ export const createIndexPipelineDefinitions = ( // TODO: add back descriptions (see: https://github.com/elastic/elasticsearch-specification/issues/1827) esClient.ingest.putPipeline({ description: `Enterprise Search Machine Learning Inference pipeline for the '${indexName}' index`, - id: `${indexName}@ml-inference`, + id: getInferencePipelineNameFromIndexName(indexName), processors: [], version: 1, }); @@ -97,7 +99,7 @@ export const createIndexPipelineDefinitions = ( { pipeline: { if: 'ctx?._run_ml_inference == true', - name: `${indexName}@ml-inference`, + name: getInferencePipelineNameFromIndexName(indexName), on_failure: [ { append: { @@ -228,7 +230,9 @@ export const createIndexPipelineDefinitions = ( ], version: 1, }); - return { created: [indexName, `${indexName}@custom`, `${indexName}@ml-inference`] }; + return { + created: [indexName, `${indexName}@custom`, getInferencePipelineNameFromIndexName(indexName)], + }; }; /** diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts index b86f1dce83482..435fb0892019f 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.test.ts @@ -191,7 +191,7 @@ describe('Enterprise Search Managed Indices', () => { }); }); - describe('DELETE /internal/enterprise_search/indices/{indexName}/ml_inference/pipelines/{pipelineName}', () => { + describe('DELETE /internal/enterprise_search/indices/{indexName}/ml_inference/pipeline_processors/{pipelineName}', () => { const indexName = 'my-index'; const pipelineName = 'my-pipeline'; @@ -203,7 +203,7 @@ describe('Enterprise Search Managed Indices', () => { mockRouter = new MockRouter({ context, method: 'delete', - path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipelines/{pipelineName}', + path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipeline_processors/{pipelineName}', }); registerIndexRoutes({ @@ -267,7 +267,7 @@ describe('Enterprise Search Managed Indices', () => { }); }); - describe('POST /internal/enterprise_search/indices/{indexName}/ml_inference/pipelines/_simulate', () => { + describe('POST /internal/enterprise_search/indices/{indexName}/ml_inference/pipeline_processors/_simulate', () => { const pipelineBody = { description: 'Some pipeline', processors: [ @@ -299,7 +299,7 @@ describe('Enterprise Search Managed Indices', () => { mockRouter = new MockRouter({ context, method: 'post', - path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipelines/_simulate', + path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipeline_processors/_simulate', }); registerIndexRoutes({ @@ -376,7 +376,7 @@ describe('Enterprise Search Managed Indices', () => { }); }); - describe('PUT /internal/enterprise_search/indices/{indexName}/ml_inference/pipelines/{pipelineName}', () => { + describe('PUT /internal/enterprise_search/indices/{indexName}/ml_inference/pipeline_processors/{pipelineName}', () => { const pipelineName = 'some-pipeline'; const indexName = 'some-index'; const pipelineBody = { @@ -399,7 +399,7 @@ describe('Enterprise Search Managed Indices', () => { mockRouter = new MockRouter({ context, method: 'put', - path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipelines/{pipelineName}', + path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipeline_processors/{pipelineName}', }); registerIndexRoutes({ diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts index be33bc6fbe2e3..d781258599b0a 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/indices.ts @@ -39,6 +39,7 @@ import { isIndexNotFoundException, isResourceNotFoundException, } from '../../utils/identify_exceptions'; +import { getPrefixedInferencePipelineProcessorName } from '../../utils/ml_inference_pipeline_utils'; export function registerIndexRoutes({ router, @@ -460,7 +461,7 @@ export function registerIndexRoutes({ router.post( { - path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipelines/_simulate', + path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipeline_processors/_simulate', validate: { body: schema.object({ pipeline: schema.object({ @@ -513,7 +514,7 @@ export function registerIndexRoutes({ router.put( { - path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipelines/{pipelineName}', + path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipeline_processors/{pipelineName}', validate: { body: schema.object({ description: schema.maybe(schema.string()), @@ -530,7 +531,7 @@ export function registerIndexRoutes({ const indexName = decodeURIComponent(request.params.indexName); const pipelineName = decodeURIComponent(request.params.pipelineName); const { client } = (await context.core).elasticsearch; - const pipelineId = `ml-inference-${pipelineName}`; + const pipelineId = getPrefixedInferencePipelineProcessorName(pipelineName); const defaultDescription = `ML inference pipeline for index ${indexName}`; if (!(await indexOrAliasExists(client, indexName))) { @@ -571,7 +572,7 @@ export function registerIndexRoutes({ router.delete( { - path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipelines/{pipelineName}', + path: '/internal/enterprise_search/indices/{indexName}/ml_inference/pipeline_processors/{pipelineName}', validate: { params: schema.object({ indexName: schema.string(), diff --git a/x-pack/plugins/enterprise_search/server/utils/create_ml_inference_pipeline.test.ts b/x-pack/plugins/enterprise_search/server/utils/create_ml_inference_pipeline.test.ts index e4100a003eb81..e5ab370216707 100644 --- a/x-pack/plugins/enterprise_search/server/utils/create_ml_inference_pipeline.test.ts +++ b/x-pack/plugins/enterprise_search/server/utils/create_ml_inference_pipeline.test.ts @@ -11,13 +11,17 @@ import { createMlInferencePipeline, addSubPipelineToIndexSpecificMlPipeline, } from './create_ml_inference_pipeline'; +import { + getInferencePipelineNameFromIndexName, + getPrefixedInferencePipelineProcessorName, +} from './ml_inference_pipeline_utils'; describe('createMlInferencePipeline util function', () => { const pipelineName = 'my-pipeline'; const modelId = 'my-model-id'; const sourceField = 'my-source-field'; const destinationField = 'my-dest-field'; - const inferencePipelineGeneratedName = `ml-inference-${pipelineName}`; + const inferencePipelineGeneratedName = getPrefixedInferencePipelineProcessorName(pipelineName); const mockClient = { ingest: { @@ -87,7 +91,7 @@ describe('createMlInferencePipeline util function', () => { describe('addSubPipelineToIndexSpecificMlPipeline util function', () => { const indexName = 'my-index'; - const parentPipelineId = `${indexName}@ml-inference`; + const parentPipelineId = getInferencePipelineNameFromIndexName(indexName); const pipelineName = 'ml-inference-my-pipeline'; const mockClient = { diff --git a/x-pack/plugins/enterprise_search/server/utils/create_ml_inference_pipeline.ts b/x-pack/plugins/enterprise_search/server/utils/create_ml_inference_pipeline.ts index 3e1b51efabaf8..7131b4bc5cf81 100644 --- a/x-pack/plugins/enterprise_search/server/utils/create_ml_inference_pipeline.ts +++ b/x-pack/plugins/enterprise_search/server/utils/create_ml_inference_pipeline.ts @@ -12,6 +12,11 @@ import { ErrorCode } from '../../common/types/error_codes'; import { formatMlPipelineBody } from '../lib/pipelines/create_pipeline_definitions'; +import { + getInferencePipelineNameFromIndexName, + getPrefixedInferencePipelineProcessorName, +} from './ml_inference_pipeline_utils'; + /** * Details of a created pipeline. */ @@ -74,7 +79,7 @@ export const createMlInferencePipeline = async ( destinationField: string, esClient: ElasticsearchClient ): Promise => { - const inferencePipelineGeneratedName = `ml-inference-${pipelineName}`; + const inferencePipelineGeneratedName = getPrefixedInferencePipelineProcessorName(pipelineName); // Check that a pipeline with the same name doesn't already exist let pipelineByName: IngestGetPipelineResponse | undefined; @@ -120,7 +125,7 @@ export const addSubPipelineToIndexSpecificMlPipeline = async ( pipelineName: string, esClient: ElasticsearchClient ): Promise => { - const parentPipelineId = `${indexName}@ml-inference`; + const parentPipelineId = getInferencePipelineNameFromIndexName(indexName); // Fetch the parent pipeline let parentPipeline: IngestPipeline | undefined; diff --git a/x-pack/plugins/enterprise_search/server/utils/ml_inference_pipeline_utils.ts b/x-pack/plugins/enterprise_search/server/utils/ml_inference_pipeline_utils.ts new file mode 100644 index 0000000000000..ebced2c60280c --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/utils/ml_inference_pipeline_utils.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const getInferencePipelineNameFromIndexName = (indexName: string) => + `${indexName}@ml-inference`; + +export const getPrefixedInferencePipelineProcessorName = (pipelineName: string) => + pipelineName.startsWith('ml-inference-') ? pipelineName : `ml-inference-${pipelineName}`; From 84cc8c82ada5697e336e4928a76dfdad6597bacc Mon Sep 17 00:00:00 2001 From: Karl Godard Date: Tue, 20 Sep 2022 09:07:33 -0700 Subject: [PATCH 20/76] beta badge added to tty player (#141014) Co-authored-by: Karl Godard --- .../public/components/tty_player/index.tsx | 14 ++++++++++++-- .../public/components/tty_player/styles.ts | 5 +++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/session_view/public/components/tty_player/index.tsx b/x-pack/plugins/session_view/public/components/tty_player/index.tsx index bb93e6f0dddcf..394ade4a5439e 100644 --- a/x-pack/plugins/session_view/public/components/tty_player/index.tsx +++ b/x-pack/plugins/session_view/public/components/tty_player/index.tsx @@ -5,7 +5,14 @@ * 2.0. */ import React, { useRef, useState, useCallback, useMemo, useEffect } from 'react'; -import { EuiPanel, EuiFlexGroup, EuiFlexItem, EuiButtonIcon, EuiButton } from '@elastic/eui'; +import { + EuiPanel, + EuiFlexGroup, + EuiFlexItem, + EuiButtonIcon, + EuiButton, + EuiBetaBadge, +} from '@elastic/eui'; import { throttle } from 'lodash'; import { ProcessEvent } from '../../../common/types/process_tree'; import { TTYSearchBar } from '../tty_search_bar'; @@ -18,7 +25,7 @@ import { } from '../../../common/constants'; import { useFetchIOEvents, useIOLines, useXtermPlayer } from './hooks'; import { TTYPlayerControls } from '../tty_player_controls'; -import { TOGGLE_TTY_PLAYER, DETAIL_PANEL } from '../session_view/translations'; +import { BETA, TOGGLE_TTY_PLAYER, DETAIL_PANEL } from '../session_view/translations'; export interface TTYPlayerDeps { show: boolean; @@ -115,6 +122,9 @@ export const TTYPlayer = ({

+ + + diff --git a/x-pack/plugins/session_view/public/components/tty_player/styles.ts b/x-pack/plugins/session_view/public/components/tty_player/styles.ts index 34c68a8a94fae..91852d3d02b90 100644 --- a/x-pack/plugins/session_view/public/components/tty_player/styles.ts +++ b/x-pack/plugins/session_view/public/components/tty_player/styles.ts @@ -80,11 +80,16 @@ export const useStyles = (tty?: Teletype, show?: boolean) => { backgroundColor: colors.ink, }; + const betaBadge: CSSObject = { + backgroundColor: `${colors.emptyShade}`, + }; + return { container, header, terminal, scrollPane, + betaBadge, }; }, [euiTheme, show, euiVars.euiFormBackgroundDisabledColor, tty?.rows, tty?.columns]); From 0f95c927dbf59c95a7acdef0fd93de3abb8e60dd Mon Sep 17 00:00:00 2001 From: "Joey F. Poon" Date: Tue, 20 Sep 2022 11:16:03 -0500 Subject: [PATCH 21/76] [Fleet] force update agent policy schema_version (#141109) --- x-pack/plugins/fleet/server/services/agent_policy.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 606cf82f10b31..b9978d85ee73d 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -769,9 +769,15 @@ class AgentPolicyService { .map((fleetServerPolicy) => // There are some potential performance concerns around using `agentPolicyService.update` in this context. // This could potentially be a bottleneck in environments with several thousand agent policies being deployed here. - agentPolicyService.update(soClient, esClient, fleetServerPolicy.policy_id, { - schema_version: FLEET_AGENT_POLICIES_SCHEMA_VERSION, - }) + agentPolicyService.update( + soClient, + esClient, + fleetServerPolicy.policy_id, + { + schema_version: FLEET_AGENT_POLICIES_SCHEMA_VERSION, + }, + { force: true } + ) ) ); } From 977044788f44f66a84ea631669e429625ea7fe0b Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Tue, 20 Sep 2022 19:17:22 +0300 Subject: [PATCH 22/76] Fix (#141113) --- .../components/all_cases/index.test.tsx | 84 +++++-------------- 1 file changed, 22 insertions(+), 62 deletions(-) diff --git a/x-pack/plugins/cases/public/components/all_cases/index.test.tsx b/x-pack/plugins/cases/public/components/all_cases/index.test.tsx index c17607918cb4f..6fd1556f7d4e1 100644 --- a/x-pack/plugins/cases/public/components/all_cases/index.test.tsx +++ b/x-pack/plugins/cases/public/components/all_cases/index.test.tsx @@ -6,7 +6,6 @@ */ import React from 'react'; -import { mount } from 'enzyme'; import { waitFor } from '@testing-library/react'; import { AllCases } from '.'; @@ -14,7 +13,6 @@ import { AppMockRenderer, createAppMockRenderer, noCreateCasesPermissions, - TestProviders, } from '../../common/mock'; import { useGetActionLicense } from '../../containers/use_get_action_license'; import { casesStatus, connectorsMock, useGetCasesMockState } from '../../containers/mock'; @@ -46,8 +44,7 @@ const useGetActionLicenseMock = useGetActionLicense as jest.Mock; const useGetCurrentUserProfileMock = useGetCurrentUserProfile as jest.Mock; const useBulkGetUserProfilesMock = useBulkGetUserProfiles as jest.Mock; -// FLAKY: https://github.com/elastic/kibana/issues/139677 -describe.skip('AllCases', () => { +describe('AllCases', () => { const refetchCases = jest.fn(); const setFilters = jest.fn(); const setQueryParams = jest.fn(); @@ -126,58 +123,33 @@ describe.skip('AllCases', () => { ...defaultGetCases, }); - const wrapper = mount( - - - - ); + const result = appMockRender.render(); + + await waitFor(() => { + expect(result.getByTestId('openStatsHeader')).toBeInTheDocument(); + expect(result.getByText('20')).toBeInTheDocument(); + }); + + await waitFor(() => { + expect(result.getByTestId('inProgressStatsHeader')).toBeInTheDocument(); + expect(result.getByText('40')).toBeInTheDocument(); + }); await waitFor(() => { - expect(wrapper.find('[data-test-subj="openStatsHeader"]').exists()).toBeTruthy(); - expect( - wrapper - .find('[data-test-subj="openStatsHeader"] .euiDescriptionList__description') - .first() - .text() - ).toBe('20'); - - expect(wrapper.find('[data-test-subj="inProgressStatsHeader"]').exists()).toBeTruthy(); - expect( - wrapper - .find('[data-test-subj="inProgressStatsHeader"] .euiDescriptionList__description') - .first() - .text() - ).toBe('40'); - - expect(wrapper.find('[data-test-subj="closedStatsHeader"]').exists()).toBeTruthy(); - expect( - wrapper - .find('[data-test-subj="closedStatsHeader"] .euiDescriptionList__description') - .first() - .text() - ).toBe('130'); + expect(result.getByTestId('closedStatsHeader')).toBeInTheDocument(); + expect(result.getByText('130')).toBeInTheDocument(); }); }); it('should render the loading spinner when loading stats', async () => { useGetCasesStatusMock.mockReturnValue({ ...defaultCasesStatus, isLoading: true }); - const wrapper = mount( - - - - ); + const result = appMockRender.render(); await waitFor(() => { - expect( - wrapper.find('[data-test-subj="openStatsHeader-loading-spinner"]').exists() - ).toBeTruthy(); - expect( - wrapper.find('[data-test-subj="inProgressStatsHeader-loading-spinner"]').exists() - ).toBeTruthy(); - expect( - wrapper.find('[data-test-subj="closedStatsHeader-loading-spinner"]').exists() - ).toBeTruthy(); + expect(result.getByTestId('openStatsHeader-loading-spinner')).toBeInTheDocument(); + expect(result.getByTestId('inProgressStatsHeader-loading-spinner')).toBeInTheDocument(); + expect(result.getByTestId('closedStatsHeader-loading-spinner')).toBeInTheDocument(); }); }); @@ -194,16 +166,10 @@ describe.skip('AllCases', () => { }, }); - const wrapper = mount( - - - - ); + const result = appMockRender.render(); await waitFor(() => { - expect( - wrapper.find('[data-test-subj="configure-case-button"]').first().prop('isDisabled') - ).toBeTruthy(); + expect(result.getByTestId('configure-case-button')).toBeDisabled(); }); }); @@ -220,16 +186,10 @@ describe.skip('AllCases', () => { }, }); - const wrapper = mount( - - - - ); + const result = appMockRender.render(); await waitFor(() => { - expect( - wrapper.find('[data-test-subj="configure-case-button"]').first().prop('isDisabled') - ).toBeFalsy(); + expect(result.getByTestId('configure-case-button')).not.toBeDisabled(); }); }); From 5b8af091b426a8c39f85546fbaf7b2f01d46eb40 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 20 Sep 2022 11:18:14 -0500 Subject: [PATCH 23/76] skip suite failing es promotion. #141134 --- .../apis/management/index_lifecycle_management/nodes.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/api_integration/apis/management/index_lifecycle_management/nodes.js b/x-pack/test/api_integration/apis/management/index_lifecycle_management/nodes.js index 7514d079e5b41..d0603999c1b71 100644 --- a/x-pack/test/api_integration/apis/management/index_lifecycle_management/nodes.js +++ b/x-pack/test/api_integration/apis/management/index_lifecycle_management/nodes.js @@ -17,7 +17,8 @@ export default function ({ getService }) { const { getNodesStats } = initElasticsearchHelpers(getService); const { loadNodes, getNodeDetails } = registerHelpers({ supertest }); - describe('nodes', function () { + // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/141134 + describe.skip('nodes', function () { // Cloud disallows setting custom node attributes, so we can't use `NODE_CUSTOM_ATTRIBUTE` // to retrieve the IDs we expect. this.tags(['skipCloud']); From 2c8168e889521fe5df3ba17732d01daf2ae36e3f Mon Sep 17 00:00:00 2001 From: Candace Park <56409205+parkiino@users.noreply.github.com> Date: Tue, 20 Sep 2022 12:21:29 -0400 Subject: [PATCH 24/76] [Security Solution][Admin][Responder] Ctrl+a and Meta+a selects all text in console input (#140892) --- .../command_input/command_input.test.tsx | 12 ++++++++++ .../components/input_capture.tsx | 22 ++++++++++++++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/components/console/components/command_input/command_input.test.tsx b/x-pack/plugins/security_solution/public/management/components/console/components/command_input/command_input.test.tsx index 0ea71d71243af..9abfda11f1371 100644 --- a/x-pack/plugins/security_solution/public/management/components/console/components/command_input/command_input.test.tsx +++ b/x-pack/plugins/security_solution/public/management/components/console/components/command_input/command_input.test.tsx @@ -352,6 +352,18 @@ describe('When entering data into the Console input', () => { expect(getRightOfCursorText()).toEqual('ate'); }); + it('should select all text when ctrl or cmd + a is pressed', () => { + typeKeyboardKey('{ctrl>}a{/ctrl}'); + let selection = window.getSelection(); + expect(selection!.toString()).toEqual('isolate'); + + selection!.removeAllRanges(); + + typeKeyboardKey('{meta>}a{/meta}'); + selection = window.getSelection(); + expect(selection!.toString()).toEqual('isolate'); + }); + // FIXME:PT uncomment once task OLM task #4384 is implemented it.skip('should return original cursor position if input history is closed with no selection', async () => { typeKeyboardKey('{Enter}'); // add `isolate` to the input history diff --git a/x-pack/plugins/security_solution/public/management/components/console/components/command_input/components/input_capture.tsx b/x-pack/plugins/security_solution/public/management/components/console/components/command_input/components/input_capture.tsx index d6ed5a6cc243b..b4905807538f0 100644 --- a/x-pack/plugins/security_solution/public/management/components/console/components/command_input/components/input_capture.tsx +++ b/x-pack/plugins/security_solution/public/management/components/console/components/command_input/components/input_capture.tsx @@ -100,7 +100,7 @@ export const InputCapture = memo( const getTestId = useTestIdGenerator(useDataTestSubj()); // Reference to the `
` that take in focus (`tabIndex`) const focusEleRef = useRef(null); - + const childrenEleRef = useRef(null); const hiddenInputEleRef = useRef(null); const getTextSelection = useCallback((): string => { @@ -130,8 +130,22 @@ export const InputCapture = memo( const handleOnKeyDown = useCallback( (ev) => { - // allows for clipboard events to be captured via onPaste event handler + // handles the ctrl + a select and allows for clipboard events to be captured via onPaste event handler if (ev.metaKey || ev.ctrlKey) { + if (ev.key === 'a') { + ev.preventDefault(); + const selection = window.getSelection(); + if (selection && childrenEleRef.current) { + const range = document.createRange(); + range.selectNodeContents(childrenEleRef.current); + if (range.toString().length > 0) { + // clear any current selection + selection.removeAllRanges(); + // add the input text selection + selection.addRange(range); + } + } + } return; } @@ -260,7 +274,9 @@ export const InputCapture = memo( `Selection` object for 'focusNode` and `anchorNode` are within the input capture area. */}
- {children} +
+ {children} +
Date: Tue, 20 Sep 2022 18:22:54 +0200 Subject: [PATCH 25/76] [Fleet] Add fleet package policy bulkUpdate method (#140987) --- x-pack/plugins/fleet/server/mocks/index.ts | 1 + .../server/services/package_policy.test.ts | 534 ++++++++++++++++++ .../fleet/server/services/package_policy.ts | 152 ++++- .../server/services/package_policy_service.ts | 9 + 4 files changed, 673 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 47a918ed377f0..eb98d0e68eca8 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -120,6 +120,7 @@ export const createPackagePolicyServiceMock = (): jest.Mocked { }); }); + describe('bulkUpdate', () => { + beforeEach(() => { + appContextService.start(createAppContextStartContractMock()); + }); + afterEach(() => { + appContextService.stop(); + }); + + it('should throw if the user try to update input vars that are frozen', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const mockPackagePolicy = createPackagePolicyMock(); + const mockInputs = [ + { + config: {}, + enabled: true, + keep_enabled: true, + type: 'endpoint', + vars: { + dog: { + type: 'text', + value: 'dalmatian', + }, + cat: { + type: 'text', + value: 'siamese', + frozen: true, + }, + }, + streams: [ + { + data_stream: { + type: 'birds', + dataset: 'migratory.patterns', + }, + enabled: false, + id: `endpoint-migratory.patterns-${mockPackagePolicy.id}`, + vars: { + paths: { + value: ['north', 'south'], + type: 'text', + frozen: true, + }, + period: { + value: '6mo', + type: 'text', + }, + }, + }, + ], + }, + ]; + const inputsUpdate = [ + { + config: {}, + enabled: false, + type: 'endpoint', + vars: { + dog: { + type: 'text', + value: 'labrador', + }, + cat: { + type: 'text', + value: 'tabby', + }, + }, + streams: [ + { + data_stream: { + type: 'birds', + dataset: 'migratory.patterns', + }, + enabled: false, + id: `endpoint-migratory.patterns-${mockPackagePolicy.id}`, + vars: { + paths: { + value: ['east', 'west'], + type: 'text', + }, + period: { + value: '12mo', + type: 'text', + }, + }, + }, + ], + }, + ]; + const attributes = { + ...mockPackagePolicy, + inputs: mockInputs, + }; + + savedObjectsClient.bulkGet.mockResolvedValue({ + saved_objects: [ + { + id: 'test', + type: 'abcd', + references: [], + version: 'test', + attributes, + }, + ], + }); + + savedObjectsClient.update.mockImplementation( + async ( + type: string, + id: string, + attrs: any + ): Promise> => { + savedObjectsClient.get.mockResolvedValue({ + id: 'test', + type: 'abcd', + references: [], + version: 'test', + attributes: attrs, + }); + return attrs; + } + ); + const elasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const res = packagePolicyService.bulkUpdate( + savedObjectsClient, + elasticsearchClient, + + [{ ...mockPackagePolicy, inputs: inputsUpdate }] + ); + + await expect(res).rejects.toThrow('cat is a frozen variable and cannot be modified'); + }); + + it('should allow to update input vars that are frozen with the force flag', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const mockPackagePolicy = createPackagePolicyMock(); + const mockInputs = [ + { + config: {}, + enabled: true, + keep_enabled: true, + type: 'endpoint', + vars: { + dog: { + type: 'text', + value: 'dalmatian', + }, + cat: { + type: 'text', + value: 'siamese', + frozen: true, + }, + }, + streams: [ + { + data_stream: { + type: 'birds', + dataset: 'migratory.patterns', + }, + enabled: false, + id: `endpoint-migratory.patterns-${mockPackagePolicy.id}`, + vars: { + paths: { + value: ['north', 'south'], + type: 'text', + frozen: true, + }, + period: { + value: '6mo', + type: 'text', + }, + }, + }, + ], + }, + ]; + const inputsUpdate = [ + { + config: {}, + enabled: false, + type: 'endpoint', + vars: { + dog: { + type: 'text', + value: 'labrador', + }, + cat: { + type: 'text', + value: 'tabby', + }, + }, + streams: [ + { + data_stream: { + type: 'birds', + dataset: 'migratory.patterns', + }, + enabled: false, + id: `endpoint-migratory.patterns-${mockPackagePolicy.id}`, + vars: { + paths: { + value: ['east', 'west'], + type: 'text', + }, + period: { + value: '12mo', + type: 'text', + }, + }, + }, + ], + }, + ]; + const attributes = { + ...mockPackagePolicy, + inputs: mockInputs, + }; + + savedObjectsClient.bulkGet.mockResolvedValue({ + saved_objects: [ + { + id: 'test', + type: 'abcd', + references: [], + version: 'test', + attributes, + }, + ], + }); + + savedObjectsClient.bulkUpdate.mockImplementation( + async ( + objs: Array<{ + type: string; + id: string; + attributes: any; + }> + ) => { + const newObjs = objs.map((obj) => ({ + id: 'test', + type: 'abcd', + references: [], + version: 'test', + attributes: obj.attributes, + })); + savedObjectsClient.bulkGet.mockResolvedValue({ + saved_objects: newObjs, + }); + return { + saved_objects: newObjs, + }; + } + ); + const elasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const result = await packagePolicyService.bulkUpdate( + savedObjectsClient, + elasticsearchClient, + [{ ...mockPackagePolicy, inputs: inputsUpdate }], + { force: true } + ); + + const [modifiedInput] = result![0].inputs; + expect(modifiedInput.enabled).toEqual(true); + expect(modifiedInput.vars!.dog.value).toEqual('labrador'); + expect(modifiedInput.vars!.cat.value).toEqual('tabby'); + const [modifiedStream] = modifiedInput.streams; + expect(modifiedStream.vars!.paths.value).toEqual(expect.arrayContaining(['east', 'west'])); + expect(modifiedStream.vars!.period.value).toEqual('12mo'); + }); + + it('should add new input vars when updating', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const mockPackagePolicy = createPackagePolicyMock(); + const mockInputs = [ + { + config: {}, + enabled: true, + keep_enabled: true, + type: 'endpoint', + vars: { + dog: { + type: 'text', + value: 'dalmatian', + }, + cat: { + type: 'text', + value: 'siamese', + frozen: true, + }, + }, + streams: [ + { + data_stream: { + type: 'birds', + dataset: 'migratory.patterns', + }, + enabled: false, + id: `endpoint-migratory.patterns-${mockPackagePolicy.id}`, + vars: { + paths: { + value: ['north', 'south'], + type: 'text', + frozen: true, + }, + }, + }, + ], + }, + ]; + const inputsUpdate = [ + { + config: {}, + enabled: false, + type: 'endpoint', + vars: { + dog: { + type: 'text', + value: 'labrador', + }, + cat: { + type: 'text', + value: 'siamese', + }, + }, + streams: [ + { + data_stream: { + type: 'birds', + dataset: 'migratory.patterns', + }, + enabled: false, + id: `endpoint-migratory.patterns-${mockPackagePolicy.id}`, + vars: { + paths: { + value: ['north', 'south'], + type: 'text', + }, + period: { + value: '12mo', + type: 'text', + }, + }, + }, + ], + }, + ]; + const attributes = { + ...mockPackagePolicy, + inputs: mockInputs, + }; + + savedObjectsClient.bulkGet.mockResolvedValue({ + saved_objects: [ + { + id: 'test', + type: 'abcd', + references: [], + version: 'test', + attributes, + }, + ], + }); + + savedObjectsClient.bulkUpdate.mockImplementation( + async ( + objs: Array<{ + type: string; + id: string; + attributes: any; + }> + ) => { + const newObjs = objs.map((obj) => ({ + id: 'test', + type: 'abcd', + references: [], + version: 'test', + attributes: obj.attributes, + })); + savedObjectsClient.bulkGet.mockResolvedValue({ + saved_objects: newObjs, + }); + return { + saved_objects: newObjs, + }; + } + ); + + const elasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const result = await packagePolicyService.bulkUpdate( + savedObjectsClient, + elasticsearchClient, + [{ ...mockPackagePolicy, inputs: inputsUpdate }] + ); + + const [modifiedInput] = result![0].inputs; + expect(modifiedInput.enabled).toEqual(true); + expect(modifiedInput.vars!.dog.value).toEqual('labrador'); + expect(modifiedInput.vars!.cat.value).toEqual('siamese'); + const [modifiedStream] = modifiedInput.streams; + expect(modifiedStream.vars!.paths.value).toEqual(expect.arrayContaining(['north', 'south'])); + expect(modifiedStream.vars!.period.value).toEqual('12mo'); + }); + + it('should update elasticsearch.privileges.cluster when updating', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const mockPackagePolicy = createPackagePolicyMock(); + + const attributes = { + ...mockPackagePolicy, + inputs: [], + }; + (getPackageInfo as jest.Mock).mockImplementation(async (params) => { + return Promise.resolve({ + ...(await mockedGetPackageInfo(params)), + elasticsearch: { + privileges: { + cluster: ['monitor'], + }, + }, + } as PackageInfo); + }); + + savedObjectsClient.bulkGet.mockResolvedValue({ + saved_objects: [ + { + id: 'test', + type: 'abcd', + references: [], + version: 'test', + attributes, + }, + ], + }); + + savedObjectsClient.bulkUpdate.mockImplementation( + async ( + objs: Array<{ + type: string; + id: string; + attributes: any; + }> + ) => { + const newObjs = objs.map((obj) => ({ + id: 'test', + type: 'abcd', + references: [], + version: 'test', + attributes: obj.attributes, + })); + savedObjectsClient.bulkGet.mockResolvedValue({ + saved_objects: newObjs, + }); + return { + saved_objects: newObjs, + }; + } + ); + + const elasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const result = await packagePolicyService.bulkUpdate( + savedObjectsClient, + elasticsearchClient, + [{ ...mockPackagePolicy, inputs: [] }] + ); + + expect(result![0].elasticsearch).toMatchObject({ privileges: { cluster: ['monitor'] } }); + }); + + it('should not mutate packagePolicyUpdate object when trimming whitespace', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const mockPackagePolicy = createPackagePolicyMock(); + + const attributes = { + ...mockPackagePolicy, + inputs: [], + }; + + savedObjectsClient.bulkGet.mockResolvedValue({ + saved_objects: [ + { + id: 'test', + type: 'abcd', + references: [], + version: 'test', + attributes, + }, + ], + }); + + savedObjectsClient.bulkUpdate.mockImplementation( + async ( + objs: Array<{ + type: string; + id: string; + attributes: any; + }> + ) => { + const newObjs = objs.map((obj) => ({ + id: 'test', + type: 'abcd', + references: [], + version: 'test', + attributes: obj.attributes, + })); + savedObjectsClient.bulkGet.mockResolvedValue({ + saved_objects: newObjs, + }); + return { + saved_objects: newObjs, + }; + } + ); + + const elasticsearchClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const result = await packagePolicyService.bulkUpdate( + savedObjectsClient, + elasticsearchClient, + // this mimics the way that OSQuery plugin create immutable objects + [ + produce( + { ...mockPackagePolicy, name: ' test ', inputs: [] }, + (draft) => draft + ), + ] + ); + + expect(result![0].name).toEqual('test'); + }); + }); + describe('delete', () => { it('should allow to delete a package policy', async () => {}); }); diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 0dd16f51dac5f..4b4b9615009f3 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -542,32 +542,107 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const newPolicy = (await this.get(soClient, id)) as PackagePolicy; - if (packagePolicy.package) { - await removeOldAssets({ - soClient, - pkgName: packagePolicy.package.name, - currentVersion: packagePolicy.package.version, - }); + await updatePackagePolicyVersion(packagePolicyUpdate, soClient, currentVersion); - if (packagePolicy.package.version !== currentVersion) { - const upgradeTelemetry: PackageUpdateEvent = { - packageName: packagePolicy.package.name, - currentVersion: currentVersion || 'unknown', - newVersion: packagePolicy.package.version, - status: 'success', - eventType: 'package-policy-upgrade' as UpdateEventType, - }; - sendTelemetryEvents( - appContextService.getLogger(), - appContextService.getTelemetryEventsSender(), - upgradeTelemetry - ); - appContextService.getLogger().info(`Package policy upgraded successfully`); - appContextService.getLogger().debug(JSON.stringify(upgradeTelemetry)); - } + return newPolicy; + } + + public async bulkUpdate( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + packagePolicyUpdates: Array, + options?: { user?: AuthenticatedUser; force?: boolean }, + currentVersion?: string + ): Promise { + const oldPackagePolicies = await this.getByIDs( + soClient, + packagePolicyUpdates.map((p) => p.id) + ); + + if (!oldPackagePolicies || oldPackagePolicies.length === 0) { + throw new Error('Package policy not found'); } - return newPolicy; + const packageInfos = await getPackageInfoForPackagePolicies(packagePolicyUpdates, soClient); + + await soClient.bulkUpdate( + await pMap( + packagePolicyUpdates, + async (packagePolicyUpdate) => { + const id = packagePolicyUpdate.id; + const packagePolicy = { ...packagePolicyUpdate, name: packagePolicyUpdate.name.trim() }; + const oldPackagePolicy = oldPackagePolicies.find((p) => p.id === id); + if (!oldPackagePolicy) { + throw new Error('Package policy not found'); + } + + const { version, ...restOfPackagePolicy } = packagePolicy; + + if (packagePolicyUpdate.is_managed && !options?.force) { + throw new PackagePolicyRestrictionRelatedError(`Cannot update package policy ${id}`); + } + + let inputs = restOfPackagePolicy.inputs.map((input) => + assignStreamIdToInput(oldPackagePolicy.id, input) + ); + + inputs = enforceFrozenInputs(oldPackagePolicy.inputs, inputs, options?.force); + let elasticsearch: PackagePolicy['elasticsearch']; + if (packagePolicy.package?.name) { + const pkgInfo = packageInfos.get( + `${packagePolicy.package.name}-${packagePolicy.package.version}` + ); + if (pkgInfo) { + validatePackagePolicyOrThrow(packagePolicy, pkgInfo); + + inputs = await _compilePackagePolicyInputs(pkgInfo, packagePolicy.vars || {}, inputs); + elasticsearch = pkgInfo.elasticsearch; + } + } + + // Handle component template/mappings updates for experimental features, e.g. synthetic source + await handleExperimentalDatastreamFeatureOptIn({ soClient, esClient, packagePolicy }); + + return { + type: SAVED_OBJECT_TYPE, + id, + attributes: { + ...restOfPackagePolicy, + inputs, + elasticsearch, + revision: oldPackagePolicy.revision + 1, + updated_at: new Date().toISOString(), + updated_by: options?.user?.username ?? 'system', + }, + version, + }; + }, + { concurrency: 50 } + ) + ); + + const agentPolicyIds = new Set(packagePolicyUpdates.map((p) => p.policy_id)); + + await pMap(agentPolicyIds, async (agentPolicyId) => { + // Bump revision of associated agent policy + await agentPolicyService.bumpRevision(soClient, esClient, agentPolicyId, { + user: options?.user, + }); + }); + + const newPolicies = await this.getByIDs( + soClient, + packagePolicyUpdates.map((p) => p.id) + ); + + await pMap( + packagePolicyUpdates, + async (packagePolicy) => + await updatePackagePolicyVersion(packagePolicy, soClient, currentVersion), + { concurrency: 50 } + ); + + return newPolicies; } public async delete( @@ -1768,6 +1843,37 @@ async function validateIsNotHostedPolicy( } } +export async function updatePackagePolicyVersion( + packagePolicy: (NewPackagePolicy & { version?: string; id: string }) | UpdatePackagePolicy, + soClient: SavedObjectsClientContract, + currentVersion?: string +) { + if (packagePolicy.package) { + await removeOldAssets({ + soClient, + pkgName: packagePolicy.package.name, + currentVersion: packagePolicy.package.version, + }); + + if (packagePolicy.package.version !== currentVersion) { + const upgradeTelemetry: PackageUpdateEvent = { + packageName: packagePolicy.package.name, + currentVersion: currentVersion || 'unknown', + newVersion: packagePolicy.package.version, + status: 'success', + eventType: 'package-policy-upgrade' as UpdateEventType, + }; + sendTelemetryEvents( + appContextService.getLogger(), + appContextService.getTelemetryEventsSender(), + upgradeTelemetry + ); + appContextService.getLogger().info(`Package policy upgraded successfully`); + appContextService.getLogger().debug(JSON.stringify(upgradeTelemetry)); + } + } +} + function deepMergeVars(original: any, override: any, keepOriginalValue = false): any { if (!original.vars) { original.vars = { ...override.vars }; diff --git a/x-pack/plugins/fleet/server/services/package_policy_service.ts b/x-pack/plugins/fleet/server/services/package_policy_service.ts index 36f19207bf107..4dd36a8c4fdf4 100644 --- a/x-pack/plugins/fleet/server/services/package_policy_service.ts +++ b/x-pack/plugins/fleet/server/services/package_policy_service.ts @@ -24,6 +24,7 @@ import type { import type { ExperimentalDataStreamFeature } from '../../common/types'; import type { NewPackagePolicy, UpdatePackagePolicy, PackagePolicy } from '../types'; import type { ExternalCallback } from '..'; + import type { NewPackagePolicyWithId } from './package_policy'; export interface PackagePolicyService { @@ -60,6 +61,14 @@ export interface PackagePolicyClient { } ): Promise; + bulkUpdate( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + packagePolicyUpdates: Array, + options?: { user?: AuthenticatedUser; force?: boolean }, + currentVersion?: string + ): Promise; + get(soClient: SavedObjectsClientContract, id: string): Promise; findAllForAgentPolicy( From e46863227f09cbf336d0d453dfe6718c6b7d0952 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Tue, 20 Sep 2022 11:28:40 -0500 Subject: [PATCH 26/76] skip flaky suite. #115307 --- .../edit_policy/form_validation/timing.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/form_validation/timing.test.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/form_validation/timing.test.ts index 3e29bc5d76580..7db483d6d0ef2 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/form_validation/timing.test.ts +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/form_validation/timing.test.ts @@ -12,7 +12,8 @@ import { PhaseWithTiming } from '../../../../common/types'; import { setupEnvironment } from '../../helpers'; import { setupValidationTestBed, ValidationTestBed } from './validation.helpers'; -describe(' timing validation', () => { +// FLAKY: https://github.com/elastic/kibana/issues/115307 +describe.skip(' timing validation', () => { let testBed: ValidationTestBed; let actions: ValidationTestBed['actions']; const { httpSetup, httpRequestsMockHelpers } = setupEnvironment(); From 6dcaa00ebfd35317100fc2a0bafa0afa286d924c Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Tue, 20 Sep 2022 18:30:08 +0200 Subject: [PATCH 27/76] show No anomalies found instead of empty cells (#141098) --- .../anomaly_detection_panel/table.tsx | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/ml/public/application/overview/components/anomaly_detection_panel/table.tsx b/x-pack/plugins/ml/public/application/overview/components/anomaly_detection_panel/table.tsx index f5d842d5d7493..2e2444fff853b 100644 --- a/x-pack/plugins/ml/public/application/overview/components/anomaly_detection_panel/table.tsx +++ b/x-pack/plugins/ml/public/application/overview/components/anomaly_detection_panel/table.tsx @@ -93,19 +93,14 @@ export const AnomalyDetectionTable: FC = ({ items, statsBarData, chartsSe if (!swimLaneData) return null; - const hasResults = swimLaneData.points.length > 0; - - const noDatWarning = hasResults ? ( - - ) : ( - - ); + if (swimLaneData.points.length > 0 && swimLaneData.points.every((v) => v.value === 0)) { + return ( + + ); + } return ( = ({ items, statsBarData, chartsSe showTimeline={false} showYAxis={false} showLegend={false} - noDataWarning={noDatWarning} + noDataWarning={ + + } chartsService={chartsService} /> ); From 063f50dc7e87044bebab58f49e588ffe9fa6cc85 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Tue, 20 Sep 2022 09:34:49 -0700 Subject: [PATCH 28/76] Bumps version manifest for 8.4 to 8.4.3 (#141115) --- versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.json b/versions.json index 11776f57850ee..5b70f97320e23 100644 --- a/versions.json +++ b/versions.json @@ -8,7 +8,7 @@ "currentMinor": true }, { - "version": "8.4.2", + "version": "8.4.3", "branch": "8.4", "currentMajor": true, "previousMinor": true From 19097ddae95f32b113a928c41e2f77cbec8c90f4 Mon Sep 17 00:00:00 2001 From: Rodney Norris Date: Tue, 20 Sep 2022 11:43:04 -0500 Subject: [PATCH 29/76] remove not deployed from inference pipeline card (#141091) --- .../pipelines/inference_pipeline_card.test.tsx | 6 ------ .../pipelines/inference_pipeline_card.tsx | 15 +++++---------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.test.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.test.tsx index fab7620f7f232..f329e0bf2b677 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.test.tsx @@ -38,10 +38,4 @@ describe('InfererencePipelineCard', () => { const health = wrapper.find(EuiHealth); expect(health.prop('children')).toEqual('Deployed'); }); - - it('renders an undeployed item', () => { - const wrapper = shallow(); - const health = wrapper.find(EuiHealth); - expect(health.prop('children')).toEqual('Not deployed'); - }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.tsx index 40339a90ab423..270e2dc5d1714 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/pipelines/inference_pipeline_card.tsx @@ -49,11 +49,6 @@ export const InferencePipelineCard: React.FC = ({ defaultMessage: 'Deployed', }); - const notDeployedText = i18n.translate( - 'xpack.enterpriseSearch.inferencePipelineCard.isNotDeployed', - { defaultMessage: 'Not deployed' } - ); - const actionButton = ( = ({ - - - {isDeployed ? deployedText : notDeployedText} - - + {isDeployed && ( + + {deployedText} + + )} From 6fda4152787072e4fb0f90aa8213b0f25f3bbe5f Mon Sep 17 00:00:00 2001 From: Marta Bondyra <4283304+mbondyra@users.noreply.github.com> Date: Tue, 20 Sep 2022 18:46:29 +0200 Subject: [PATCH 30/76] [Lens] Tsvb to lens for annotations (#140718) * fix typo * [TSVB to Lens] tsvb part * Lens part * tests * add ignoreGlobalFilters * import corrected * adding star filled * widen the limits, spread the horizons Co-authored-by: Joe Reuter --- packages/kbn-optimizer/limits.yml | 2 +- .../expression_xy/common/constants.ts | 1 + .../expression_xy/public/helpers/icon.ts | 6 + .../event_annotation/common/constants.ts | 1 + .../public/convert_to_lens/index.ts | 2 +- .../lib/configurations/xy/layers.test.ts | 240 +++++++++++++++++- .../lib/configurations/xy/layers.ts | 98 ++++++- .../convert_to_lens/timeseries/index.ts | 7 +- .../public/convert_to_lens/top_n/index.ts | 4 +- .../common/convert_to_lens/index.ts | 1 + .../convert_to_lens/types/configurations.ts | 29 ++- .../common/convert_to_lens/utils.ts | 13 + .../lens/public/app_plugin/mounter.tsx | 2 +- .../editor_frame/state_helpers.ts | 10 +- x-pack/plugins/lens/public/types.ts | 6 +- .../public/visualizations/xy/state_helpers.ts | 17 +- .../visualizations/xy/visualization.tsx | 4 +- .../annotations_config_panel/icon_set.ts | 6 + 18 files changed, 430 insertions(+), 19 deletions(-) create mode 100644 src/plugins/visualizations/common/convert_to_lens/utils.ts diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 0c6b68f79cebd..b8de0b6daa3b5 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -32,7 +32,7 @@ pageLoadAssetSize: embeddableEnhanced: 22107 enterpriseSearch: 35741 esUiShared: 326654 - eventAnnotation: 19500 + eventAnnotation: 20500 expressionError: 22127 expressionGauge: 25000 expressionHeatmap: 27505 diff --git a/src/plugins/chart_expressions/expression_xy/common/constants.ts b/src/plugins/chart_expressions/expression_xy/common/constants.ts index 8116f0146f19a..1c15be76bc509 100644 --- a/src/plugins/chart_expressions/expression_xy/common/constants.ts +++ b/src/plugins/chart_expressions/expression_xy/common/constants.ts @@ -123,6 +123,7 @@ export const AvailableReferenceLineIcons = { MAP_MARKER: 'mapMarker', PIN_FILLED: 'pinFilled', STAR_EMPTY: 'starEmpty', + STAR_FILLED: 'starFilled', TAG: 'tag', TRIANGLE: 'triangle', } as const; diff --git a/src/plugins/chart_expressions/expression_xy/public/helpers/icon.ts b/src/plugins/chart_expressions/expression_xy/public/helpers/icon.ts index a74d4196f487d..652dbe168f58b 100644 --- a/src/plugins/chart_expressions/expression_xy/public/helpers/icon.ts +++ b/src/plugins/chart_expressions/expression_xy/public/helpers/icon.ts @@ -94,6 +94,12 @@ export const iconSet = [ value: AvailableReferenceLineIcons.STAR_EMPTY, label: i18n.translate('expressionXY.xyChart.iconSelect.starLabel', { defaultMessage: 'Star' }), }, + { + value: AvailableReferenceLineIcons.STAR_FILLED, + label: i18n.translate('expressionXY.xyChart.iconSelect.starFilledLabel', { + defaultMessage: 'Star filled', + }), + }, { value: AvailableReferenceLineIcons.TAG, label: i18n.translate('expressionXY.xyChart.iconSelect.tagIconLabel', { diff --git a/src/plugins/event_annotation/common/constants.ts b/src/plugins/event_annotation/common/constants.ts index 3338450b64ce5..3f3f9877b9786 100644 --- a/src/plugins/event_annotation/common/constants.ts +++ b/src/plugins/event_annotation/common/constants.ts @@ -19,6 +19,7 @@ export const AvailableAnnotationIcons = { MAP_MARKER: 'mapMarker', PIN_FILLED: 'pinFilled', STAR_EMPTY: 'starEmpty', + STAR_FILLED: 'starFilled', TAG: 'tag', TRIANGLE: 'triangle', } as const; diff --git a/src/plugins/vis_types/timeseries/public/convert_to_lens/index.ts b/src/plugins/vis_types/timeseries/public/convert_to_lens/index.ts index c9e72040e26dd..5b92c0ab21668 100644 --- a/src/plugins/vis_types/timeseries/public/convert_to_lens/index.ts +++ b/src/plugins/vis_types/timeseries/public/convert_to_lens/index.ts @@ -33,7 +33,7 @@ const getConvertFnByType = (type: PANEL_TYPES) => { */ export const convertTSVBtoLensConfiguration = async (model: Panel, timeRange?: TimeRange) => { // Disables the option for not supported charts, for the string mode and for series with annotations - if (!model.use_kibana_indexes || (model.annotations && model.annotations.length > 0)) { + if (!model.use_kibana_indexes) { return null; } // Disables if model is invalid diff --git a/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/configurations/xy/layers.test.ts b/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/configurations/xy/layers.test.ts index 656dc3a8c4290..f869ac64380c6 100644 --- a/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/configurations/xy/layers.test.ts +++ b/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/configurations/xy/layers.test.ts @@ -17,6 +17,12 @@ import { } from '../../convert'; import { getLayers } from './layers'; import { createPanel, createSeries } from '../../__mocks__'; +import { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; +import type { DataView } from '@kbn/data-views-plugin/public'; + +jest.mock('uuid', () => ({ + v4: () => 'test-id', +})); describe('getLayers', () => { const dataSourceLayers: Record = [ @@ -200,6 +206,84 @@ describe('getLayers', () => { const panelWithPercentileRankMetric = createPanel({ series: [createSeries({ metrics: percentileRankMetrics })], }); + const panelWithSingleAnnotation = createPanel({ + annotations: [ + { + fields: 'geo.src,host', + template: 'Security Error from {{geo.src}} on {{host}}', + query_string: { + query: 'tags:error AND tags:security', + language: 'lucene', + }, + id: 'ann1', + color: 'rgba(211,49,21,0.7)', + time_field: 'timestamp', + icon: 'fa-asterisk', + ignore_global_filters: 1, + ignore_panel_filters: 1, + hidden: true, + index_pattern: { + id: 'test', + }, + }, + ], + series: [createSeries({ metrics: staticValueMetric })], + }); + const panelWithMultiAnnotations = createPanel({ + annotations: [ + { + fields: 'geo.src,host', + template: 'Security Error from {{geo.src}} on {{host}}', + query_string: { + query: 'tags:error AND tags:security', + language: 'lucene', + }, + id: 'ann1', + color: 'rgba(211,49,21,0.7)', + time_field: 'timestamp', + icon: 'fa-asterisk', + ignore_global_filters: 1, + ignore_panel_filters: 1, + hidden: true, + index_pattern: { + id: 'test', + }, + }, + { + query_string: { + query: 'tags: error AND tags: security', + language: 'kql', + }, + id: 'ann2', + color: 'blue', + time_field: 'timestamp', + icon: 'error-icon', + ignore_global_filters: 0, // todo test ignore when PR is r + ignore_panel_filters: 0, + index_pattern: { + id: 'test', + }, + }, + { + fields: 'category.keyword,price', + template: 'Will be ignored', + query_string: { + query: 'category.keyword:*', + language: 'kql', + }, + id: 'ann3', + color: 'red', + time_field: 'order_date', + icon: undefined, + ignore_global_filters: 1, + ignore_panel_filters: 1, + index_pattern: { + id: 'test2', + }, + }, + ], + series: [createSeries({ metrics: staticValueMetric })], + }); test.each<[string, [Record, Panel], Array>]>([ [ @@ -282,7 +366,159 @@ describe('getLayers', () => { }, ], ], - ])('should return %s', (_, input, expected) => { - expect(getLayers(...input)).toEqual(expected.map(expect.objectContaining)); + [ + 'annotation layer gets correct params and converts color, extraFields and icons', + [dataSourceLayersWithStatic, panelWithSingleAnnotation], + [ + { + layerType: 'referenceLine', + accessors: ['column-id-1'], + layerId: 'test-layer-1', + yConfig: [ + { + forAccessor: 'column-id-1', + axisMode: 'right', + color: '#68BC00', + fill: 'below', + }, + ], + }, + { + layerId: 'test-id', + layerType: 'annotations', + ignoreGlobalFilters: true, + annotations: [ + { + color: '#D33115', + extraFields: ['geo.src'], + filter: { + language: 'lucene', + query: 'tags:error AND tags:security', + type: 'kibana_query', + }, + icon: 'asterisk', + id: 'ann1', + isHidden: true, + key: { + type: 'point_in_time', + }, + label: 'Event', + timeField: 'timestamp', + type: 'query', + }, + ], + indexPatternId: 'test', + }, + ], + ], + [ + 'multiple annotations with different data views create separate layers', + [dataSourceLayersWithStatic, panelWithMultiAnnotations], + [ + { + layerType: 'referenceLine', + accessors: ['column-id-1'], + layerId: 'test-layer-1', + yConfig: [ + { + forAccessor: 'column-id-1', + axisMode: 'right', + color: '#68BC00', + fill: 'below', + }, + ], + }, + { + layerId: 'test-id', + layerType: 'annotations', + ignoreGlobalFilters: true, + annotations: [ + { + color: '#D33115', + extraFields: ['geo.src'], + filter: { + language: 'lucene', + query: 'tags:error AND tags:security', + type: 'kibana_query', + }, + icon: 'asterisk', + id: 'ann1', + isHidden: true, + key: { + type: 'point_in_time', + }, + label: 'Event', + timeField: 'timestamp', + type: 'query', + }, + { + color: '#0000FF', + filter: { + language: 'kql', + query: 'tags: error AND tags: security', + type: 'kibana_query', + }, + icon: 'triangle', + id: 'ann2', + key: { + type: 'point_in_time', + }, + label: 'Event', + timeField: 'timestamp', + type: 'query', + }, + ], + indexPatternId: 'test', + }, + { + layerId: 'test-id', + layerType: 'annotations', + ignoreGlobalFilters: true, + annotations: [ + { + color: '#FF0000', + extraFields: ['category.keyword', 'price'], + filter: { + language: 'kql', + query: 'category.keyword:*', + type: 'kibana_query', + }, + icon: 'triangle', + id: 'ann3', + key: { + type: 'point_in_time', + }, + label: 'Event', + timeField: 'order_date', + type: 'query', + }, + ], + indexPatternId: 'test2', + }, + ], + ], + ])('should return %s', async (_, input, expected) => { + const layers = await getLayers(...input, indexPatternsService as DataViewsPublicPluginStart); + expect(layers).toEqual(expected.map(expect.objectContaining)); }); }); + +const mockedIndices = [ + { + id: 'test', + title: 'test', + getFieldByName: (name: string) => ({ aggregatable: name !== 'host' }), + }, +] as unknown as DataView[]; + +const indexPatternsService = { + getDefault: jest.fn(() => Promise.resolve({ id: 'default', title: 'index' })), + get: jest.fn(() => Promise.resolve(mockedIndices[0])), + find: jest.fn((search: string, size: number) => { + if (size !== 1) { + // shouldn't request more than one data view since there is a significant performance penalty + throw new Error('trying to fetch too many data views'); + } + return Promise.resolve(mockedIndices || []); + }), +} as unknown as DataViewsPublicPluginStart; diff --git a/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/configurations/xy/layers.ts b/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/configurations/xy/layers.ts index 0f411cc9d8fe6..064471e9fcfaf 100644 --- a/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/configurations/xy/layers.ts +++ b/src/plugins/vis_types/timeseries/public/convert_to_lens/lib/configurations/xy/layers.ts @@ -7,13 +7,23 @@ */ import { + EventAnnotationConfig, FillTypes, + XYAnnotationsLayerConfig, XYLayerConfig, YAxisMode, } from '@kbn/visualizations-plugin/common/convert_to_lens'; import { PaletteOutput } from '@kbn/coloring'; +import { v4 } from 'uuid'; +import { transparentize } from '@elastic/eui'; +import Color from 'color'; +import { euiLightVars } from '@kbn/ui-theme'; +import { groupBy } from 'lodash'; +import { DataViewsPublicPluginStart, DataView } from '@kbn/data-plugin/public/data_views'; +import { fetchIndexPattern } from '../../../../../common/index_patterns_utils'; +import { ICON_TYPES_MAP } from '../../../../application/visualizations/constants'; import { SUPPORTED_METRICS } from '../../metrics'; -import type { Metric, Panel } from '../../../../../common/types'; +import type { Annotation, Metric, Panel } from '../../../../../common/types'; import { getSeriesAgg } from '../../series'; import { isPercentileRanksColumnWithMeta, @@ -51,11 +61,16 @@ function getColor( return seriesColor; } -export const getLayers = ( +function nonNullable(value: T): value is NonNullable { + return value !== null && value !== undefined; +} + +export const getLayers = async ( dataSourceLayers: Record, - model: Panel -): XYLayerConfig[] => { - return Object.keys(dataSourceLayers).map((key) => { + model: Panel, + dataViews: DataViewsPublicPluginStart +): Promise => { + const nonAnnotationsLayers: XYLayerConfig[] = Object.keys(dataSourceLayers).map((key) => { const series = model.series[parseInt(key, 10)]; const { metrics, seriesAgg } = getSeriesAgg(series.metrics); const dataSourceLayer = dataSourceLayers[parseInt(key, 10)]; @@ -112,4 +127,77 @@ export const getLayers = ( }; } }); + if (!model.annotations || !model.annotations.length) { + return nonAnnotationsLayers; + } + + const annotationsByIndexPattern = groupBy( + model.annotations, + (a) => typeof a.index_pattern === 'object' && 'id' in a.index_pattern && a.index_pattern.id + ); + + const annotationsLayers: Array = await Promise.all( + Object.entries(annotationsByIndexPattern).map(async ([indexPatternId, annotations]) => { + const convertedAnnotations: EventAnnotationConfig[] = []; + const { indexPattern } = (await fetchIndexPattern({ id: indexPatternId }, dataViews)) || {}; + + if (indexPattern) { + annotations.forEach((a: Annotation) => { + const lensAnnotation = convertAnnotation(a, indexPattern); + if (lensAnnotation) { + convertedAnnotations.push(lensAnnotation); + } + }); + return { + layerId: v4(), + layerType: 'annotations', + ignoreGlobalFilters: true, + annotations: convertedAnnotations, + indexPatternId, + }; + } + }) + ); + + return nonAnnotationsLayers.concat(...annotationsLayers.filter(nonNullable)); +}; + +const convertAnnotation = ( + annotation: Annotation, + dataView: DataView +): EventAnnotationConfig | undefined => { + if (annotation.query_string) { + const extraFields = annotation.fields + ? annotation.fields + ?.replace(/\s/g, '') + ?.split(',') + .map((field) => { + const dataViewField = dataView.getFieldByName(field); + return dataViewField && dataViewField.aggregatable ? field : undefined; + }) + .filter(nonNullable) + : undefined; + return { + type: 'query', + id: annotation.id, + label: 'Event', + key: { + type: 'point_in_time', + }, + color: new Color(transparentize(annotation.color || euiLightVars.euiColorAccent, 1)).hex(), + timeField: annotation.time_field, + icon: + annotation.icon && + ICON_TYPES_MAP[annotation.icon] && + typeof ICON_TYPES_MAP[annotation.icon] === 'string' + ? ICON_TYPES_MAP[annotation.icon] + : 'triangle', + filter: { + type: 'kibana_query', + ...annotation.query_string, + }, + extraFields, + isHidden: annotation.hidden, + }; + } }; diff --git a/src/plugins/vis_types/timeseries/public/convert_to_lens/timeseries/index.ts b/src/plugins/vis_types/timeseries/public/convert_to_lens/timeseries/index.ts index c3f5900b35eb3..25cd1fa962d0d 100644 --- a/src/plugins/vis_types/timeseries/public/convert_to_lens/timeseries/index.ts +++ b/src/plugins/vis_types/timeseries/public/convert_to_lens/timeseries/index.ts @@ -9,6 +9,7 @@ import { parseTimeShift } from '@kbn/data-plugin/common'; import { Layer } from '@kbn/visualizations-plugin/common/convert_to_lens'; import uuid from 'uuid'; +import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public'; import { Panel } from '../../../common/types'; import { PANEL_TYPES } from '../../../common/enums'; import { getDataViewsStart } from '../../services'; @@ -37,7 +38,7 @@ const excludeMetaFromLayers = (layers: Record): Record { - const dataViews = getDataViewsStart(); + const dataViews: DataViewsPublicPluginStart = getDataViewsStart(); const extendedLayers: Record = {}; const seriesNum = model.series.filter((series) => !series.hidden).length; @@ -96,9 +97,11 @@ export const convertToLens: ConvertTsvbToLensVisualization = async (model: Panel }; } + const configLayers = await getLayers(extendedLayers, model, dataViews); + return { type: 'lnsXY', layers: Object.values(excludeMetaFromLayers(extendedLayers)), - configuration: getConfiguration(model, getLayers(extendedLayers, model)), + configuration: getConfiguration(model, configLayers), }; }; diff --git a/src/plugins/vis_types/timeseries/public/convert_to_lens/top_n/index.ts b/src/plugins/vis_types/timeseries/public/convert_to_lens/top_n/index.ts index 1afbd89e7027f..2622fadf5e753 100644 --- a/src/plugins/vis_types/timeseries/public/convert_to_lens/top_n/index.ts +++ b/src/plugins/vis_types/timeseries/public/convert_to_lens/top_n/index.ts @@ -79,9 +79,11 @@ export const convertToLens: ConvertTsvbToLensVisualization = async (model, timeR }; } + const configLayers = await getLayers(extendedLayers, model, dataViews); + return { type: 'lnsXY', layers: Object.values(excludeMetaFromLayers(extendedLayers)), - configuration: getConfiguration(model, getLayers(extendedLayers, model)), + configuration: getConfiguration(model, configLayers), }; }; diff --git a/src/plugins/visualizations/common/convert_to_lens/index.ts b/src/plugins/visualizations/common/convert_to_lens/index.ts index f47605f92719d..d336872cff6bb 100644 --- a/src/plugins/visualizations/common/convert_to_lens/index.ts +++ b/src/plugins/visualizations/common/convert_to_lens/index.ts @@ -8,3 +8,4 @@ export * from './types'; export * from './constants'; +export * from './utils'; diff --git a/src/plugins/visualizations/common/convert_to_lens/types/configurations.ts b/src/plugins/visualizations/common/convert_to_lens/types/configurations.ts index b908ee2c5346b..4702ebae80826 100644 --- a/src/plugins/visualizations/common/convert_to_lens/types/configurations.ts +++ b/src/plugins/visualizations/common/convert_to_lens/types/configurations.ts @@ -9,6 +9,7 @@ import { HorizontalAlignment, Position, VerticalAlignment } from '@elastic/charts'; import { $Values } from '@kbn/utility-types'; import type { PaletteOutput } from '@kbn/coloring'; +import { KibanaQueryOutput } from '@kbn/data-plugin/common'; import { LegendSize } from '../../constants'; export const XYCurveTypes = { @@ -90,7 +91,33 @@ export interface XYReferenceLineLayerConfig { layerType: 'referenceLine'; } -export type XYLayerConfig = XYDataLayerConfig | XYReferenceLineLayerConfig; +export interface EventAnnotationConfig { + id: string; + filter: KibanaQueryOutput; + timeField?: string; + extraFields?: string[]; + label: string; + color?: string; + isHidden?: boolean; + icon?: string; + type: 'query'; + key: { + type: 'point_in_time'; + }; +} + +export interface XYAnnotationsLayerConfig { + layerId: string; + annotations: EventAnnotationConfig[]; + ignoreGlobalFilters: boolean; + layerType: 'annotations'; + indexPatternId: string; +} + +export type XYLayerConfig = + | XYDataLayerConfig + | XYReferenceLineLayerConfig + | XYAnnotationsLayerConfig; export interface AxesSettingsConfig { x: boolean; diff --git a/src/plugins/visualizations/common/convert_to_lens/utils.ts b/src/plugins/visualizations/common/convert_to_lens/utils.ts new file mode 100644 index 0000000000000..1beba70bc2a1b --- /dev/null +++ b/src/plugins/visualizations/common/convert_to_lens/utils.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { XYAnnotationsLayerConfig, XYLayerConfig } from './types'; + +export const isAnnotationsLayer = ( + layer: Pick +): layer is XYAnnotationsLayerConfig => layer.layerType === 'annotations'; diff --git a/x-pack/plugins/lens/public/app_plugin/mounter.tsx b/x-pack/plugins/lens/public/app_plugin/mounter.tsx index 62568a3cbca5e..6aa7bef84d3e2 100644 --- a/x-pack/plugins/lens/public/app_plugin/mounter.tsx +++ b/x-pack/plugins/lens/public/app_plugin/mounter.tsx @@ -203,7 +203,7 @@ export async function mountApp( }); } }; - // get state from location, used for nanigating from Visualize/Discover to Lens + // get state from location, used for navigating from Visualize/Discover to Lens const initialContext = historyLocationState && (historyLocationState.type === ACTION_VISUALIZE_LENS_FIELD || diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts index 26c60127b693b..c886c21cfa6b8 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts @@ -13,6 +13,7 @@ import { difference } from 'lodash'; import type { DataViewsContract, DataViewSpec } from '@kbn/data-views-plugin/public'; import { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; import { DataViewPersistableStateService } from '@kbn/data-views-plugin/common'; +import { isAnnotationsLayer } from '@kbn/visualizations-plugin/common/convert_to_lens'; import { Datasource, DatasourceLayers, @@ -53,6 +54,9 @@ function getIndexPatterns( for (const { indexPatternId } of initialContext.layers) { indexPatternIds.push(indexPatternId); } + for (const l of initialContext.configuration.layers) { + if (isAnnotationsLayer(l)) indexPatternIds.push(l.indexPatternId); + } } else { indexPatternIds.push(initialContext.dataViewSpec.id!); } @@ -209,6 +213,7 @@ export async function initializeSources( visualizationMap, visualizationState, references, + initialContext, }), }; } @@ -217,16 +222,19 @@ export function initializeVisualization({ visualizationMap, visualizationState, references, + initialContext, }: { visualizationState: VisualizationState; visualizationMap: VisualizationMap; references?: SavedObjectReference[]; + initialContext?: VisualizeFieldContext | VisualizeEditorContext; }) { if (visualizationState?.activeId) { return ( visualizationMap[visualizationState.activeId]?.fromPersistableState?.( visualizationState.state, - references + references, + initialContext ) ?? visualizationState.state ); } diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index b49149b92317b..e581064bcdbf2 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -917,7 +917,11 @@ export interface Visualization { /** Visualizations can have references as well */ getPersistableState?: (state: T) => { state: P; savedObjectReferences: SavedObjectReference[] }; /** Hydrate from persistable state and references to final state */ - fromPersistableState?: (state: P, references?: SavedObjectReference[]) => T; + fromPersistableState?: ( + state: P, + references?: SavedObjectReference[], + initialContext?: VisualizeFieldContext | VisualizeEditorContext + ) => T; /** Frame needs to know which layers the visualization is currently using */ getLayerIds: (state: T) => string[]; /** Reset button on each layer triggers this */ diff --git a/x-pack/plugins/lens/public/visualizations/xy/state_helpers.ts b/x-pack/plugins/lens/public/visualizations/xy/state_helpers.ts index d683888a88cfd..a84168b009f9e 100644 --- a/x-pack/plugins/lens/public/visualizations/xy/state_helpers.ts +++ b/x-pack/plugins/lens/public/visualizations/xy/state_helpers.ts @@ -10,11 +10,13 @@ import { EuiIconType } from '@elastic/eui/src/components/icon/icon'; import type { SavedObjectReference } from '@kbn/core/public'; import { isQueryAnnotationConfig } from '@kbn/event-annotation-plugin/public'; import { i18n } from '@kbn/i18n'; +import { VisualizeFieldContext } from '@kbn/ui-actions-plugin/public'; import { validateQuery } from '../../shared_components'; import type { FramePublicAPI, DatasourcePublicAPI, VisualizationDimensionGroupConfig, + VisualizeEditorContext, } from '../../types'; import { visualizationTypes, @@ -26,6 +28,7 @@ import { XYState, XYPersistedState, State, + XYAnnotationLayerConfig, } from './types'; import { getDataLayers, isAnnotationsLayer, isDataLayer } from './visualization_helpers'; @@ -140,11 +143,13 @@ export function extractReferences(state: XYState) { export function injectReferences( state: XYPersistedState, - references?: SavedObjectReference[] + references?: SavedObjectReference[], + initialContext?: VisualizeFieldContext | VisualizeEditorContext ): XYState { if (!references || !references.length) { return state as XYState; } + const fallbackIndexPatternId = references.find(({ type }) => type === 'index-pattern')!.id; return { ...state, @@ -155,6 +160,7 @@ export function injectReferences( return { ...layer, indexPatternId: + getIndexPatternIdFromInitialContext(layer, initialContext) || references.find(({ name }) => name === getLayerReferenceName(layer.layerId))?.id || fallbackIndexPatternId, }; @@ -162,6 +168,15 @@ export function injectReferences( }; } +function getIndexPatternIdFromInitialContext( + layer: XYAnnotationLayerConfig, + initialContext?: VisualizeFieldContext | VisualizeEditorContext +) { + if (initialContext && 'isVisualizeAction' in initialContext) { + return layer && 'indexPatternId' in layer ? layer.indexPatternId : undefined; + } +} + export function validateColumn( state: State, frame: Pick, diff --git a/x-pack/plugins/lens/public/visualizations/xy/visualization.tsx b/x-pack/plugins/lens/public/visualizations/xy/visualization.tsx index 64ffbab4c38e7..61bdd4151219c 100644 --- a/x-pack/plugins/lens/public/visualizations/xy/visualization.tsx +++ b/x-pack/plugins/lens/public/visualizations/xy/visualization.tsx @@ -184,8 +184,8 @@ export const getXyVisualization = ({ return extractReferences(state); }, - fromPersistableState(state, references) { - return injectReferences(state, references); + fromPersistableState(state, references, initialContext) { + return injectReferences(state, references, initialContext); }, getDescription, diff --git a/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/icon_set.ts b/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/icon_set.ts index 23a4b6fb52610..45019dd204491 100644 --- a/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/icon_set.ts +++ b/x-pack/plugins/lens/public/visualizations/xy/xy_config_panel/annotations_config_panel/icon_set.ts @@ -82,6 +82,12 @@ export const annotationsIconSet: IconSet = [ value: 'starEmpty', label: i18n.translate('xpack.lens.xyChart.iconSelect.starLabel', { defaultMessage: 'Star' }), }, + { + value: 'starFilled', + label: i18n.translate('xpack.lens.xyChart.iconSelect.starFilledLabel', { + defaultMessage: 'Star filled', + }), + }, { value: 'tag', label: i18n.translate('xpack.lens.xyChart.iconSelect.tagIconLabel', { From e3894f122412475bebf13f523dab2c398095e3ae Mon Sep 17 00:00:00 2001 From: Jordan <51442161+JordanSh@users.noreply.github.com> Date: Tue, 20 Sep 2022 20:03:52 +0300 Subject: [PATCH 31/76] [Cloud Posture] Limiting back to integration button width #141100 --- .../resource_findings_table.tsx | 1 + .../public/pages/rules/index.tsx | 61 ++++++++++--------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_table.tsx b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_table.tsx index 07c84000fb607..976ddd8d8b37f 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_table.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/findings/latest_findings_by_resource/resource_findings/resource_findings_table.tsx @@ -58,6 +58,7 @@ const ResourceFindingsTableComponent = ({ ], [onAddFilter] ); + if (!loading && !items.length) return ( ) + + + + + + + + + + + + } + description={ + sharedValues.length && ( +
+ +
+ ) + } rightSideItems={[ ) /> , ]} - pageTitle={ - - - - - - - - - } - description={ - sharedValues.length && ( -
- -
- ) - } - bottomBorder /> From ddcd62dd140a731b88425884df4863f05afb1239 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Tue, 20 Sep 2022 10:04:09 -0700 Subject: [PATCH 32/76] skip flaky jest suite (#131346) Signed-off-by: Tyler Smalley --- .../synthetics/public/legacy_uptime/pages/overview.test.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/synthetics/public/legacy_uptime/pages/overview.test.tsx b/x-pack/plugins/synthetics/public/legacy_uptime/pages/overview.test.tsx index b3aa4714fa664..30ea0e361580a 100644 --- a/x-pack/plugins/synthetics/public/legacy_uptime/pages/overview.test.tsx +++ b/x-pack/plugins/synthetics/public/legacy_uptime/pages/overview.test.tsx @@ -9,7 +9,8 @@ import React from 'react'; import { OverviewPageComponent } from './overview'; import { render } from '../lib/helper/rtl_helpers'; -describe('MonitorPage', () => { +// FLAKY: https://github.com/elastic/kibana/issues/131346 +describe.skip('MonitorPage', () => { it('renders expected elements for valid props', async () => { const { findByText, findByPlaceholderText } = render(); From f4dd5156412f880aa807ddcb9454ef3bd624bbcc Mon Sep 17 00:00:00 2001 From: Cristina Amico Date: Tue, 20 Sep 2022 19:08:24 +0200 Subject: [PATCH 33/76] [Fleet] Fix broken UI when clicking on upgrade packages button (#141112) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../fleet/sections/agent_policy/services/devtools_request.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/services/devtools_request.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/services/devtools_request.tsx index 875d43d1f0ea4..51831fdfa6b7c 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/services/devtools_request.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/services/devtools_request.tsx @@ -105,7 +105,7 @@ function formatVars(vars: NewPackagePolicy['inputs'][number]['vars']) { } return Object.entries(vars).reduce((acc, [varKey, varRecord]) => { - acc[varKey] = varRecord.value; + acc[varKey] = varRecord?.value; return acc; }, {} as SimplifiedVars); From 130a8cde541f617c4b2226b1fa1a0d6754fc24cd Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Tue, 20 Sep 2022 11:37:31 -0600 Subject: [PATCH 34/76] [ML] Explain Log Rate Spikes: merge duplicate spikeAnalysisTable components (#141116) * merge duplicate spikeAnalysisTable components * [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../spike_analysis_table.tsx | 1 + .../spike_analysis_table_expanded_row.tsx | 355 ------------------ .../spike_analysis_table_groups.tsx | 20 +- 3 files changed, 18 insertions(+), 358 deletions(-) delete mode 100644 x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_expanded_row.tsx diff --git a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table.tsx b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table.tsx index 0ed43c32cdb60..5a541c9e56af2 100644 --- a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table.tsx +++ b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table.tsx @@ -237,6 +237,7 @@ export const SpikeAnalysisTable: FC = ({ ), render: (_, { pValue }) => { + if (!pValue) return NOT_AVAILABLE; const label = getFailedTransactionsCorrelationImpactLabel(pValue); return label ? {label.impact} : null; }, diff --git a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_expanded_row.tsx b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_expanded_row.tsx deleted file mode 100644 index 9eb235c07a1a1..0000000000000 --- a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_expanded_row.tsx +++ /dev/null @@ -1,355 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React, { FC, useCallback, useMemo, useState } from 'react'; -import { sortBy } from 'lodash'; - -import { - EuiBadge, - EuiBasicTable, - EuiBasicTableColumn, - EuiIcon, - EuiTableSortingType, - EuiToolTip, -} from '@elastic/eui'; - -import { i18n } from '@kbn/i18n'; -import { FormattedMessage } from '@kbn/i18n-react'; -import { escapeKuery } from '@kbn/es-query'; -import type { ChangePoint } from '@kbn/ml-agg-utils'; - -import { SEARCH_QUERY_LANGUAGE } from '../../application/utils/search_utils'; -import { useEuiTheme } from '../../hooks/use_eui_theme'; -import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; - -import { MiniHistogram } from '../mini_histogram'; - -import { getFailedTransactionsCorrelationImpactLabel } from './get_failed_transactions_correlation_impact_label'; - -const NARROW_COLUMN_WIDTH = '120px'; -const ACTIONS_COLUMN_WIDTH = '60px'; -const NOT_AVAILABLE = '--'; - -const PAGINATION_SIZE_OPTIONS = [5, 10, 20, 50]; -const DEFAULT_SORT_FIELD = 'pValue'; -const DEFAULT_SORT_DIRECTION = 'asc'; -const viewInDiscoverMessage = i18n.translate( - 'xpack.aiops.spikeAnalysisTable.linksMenu.viewInDiscover', - { - defaultMessage: 'View in Discover', - } -); - -interface SpikeAnalysisTableExpandedRowProps { - changePoints: ChangePoint[]; - dataViewId?: string; - loading: boolean; - onPinnedChangePoint?: (changePoint: ChangePoint | null) => void; - onSelectedChangePoint?: (changePoint: ChangePoint | null) => void; - selectedChangePoint?: ChangePoint; -} - -export const SpikeAnalysisTableExpandedRow: FC = ({ - changePoints, - dataViewId, - loading, - onPinnedChangePoint, - onSelectedChangePoint, - selectedChangePoint, -}) => { - const euiTheme = useEuiTheme(); - - const [pageIndex, setPageIndex] = useState(0); - const [pageSize, setPageSize] = useState(10); - const [sortField, setSortField] = useState(DEFAULT_SORT_FIELD); - const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>(DEFAULT_SORT_DIRECTION); - - const { application, share, data } = useAiopsAppContext(); - - const discoverLocator = useMemo( - () => share.url.locators.get('DISCOVER_APP_LOCATOR'), - [share.url.locators] - ); - - const discoverUrlError = useMemo(() => { - if (!application.capabilities.discover?.show) { - const discoverNotEnabled = i18n.translate( - 'xpack.aiops.spikeAnalysisTable.discoverNotEnabledErrorMessage', - { - defaultMessage: 'Discover is not enabled', - } - ); - - return discoverNotEnabled; - } - if (!discoverLocator) { - const discoverLocatorMissing = i18n.translate( - 'xpack.aiops.spikeAnalysisTable.discoverLocatorMissingErrorMessage', - { - defaultMessage: 'No locator for Discover detected', - } - ); - - return discoverLocatorMissing; - } - if (!dataViewId) { - const autoGeneratedDiscoverLinkError = i18n.translate( - 'xpack.aiops.spikeAnalysisTable.autoGeneratedDiscoverLinkErrorMessage', - { - defaultMessage: 'Unable to link to Discover; no data view exists for this index', - } - ); - - return autoGeneratedDiscoverLinkError; - } - }, [application.capabilities.discover?.show, dataViewId, discoverLocator]); - - const generateDiscoverUrl = async (changePoint: ChangePoint) => { - if (discoverLocator !== undefined) { - const url = await discoverLocator.getRedirectUrl({ - indexPatternId: dataViewId, - timeRange: data.query.timefilter.timefilter.getTime(), - filters: data.query.filterManager.getFilters(), - query: { - language: SEARCH_QUERY_LANGUAGE.KUERY, - query: `${escapeKuery(changePoint.fieldName)}:${escapeKuery( - String(changePoint.fieldValue) - )}`, - }, - }); - - return url; - } - }; - - const columns: Array> = [ - { - 'data-test-subj': 'aiopsSpikeAnalysisTableColumnFieldName', - field: 'fieldName', - name: i18n.translate('xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.fieldNameLabel', { - defaultMessage: 'Field name', - }), - sortable: true, - }, - { - 'data-test-subj': 'aiopsSpikeAnalysisTableColumnFieldValue', - field: 'fieldValue', - name: i18n.translate('xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.fieldValueLabel', { - defaultMessage: 'Field value', - }), - render: (_, { fieldValue }) => String(fieldValue).slice(0, 50), - sortable: true, - }, - { - 'data-test-subj': 'aiopsSpikeAnalysisTableColumnLogRate', - width: NARROW_COLUMN_WIDTH, - field: 'pValue', - name: ( - - <> - - - - - ), - render: (_, { histogram, fieldName, fieldValue }) => { - if (!histogram) return NOT_AVAILABLE; - return ( - - ); - }, - sortable: false, - }, - { - 'data-test-subj': 'aiopsSpikeAnalysisTableColumnPValue', - width: NARROW_COLUMN_WIDTH, - field: 'pValue', - name: ( - - <> - - - - - ), - render: (pValue: number | null) => pValue?.toPrecision(3) ?? NOT_AVAILABLE, - sortable: true, - }, - { - 'data-test-subj': 'aiopsSpikeAnalysisTableColumnImpact', - width: NARROW_COLUMN_WIDTH, - field: 'pValue', - name: ( - - <> - - - - - ), - render: (_, { pValue }) => { - if (!pValue) return NOT_AVAILABLE; - const label = getFailedTransactionsCorrelationImpactLabel(pValue); - return label ? {label.impact} : null; - }, - sortable: true, - }, - { - 'data-test-subj': 'aiOpsSpikeAnalysisTableColumnAction', - name: i18n.translate('xpack.aiops.spikeAnalysisTable.actionsColumnName', { - defaultMessage: 'Actions', - }), - actions: [ - { - name: () => ( - - - - ), - description: viewInDiscoverMessage, - type: 'button', - onClick: async (changePoint) => { - const openInDiscoverUrl = await generateDiscoverUrl(changePoint); - if (typeof openInDiscoverUrl === 'string') { - await application.navigateToUrl(openInDiscoverUrl); - } - }, - enabled: () => discoverUrlError === undefined, - }, - ], - width: ACTIONS_COLUMN_WIDTH, - }, - ]; - - const onChange = useCallback((tableSettings) => { - const { index, size } = tableSettings.page; - const { field, direction } = tableSettings.sort; - - setPageIndex(index); - setPageSize(size); - setSortField(field); - setSortDirection(direction); - }, []); - - const { pagination, pageOfItems, sorting } = useMemo(() => { - const pageStart = pageIndex * pageSize; - const itemCount = changePoints?.length ?? 0; - - let items: ChangePoint[] = changePoints ?? []; - items = sortBy(changePoints, (item) => { - if (item && typeof item[sortField] === 'string') { - // @ts-ignore Object is possibly null or undefined - return item[sortField].toLowerCase(); - } - return item[sortField]; - }); - items = sortDirection === 'asc' ? items : items.reverse(); - - return { - pageOfItems: items.slice(pageStart, pageStart + pageSize), - pagination: { - pageIndex, - pageSize, - totalItemCount: itemCount, - pageSizeOptions: PAGINATION_SIZE_OPTIONS, - }, - sorting: { - sort: { - field: sortField, - direction: sortDirection, - }, - }, - }; - }, [pageIndex, pageSize, sortField, sortDirection, changePoints]); - - // Don't pass on the `loading` state to the table itself because - // it disables hovering events. Because the mini histograms take a while - // to load, hovering would not update the main chart. Instead, - // the loading state is shown by the progress bar on the outer component level. - // The outer component also will display a prompt when no data was returned - // running the analysis and will hide this table. - - return ( - } - rowProps={(changePoint) => { - return { - 'data-test-subj': `aiopsSpikeAnalysisTableRow row-${changePoint.fieldName}-${changePoint.fieldValue}`, - onClick: () => { - if (onPinnedChangePoint) { - onPinnedChangePoint(changePoint); - } - }, - onMouseEnter: () => { - if (onSelectedChangePoint) { - onSelectedChangePoint(changePoint); - } - }, - onMouseLeave: () => { - if (onSelectedChangePoint) { - onSelectedChangePoint(null); - } - }, - style: - selectedChangePoint && - selectedChangePoint.fieldValue === changePoint.fieldValue && - selectedChangePoint.fieldName === changePoint.fieldName - ? { - backgroundColor: euiTheme.euiColorLightestShade, - } - : null, - }; - }} - /> - ); -}; diff --git a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx index 7a6f5508aea82..a046250db20b2 100644 --- a/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx +++ b/x-pack/plugins/aiops/public/components/spike_analysis_table/spike_analysis_table_groups.tsx @@ -25,7 +25,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import type { ChangePoint } from '@kbn/ml-agg-utils'; import { useEuiTheme } from '../../hooks/use_eui_theme'; -import { SpikeAnalysisTableExpandedRow } from './spike_analysis_table_expanded_row'; +import { SpikeAnalysisTable } from './spike_analysis_table'; const NARROW_COLUMN_WIDTH = '120px'; const EXPAND_COLUMN_WIDTH = '40px'; @@ -100,7 +100,7 @@ export const SpikeAnalysisGroupsTable: FC = ({ } itemIdToExpandedRowMapValues[item.id] = ( - = ({ toggleDetails(item)} - aria-label={itemIdToExpandedRowMap[item.id] ? 'Collapse' : 'Expand'} + aria-label={ + itemIdToExpandedRowMap[item.id] + ? i18n.translate( + 'xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.collapseAriaLabel', + { + defaultMessage: 'Collapse', + } + ) + : i18n.translate( + 'xpack.aiops.explainLogRateSpikes.spikeAnalysisTable.expandAriaLabel', + { + defaultMessage: 'Expand', + } + ) + } iconType={itemIdToExpandedRowMap[item.id] ? 'arrowUp' : 'arrowDown'} /> ), From a9cf105486fba79437cbf2a6b8787c87675a32f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20S=C3=A1nchez?= Date: Tue, 20 Sep 2022 19:41:43 +0200 Subject: [PATCH 35/76] [Security Solution][Endpoint] Follow up pr of `140942` with unit tests (#141060) --- .../components/actions_log_empty_state.tsx | 74 ++++++++++--------- .../response_actions_log.test.tsx | 25 ++++++- .../response_actions_log.tsx | 46 ++++++------ .../management_empty_state_wrapper.tsx | 22 ++++-- .../view/response_actions_list_page.test.tsx | 29 +++++++- .../view/response_actions_list_page.tsx | 4 +- .../endpoint/services/actions/action_list.ts | 3 +- 7 files changed, 135 insertions(+), 68 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/components/actions_log_empty_state.tsx b/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/components/actions_log_empty_state.tsx index 3cc75f33b0f7d..a5a6f15f2de9d 100644 --- a/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/components/actions_log_empty_state.tsx +++ b/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/components/actions_log_empty_state.tsx @@ -6,49 +6,53 @@ */ import React, { memo } from 'react'; -import styled, { css } from 'styled-components'; +import styled from 'styled-components'; import { EuiLink, EuiEmptyPrompt } from '@elastic/eui'; import { useKibana } from '@kbn/kibana-react-plugin/public'; import { i18n } from '@kbn/i18n'; import { ManagementEmptyStateWrapper } from '../../management_empty_state_wrapper'; const EmptyPrompt = styled(EuiEmptyPrompt)` - ${() => css` - max-width: 100%; - `} + max-width: 100%; `; -export const ActionsLogEmptyState = memo(() => { - const { docLinks } = useKibana().services; +export const ActionsLogEmptyState = memo( + ({ 'data-test-subj': dataTestSubj }: { 'data-test-subj'?: string }) => { + const { docLinks } = useKibana().services; - return ( - - - {i18n.translate('xpack.securitySolution.actions_log.empty.title', { - defaultMessage: 'Actions history is empty', - })} - - } - body={ -
- {i18n.translate('xpack.securitySolution.actions_log.empty.content', { - defaultMessage: 'No response actions performed', - })} -
- } - actions={[ - - {i18n.translate('xpack.securitySolution.actions_log.empty.link', { - defaultMessage: 'Read more about response actions', - })} - , - ]} - /> - - ); -}); + return ( + + + {i18n.translate('xpack.securitySolution.actions_log.empty.title', { + defaultMessage: 'Actions history is empty', + })} + + } + body={ +
+ {i18n.translate('xpack.securitySolution.actions_log.empty.content', { + defaultMessage: 'No response actions performed', + })} +
+ } + actions={[ + + {i18n.translate('xpack.securitySolution.actions_log.empty.link', { + defaultMessage: 'Read more about response actions', + })} + , + ]} + /> + + ); + } +); ActionsLogEmptyState.displayName = 'ActionsLogEmptyState'; diff --git a/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/response_actions_log.test.tsx b/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/response_actions_log.test.tsx index e76fb65e582ce..549b03fedfbf1 100644 --- a/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/response_actions_log.test.tsx +++ b/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/response_actions_log.test.tsx @@ -9,6 +9,7 @@ import React from 'react'; import * as reactTestingLibrary from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { waitForEuiPopoverOpen } from '@elastic/eui/lib/test/rtl'; +import type { IHttpFetchError } from '@kbn/core-http-browser'; import { createAppRootMockRenderer, type AppContextTestRender, @@ -23,7 +24,7 @@ import uuid from 'uuid'; let mockUseGetEndpointActionList: { isFetched?: boolean; isFetching?: boolean; - error?: null; + error?: Partial | null; data?: ActionListApiResponse; refetch: () => unknown; }; @@ -166,6 +167,28 @@ describe('Response Actions Log', () => { jest.clearAllMocks(); }); + describe('When index does not exist yet', () => { + it('should show global loader when waiting for response', () => { + mockUseGetEndpointActionList = { + ...baseMockedActionList, + isFetched: false, + isFetching: true, + }; + render(); + expect(renderResult.getByTestId(`${testPrefix}-global-loader`)).toBeTruthy(); + }); + it('should show empty page when there is no index', () => { + mockUseGetEndpointActionList = { + ...baseMockedActionList, + error: { + body: { statusCode: 404, message: 'index_not_found_exception' }, + }, + }; + render(); + expect(renderResult.getByTestId(`${testPrefix}-empty-state`)).toBeTruthy(); + }); + }); + describe('Without data', () => { it('should show date filters', () => { render(); diff --git a/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/response_actions_log.tsx b/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/response_actions_log.tsx index 5e674f83f4af2..e985aae704ebf 100644 --- a/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/response_actions_log.tsx +++ b/x-pack/plugins/security_solution/public/management/components/endpoint_response_actions_list/response_actions_log.tsx @@ -116,9 +116,9 @@ export const ResponseActionsLog = memo< Pick & { showHostNames?: boolean; isFlyout?: boolean; - setIsDataCallback?: (isData: boolean) => void; + setIsDataInResponse?: (isData: boolean) => void; } ->(({ agentIds, showHostNames = false, isFlyout = true, setIsDataCallback }) => { +>(({ agentIds, showHostNames = false, isFlyout = true, setIsDataInResponse }) => { const { pagination: paginationFromUrlParams, setPagination: setPaginationOnUrlParams } = useUrlPagination(); const { @@ -134,6 +134,7 @@ export const ResponseActionsLog = memo< [k: ActionListApiResponse['data'][number]['id']]: React.ReactNode; }>({}); + // Used to decide if display global loader or not (only the fist time tha page loads) const [isFirstAttempt, setIsFirstAttempt] = useState(true); const [queryParams, setQueryParams] = useState({ @@ -180,6 +181,26 @@ export const ResponseActionsLog = memo< { retry: false } ); + // Hide page header when there is no actions index calling the setIsDataInResponse with false value. + // Otherwise, it shows the page header calling the setIsDataInResponse with true value and it also keeps track + // if the API request was done for the first time. + useEffect(() => { + if ( + !isFetching && + error?.body?.statusCode === 404 && + error?.body?.message === 'index_not_found_exception' + ) { + if (setIsDataInResponse) { + setIsDataInResponse(false); + } + } else if (!isFetching && actionList) { + setIsFirstAttempt(false); + if (setIsDataInResponse) { + setIsDataInResponse(true); + } + } + }, [actionList, error, isFetching, setIsDataInResponse]); + // handle auto refresh data const onRefresh = useCallback(() => { if (dateRangePickerState.autoRefreshOptions.enabled) { @@ -586,27 +607,10 @@ export const ResponseActionsLog = memo< [getTestId, pagedResultsCount.fromCount, pagedResultsCount.toCount, totalItemCount] ); - useEffect(() => { - if ( - !isFetching && - error?.body?.statusCode === 404 && - error?.body?.message === 'index_not_found_exception' - ) { - if (setIsDataCallback) { - setIsDataCallback(false); - } - } else if (!isFetching && actionList) { - setIsFirstAttempt(false); - if (setIsDataCallback) { - setIsDataCallback(true); - } - } - }, [actionList, error, isFetching, setIsDataCallback]); - if (error?.body?.statusCode === 404 && error?.body?.message === 'index_not_found_exception') { - return ; + return ; } else if (isFetching && isFirstAttempt) { - return ; + return ; } return ( <> diff --git a/x-pack/plugins/security_solution/public/management/components/management_empty_state_wrapper.tsx b/x-pack/plugins/security_solution/public/management/components/management_empty_state_wrapper.tsx index 19b0cf44736f0..db25aea95f917 100644 --- a/x-pack/plugins/security_solution/public/management/components/management_empty_state_wrapper.tsx +++ b/x-pack/plugins/security_solution/public/management/components/management_empty_state_wrapper.tsx @@ -13,12 +13,20 @@ export const StyledEuiFlexGroup = styled(EuiFlexGroup)` min-height: calc(100vh - 140px); `; -export const ManagementEmptyStateWrapper = memo(({ children }) => { - return ( - - {children} - - ); -}); +export const ManagementEmptyStateWrapper = memo( + ({ + children, + 'data-test-subj': dataTestSubj, + }: { + children: React.ReactNode; + 'data-test-subj'?: string; + }) => { + return ( + + {children} + + ); + } +); ManagementEmptyStateWrapper.displayName = 'ManagementEmptyStateWrapper'; diff --git a/x-pack/plugins/security_solution/public/management/pages/response_actions/view/response_actions_list_page.test.tsx b/x-pack/plugins/security_solution/public/management/pages/response_actions/view/response_actions_list_page.test.tsx index ebb365cd44c54..c7f6315c9d9d4 100644 --- a/x-pack/plugins/security_solution/public/management/pages/response_actions/view/response_actions_list_page.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/response_actions/view/response_actions_list_page.test.tsx @@ -9,6 +9,7 @@ import React from 'react'; import * as reactTestingLibrary from '@testing-library/react'; import { waitForEuiPopoverOpen } from '@elastic/eui/lib/test/rtl'; import userEvent from '@testing-library/user-event'; +import type { IHttpFetchError } from '@kbn/core-http-browser'; import { type AppContextTestRender, createAppRootMockRenderer, @@ -22,7 +23,7 @@ import { useGetEndpointsList } from '../../../hooks/endpoint/use_get_endpoints_l let mockUseGetEndpointActionList: { isFetched?: boolean; isFetching?: boolean; - error?: null; + error?: Partial | null; data?: ActionListApiResponse; refetch: () => unknown; }; @@ -160,6 +161,32 @@ describe('Action history page', () => { jest.clearAllMocks(); }); + describe('Hide/Show header', () => { + it('should show header when data is in', () => { + reactTestingLibrary.act(() => { + history.push('/administration/action_history?page=3&pageSize=20'); + }); + render(); + const { getByTestId } = renderResult; + expect(getByTestId('responseActionsPage-header')).toBeTruthy(); + }); + + it('should not show header when there is no actions index', () => { + reactTestingLibrary.act(() => { + history.push('/administration/action_history?page=3&pageSize=20'); + }); + mockUseGetEndpointActionList = { + ...baseMockedActionList, + error: { + body: { statusCode: 404, message: 'index_not_found_exception' }, + }, + }; + render(); + const { queryByTestId } = renderResult; + expect(queryByTestId('responseActionsPage-header')).toBeNull(); + }); + }); + describe('Read from URL params', () => { it('should read and set paging values from URL params', () => { reactTestingLibrary.act(() => { diff --git a/x-pack/plugins/security_solution/public/management/pages/response_actions/view/response_actions_list_page.tsx b/x-pack/plugins/security_solution/public/management/pages/response_actions/view/response_actions_list_page.tsx index 595f5cee928c2..e1f3705b4489c 100644 --- a/x-pack/plugins/security_solution/public/management/pages/response_actions/view/response_actions_list_page.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/response_actions/view/response_actions_list_page.tsx @@ -13,7 +13,7 @@ import { UX_MESSAGES } from '../../../components/endpoint_response_actions_list/ export const ResponseActionsListPage = () => { const [hideHeader, setHideHeader] = useState(true); - const setIsDataCallback = useCallback((isData: boolean) => { + const resetPageHeader = useCallback((isData: boolean) => { setHideHeader(!isData); }, []); return ( @@ -26,7 +26,7 @@ export const ResponseActionsListPage = () => { ); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/actions/action_list.ts b/x-pack/plugins/security_solution/server/endpoint/services/actions/action_list.ts index aa5e45d3d66e6..eb53a6a4338a8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/actions/action_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/actions/action_list.ts @@ -205,9 +205,10 @@ const getActionDetailsList = async ({ throw err; } - if (!actionRequests?.body?.hits?.hits) + if (!actionRequests?.body?.hits?.hits) { // return empty details array return { actionDetails: [], totalRecords: 0 }; + } // format endpoint actions into { type, item } structure const formattedActionRequests = formatEndpointActionResults(actionRequests?.body?.hits?.hits); From 53e1c8d2b5e0d4857eabd354bf1ec2b478e5bdb9 Mon Sep 17 00:00:00 2001 From: Jordan <51442161+JordanSh@users.noreply.github.com> Date: Tue, 20 Sep 2022 21:04:02 +0300 Subject: [PATCH 36/76] [Cloud Posture] Agent link now has underline (#141118) --- .../pages/benchmarks/benchmarks_table.tsx | 69 +++++++++---------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks_table.tsx b/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks_table.tsx index 7797a9a55ef8c..35dce429d54aa 100644 --- a/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks_table.tsx +++ b/x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks_table.tsx @@ -11,9 +11,10 @@ import { type EuiBasicTableProps, type Pagination, type CriteriaWithPagination, + EuiLink, } from '@elastic/eui'; import React from 'react'; -import { Link, useHistory, generatePath } from 'react-router-dom'; +import { generatePath } from 'react-router-dom'; import { pagePathGetters } from '@kbn/fleet-plugin/public'; import { i18n } from '@kbn/i18n'; import { TimestampTableCell } from '../../components/timestamp_table_cell'; @@ -31,20 +32,34 @@ interface BenchmarksTableProps } const AgentPolicyButtonLink = ({ name, id: policyId }: { name: string; id: string }) => { - const { http, application } = useKibana().services; + const { http } = useKibana().services; const [fleetBase, path] = pagePathGetters.policy_details({ policyId }); + + return {name}; +}; + +const IntegrationButtonLink = ({ + packageName, + policyId, + packagePolicyId, +}: { + packageName: string; + packagePolicyId: string; + policyId: string; +}) => { + const { application } = useKibana().services; + return ( - { - e.stopPropagation(); - e.preventDefault(); - application.navigateToApp('fleet', { path }); - }} + - {name} - + {packageName} + ); }; @@ -55,18 +70,11 @@ const BENCHMARKS_TABLE_COLUMNS: Array> = [ defaultMessage: 'Integration', }), render: (packageName, benchmark) => ( - { - e.stopPropagation(); - }} - > - {packageName} - + ), truncateText: true, sortable: true, @@ -146,18 +154,6 @@ export const BenchmarksTable = ({ sorting, ...rest }: BenchmarksTableProps) => { - const history = useHistory(); - - const getRowProps: EuiBasicTableProps['rowProps'] = (benchmark) => ({ - onClick: () => - history.push( - generatePath(cloudPosturePages.rules.path, { - packagePolicyId: benchmark.package_policy.id, - policyId: benchmark.package_policy.policy_id, - }) - ), - }); - const pagination: Pagination = { pageIndex: Math.max(pageIndex - 1, 0), pageSize, @@ -173,7 +169,6 @@ export const BenchmarksTable = ({ data-test-subj={rest['data-test-subj']} items={benchmarks} columns={BENCHMARKS_TABLE_COLUMNS} - rowProps={getRowProps} itemId={(item) => [item.agent_policy.id, item.package_policy.id].join('/')} pagination={pagination} onChange={onChange} From 2acc1c8377190432ca184a488092dabc51c98d88 Mon Sep 17 00:00:00 2001 From: Kevin Lacabane Date: Tue, 20 Sep 2022 20:11:10 +0200 Subject: [PATCH 37/76] add padding to inner page (#140977) --- .../monitoring/public/application/pages/page_template.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/monitoring/public/application/pages/page_template.tsx b/x-pack/plugins/monitoring/public/application/pages/page_template.tsx index c67ea0df8a180..8912b38f57a20 100644 --- a/x-pack/plugins/monitoring/public/application/pages/page_template.tsx +++ b/x-pack/plugins/monitoring/public/application/pages/page_template.tsx @@ -148,7 +148,11 @@ export const PageTemplate: React.FC = ({ })} )} - {renderContent()} + + + {renderContent()} + + From 6927902014bb423581a9a9539b97adc9c24be959 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 20 Sep 2022 12:15:58 -0600 Subject: [PATCH 38/76] skip failing test suite (#140241) --- .../test/functional/apps/maps/group2/embeddable/dashboard.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/maps/group2/embeddable/dashboard.js b/x-pack/test/functional/apps/maps/group2/embeddable/dashboard.js index 00c80db1d4895..a80095f142e46 100644 --- a/x-pack/test/functional/apps/maps/group2/embeddable/dashboard.js +++ b/x-pack/test/functional/apps/maps/group2/embeddable/dashboard.js @@ -19,7 +19,8 @@ export default function ({ getPageObjects, getService }) { const security = getService('security'); const testSubjects = getService('testSubjects'); - describe('embed in dashboard', () => { + // Failing: See https://github.com/elastic/kibana/issues/140241 + describe.skip('embed in dashboard', () => { before(async () => { await security.testUser.setRoles( [ From e826b9c183e2105e5bd84496769d6e6da9ed7d74 Mon Sep 17 00:00:00 2001 From: Philippe Oberti Date: Tue, 20 Sep 2022 13:43:04 -0500 Subject: [PATCH 39/76] [TIP] Replace indicator field-selector EuiSelect by EuiComboBox (#140980) [TIP] Replace indicator field-selector EuiSelect by EuiComboBox --- .../cypress/e2e/indicators.cy.ts | 12 +- .../cypress/screens/indicators.ts | 6 + .../indicators_barchart_wrapper.test.tsx.snap | 164 +++++---- .../indicators_field_selector.test.tsx.snap | 310 ++++++++++++------ .../indicators_field_selector.test.tsx | 8 +- .../indicators_field_selector.tsx | 47 ++- .../indicators_field_selector/styles.ts | 19 ++ 7 files changed, 385 insertions(+), 181 deletions(-) create mode 100644 x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/styles.ts diff --git a/x-pack/plugins/threat_intelligence/cypress/e2e/indicators.cy.ts b/x-pack/plugins/threat_intelligence/cypress/e2e/indicators.cy.ts index ccd91253012b3..f9f1252f4765e 100644 --- a/x-pack/plugins/threat_intelligence/cypress/e2e/indicators.cy.ts +++ b/x-pack/plugins/threat_intelligence/cypress/e2e/indicators.cy.ts @@ -25,6 +25,9 @@ import { ENDING_BREADCRUMB, FIELD_BROWSER, FIELD_BROWSER_MODAL, + FIELD_SELECTOR_TOGGLE_BUTTON, + FIELD_SELECTOR_INPUT, + FIELD_SELECTOR_LIST, } from '../screens/indicators'; import { login } from '../tasks/login'; import { esArchiverLoad, esArchiverUnload } from '../tasks/es_archiver'; @@ -143,16 +146,13 @@ describe('Indicators', () => { it('should have the default selected field, then update when user selects', () => { const threatFeedName = 'threat.feed.name'; - cy.get(`${FIELD_SELECTOR}`).should('have.value', threatFeedName); + cy.get(`${FIELD_SELECTOR_INPUT}`).eq(0).should('have.text', threatFeedName); const threatIndicatorIp: string = 'threat.indicator.ip'; - cy.get(`${FIELD_SELECTOR}`) - .should('exist') - .select(threatIndicatorIp) - .should('have.value', threatIndicatorIp); + cy.get(`${FIELD_SELECTOR_TOGGLE_BUTTON}`).should('exist').click(); - cy.get(`${FIELD_SELECTOR}`).should('have.value', threatIndicatorIp); + cy.get(`${FIELD_SELECTOR_LIST}`).should('exist').contains(threatIndicatorIp); }); }); diff --git a/x-pack/plugins/threat_intelligence/cypress/screens/indicators.ts b/x-pack/plugins/threat_intelligence/cypress/screens/indicators.ts index 4ceaa0c020e99..b5582da6ce8ef 100644 --- a/x-pack/plugins/threat_intelligence/cypress/screens/indicators.ts +++ b/x-pack/plugins/threat_intelligence/cypress/screens/indicators.ts @@ -41,6 +41,12 @@ export const INDICATOR_TYPE_CELL = '[data-gridcell-column-id="threat.indicator.t export const FIELD_SELECTOR = '[data-test-subj="tiIndicatorFieldSelectorDropdown"]'; +export const FIELD_SELECTOR_INPUT = '[data-test-subj="comboBoxInput"]'; + +export const FIELD_SELECTOR_TOGGLE_BUTTON = '[data-test-subj="comboBoxToggleListButton"]'; + +export const FIELD_SELECTOR_LIST = '[data-test-subj="comboBoxOptionsList"]'; + export const FIELD_BROWSER = `[data-test-subj="show-field-browser"]`; export const FIELD_BROWSER_MODAL = `[data-test-subj="fields-browser-container"]`; diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/__snapshots__/indicators_barchart_wrapper.test.tsx.snap b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/__snapshots__/indicators_barchart_wrapper.test.tsx.snap index de4d6c198961f..bc2b71303138b 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/__snapshots__/indicators_barchart_wrapper.test.tsx.snap +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_barchart_wrapper/__snapshots__/indicators_barchart_wrapper.test.tsx.snap @@ -21,43 +21,67 @@ Object { class="euiFlexItem euiFlexItem--flexGrowZero" >
-
- + Stack by +
- + class="euiComboBoxPill euiComboBoxPill--plainText" + > + threat.feed.name + +
+ +
+
+
+
+ +
@@ -102,43 +126,67 @@ Object { class="euiFlexItem euiFlexItem--flexGrowZero" >
-
- + Stack by +
- + class="euiComboBoxPill euiComboBoxPill--plainText" + > + threat.feed.name + +
+ +
+
+
+
+ +
diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/__snapshots__/indicators_field_selector.test.tsx.snap b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/__snapshots__/indicators_field_selector.test.tsx.snap index 735a1c34718c0..9346a739b9dae 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/__snapshots__/indicators_field_selector.test.tsx.snap +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/__snapshots__/indicators_field_selector.test.tsx.snap @@ -6,43 +6,67 @@ Object { "baseElement":
-
- + Stack by +
- + class="euiComboBoxPill euiComboBoxPill--plainText" + > + threat.feed.name + +
+ +
+
+
+
+ +
@@ -50,43 +74,67 @@ Object { , "container":
-
- + Stack by +
- + class="euiComboBoxPill euiComboBoxPill--plainText" + > + threat.feed.name + +
+ +
+
+
+
+ +
@@ -151,32 +199,67 @@ Object { "baseElement":
-
- +
+
+
+
+ +
@@ -184,32 +267,67 @@ Object { , "container":
-
- +
+
+
+
+ +
diff --git a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/indicators_field_selector.test.tsx b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/indicators_field_selector.test.tsx index e3bd4129d4fc5..0d62bbcef6219 100644 --- a/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/indicators_field_selector.test.tsx +++ b/x-pack/plugins/threat_intelligence/public/modules/indicators/components/indicators_field_selector/indicators_field_selector.test.tsx @@ -9,7 +9,7 @@ import React from 'react'; import { render } from '@testing-library/react'; import { DataView, DataViewField } from '@kbn/data-views-plugin/common'; import { TestProvidersComponent } from '../../../../common/mocks/test_providers'; -import { DROPDOWN_TEST_ID, IndicatorsFieldSelector } from './indicators_field_selector'; +import { IndicatorsFieldSelector } from './indicators_field_selector'; const mockIndexPattern: DataView = { fields: [ @@ -51,11 +51,5 @@ describe('', () => { ); expect(component).toMatchSnapshot(); - - const dropdownOptions: string = component.getByTestId(DROPDOWN_TEST_ID).innerHTML; - const optionsCount: number = (dropdownOptions.match(/