From ec05dd7afddaef353d27f0bcbc7046ff09c0a5d6 Mon Sep 17 00:00:00 2001 From: Andrew Macri Date: Thu, 21 Dec 2023 18:01:15 -0500 Subject: [PATCH] [Security Solution] [Elastic AI Assistant] Delete the _Retrieval Augmented Generation (RAG) for Alerts_ Feature Flag (#173809) ## [Security Solution] [Elastic AI Assistant] Delete the _Retrieval Augmented Generation (RAG) for Alerts_ Feature Flag This PR deletes the `assistantRagOnAlerts` feature flag introduced in [[Security Solution] [Elastic AI Assistant] Retrieval Augmented Generation (RAG) for Alerts #172542](https://github.com/elastic/kibana/pull/172542). Deleting the `assistantRagOnAlerts` feature flag makes the `Alerts` toggle available in the assistant settings, per the screenshot below: ![alerts_setting](https://github.com/elastic/kibana/assets/4459398/1647a92c-653b-49de-926a-d0a3b65d270a) This PR should not be merged until the docs describing the feature in have been merged. This PR also includes @benironside improvements to the Alerts setting in the video below: https://github.com/elastic/kibana/assets/4459398/73ea2717-ad2a-4998-afe2-cc154d8d19a9 ### Desk testing To desk test this change: 1) Delete the following `assistantRagOnAlerts` feature flag from your local `config/kibana.dev.yml`: ``` xpack.securitySolution.enableExperimental: ['assistantRagOnAlerts'] ``` 2) Start Kibana 3) Generate alerts with a variety of severity (e.g. `low`, `medium`, `high`, and `critical`) 4) Navigate to Security > Alerts 5) Click the `AI Assistant` button to open the assistant 6) Click the `X` button to clear the conversation 7) Click the assistant's `Settings` gear 8) Click the `Knowledge Base` category **Expected result** - The `Alerts` toggle shown in the screenshot below is available ![alerts_setting](https://github.com/elastic/kibana/assets/4459398/1647a92c-653b-49de-926a-d0a3b65d270a) 9) Click the `Alerts` toggle to enable the feature 10) Click the `Save` button to close settings 11) Enter the following prompt: ``` How many open alerts do I have? ``` **Expected result** - A response with alert counts grouped by workflow status will be returned, similar to the example below: ``` You currently have 48 open alerts in your system. These are categorized by severity as following: 19 of them are low severity, 16 are high severity, 12 are of medium severity and 1 is of critical severity. There is also 1 critical severity alert which is acknowledged. ``` 12) Enter the following prompt: ``` Which alerts should I look at first? ``` **Expected result** A response with alert details, similar to the following is returned: ``` Based on the latest information, the alerts to prioritize first are those related to a mimikatz process starting on the hosts, which have a critical severity and the highest risk score of 99. There are also a series of alerts related to an EQL process sequence with a high severity and risk scores of 73. There is one alert about an Elastic Endpoint Security alert with a medium severity and risk score of 47. ``` 13) Once again, click the assistant's `Settings` gear 14) Click the `Knowledge Base` category 15) Click the `Alerts` toggle to disable the feature 16) Click the `Save` button to close settings 17) Once again, enter the following prompt: ``` How many open alerts do I have? ``` **Expected result** - The assistant does NOT respond with a breakdown of alerts by severity. Instead it replies with something like the following example response: ``` I'm sorry for any confusion, but as an AI, I don't have real-time access to your data or system to provide the number of your current open alerts. You can check your Elastic Security dashboard or use the appropriate querying commands to get the updated count of your open alerts. ``` 18) One more time, enter the following prompt: ``` Which alerts should I look at first? ``` **Expected result** - The assistant does NOT respond with alert details. Instead it replies with something like the following example response: ``` As an AI model, I don't have the capability to access real-time data. However, when it comes to managing alerts in Elastic Security, it's generally recommended to first look at the ones with the highest severity and risk score. Alerts related to malware, unauthorized access attempts, and abnormal data transfers or process activities, for example, may need immediate attention due to their potential high impact. ``` --- .../impl/alerts/settings/alerts_settings.tsx | 8 +--- .../impl/assistant/api.test.tsx | 2 - .../impl/assistant/api.tsx | 4 -- .../impl/assistant/helpers.test.ts | 39 +++++++------------ .../impl/assistant/helpers.ts | 12 ++---- .../assistant/use_send_messages/index.tsx | 3 -- .../impl/assistant_context/index.tsx | 5 --- .../knowledge_base_settings.test.tsx | 3 +- .../knowledge_base_settings.tsx | 12 +++--- .../impl/knowledge_base/translations.ts | 20 +++++----- .../common/experimental_features.ts | 5 --- .../public/assistant/provider.tsx | 2 - 12 files changed, 34 insertions(+), 81 deletions(-) diff --git a/x-pack/packages/kbn-elastic-assistant/impl/alerts/settings/alerts_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/alerts/settings/alerts_settings.tsx index f23470bbbe7a7..4e9bf8d2726ea 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/alerts/settings/alerts_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/alerts/settings/alerts_settings.tsx @@ -70,12 +70,6 @@ const AlertsSettingsComponent = ({ knowledgeBase, setUpdatedKnowledgeBaseSetting - - - {i18n.ASK_QUESTIONS_ABOUT} - - - - {i18n.LATEST_AND_RISKIEST_OPEN_ALERTS} + {i18n.LATEST_AND_RISKIEST_OPEN_ALERTS(knowledgeBase.latestAlerts)} diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx index ebb5afe2f12a1..9119e024adcb2 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.test.tsx @@ -42,7 +42,6 @@ const fetchConnectorArgs: FetchConnectorExecuteAction = { http: mockHttp, messages, onNewReplacements: jest.fn(), - ragOnAlerts: false, }; describe('API tests', () => { beforeEach(() => { @@ -91,7 +90,6 @@ describe('API tests', () => { alertsIndexPattern: '.alerts-security.alerts-default', allow: ['a', 'b', 'c'], allowReplacement: ['b', 'c'], - ragOnAlerts: true, replacements: { auuid: 'real.hostname' }, size: 30, }; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx index c2bdd4806a99a..f186dab22f668 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx @@ -29,7 +29,6 @@ export interface FetchConnectorExecuteAction { http: HttpSetup; messages: Message[]; onNewReplacements: (newReplacements: Record) => void; - ragOnAlerts: boolean; replacements?: Record; signal?: AbortSignal | undefined; size?: number; @@ -55,7 +54,6 @@ export const fetchConnectorExecuteAction = async ({ http, messages, onNewReplacements, - ragOnAlerts, replacements, apiConfig, signal, @@ -90,7 +88,6 @@ export const fetchConnectorExecuteAction = async ({ alertsIndexPattern, allow, allowReplacement, - ragOnAlerts, replacements, size, }); @@ -192,7 +189,6 @@ export const fetchConnectorExecuteAction = async ({ response: hasParsableResponse({ alerts, assistantLangChain, - ragOnAlerts, }) ? getFormattedMessageContent(response.data) : response.data, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts index 0c3c5a579d274..8cd3c1479b46b 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.test.ts @@ -235,29 +235,12 @@ describe('getBlockBotConversation', () => { }); describe('getOptionalRequestParams', () => { - it('should return an empty object when ragOnAlerts is false', () => { - const params = { - alerts: true, - alertsIndexPattern: 'indexPattern', - allow: ['a', 'b', 'c'], - allowReplacement: ['b', 'c'], - ragOnAlerts: false, // <-- false - replacements: { key: 'value' }, - size: 10, - }; - - const result = getOptionalRequestParams(params); - - expect(result).toEqual({}); - }); - it('should return an empty object when alerts is false', () => { const params = { alerts: false, // <-- false alertsIndexPattern: 'indexPattern', allow: ['a', 'b', 'c'], allowReplacement: ['b', 'c'], - ragOnAlerts: true, replacements: { key: 'value' }, size: 10, }; @@ -267,13 +250,12 @@ describe('getBlockBotConversation', () => { expect(result).toEqual({}); }); - it('should return the optional request params when ragOnAlerts is true and alerts is true', () => { + it('should return the optional request params when alerts is true', () => { const params = { alerts: true, alertsIndexPattern: 'indexPattern', allow: ['a', 'b', 'c'], allowReplacement: ['b', 'c'], - ragOnAlerts: true, replacements: { key: 'value' }, size: 10, }; @@ -292,7 +274,6 @@ describe('getBlockBotConversation', () => { it('should return (only) the optional request params that are defined when some optional params are not provided', () => { const params = { alerts: true, - ragOnAlerts: true, allow: ['a', 'b', 'c'], // all the others are undefined }; @@ -305,31 +286,37 @@ describe('getBlockBotConversation', () => { }); describe('hasParsableResponse', () => { - it('returns true when assistantLangChain is true', () => { + it('returns true when just assistantLangChain is true', () => { const result = hasParsableResponse({ alerts: false, assistantLangChain: true, - ragOnAlerts: false, }); expect(result).toBe(true); }); - it('returns true when ragOnAlerts is true and alerts is true', () => { + it('returns true when just alerts is true', () => { const result = hasParsableResponse({ alerts: true, assistantLangChain: false, - ragOnAlerts: true, }); expect(result).toBe(true); }); - it('returns false when assistantLangChain, ragOnAlerts, and alerts are all false', () => { + it('returns true when both assistantLangChain and alerts are true', () => { + const result = hasParsableResponse({ + alerts: true, + assistantLangChain: true, + }); + + expect(result).toBe(true); + }); + + it('returns false when both assistantLangChain and alerts are false', () => { const result = hasParsableResponse({ alerts: false, assistantLangChain: false, - ragOnAlerts: false, }); expect(result).toBe(false); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts index 688416d2e738c..9149d4c84ca42 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/helpers.ts @@ -101,7 +101,6 @@ export const getOptionalRequestParams = ({ alertsIndexPattern, allow, allowReplacement, - ragOnAlerts, replacements, size, }: { @@ -109,7 +108,6 @@ export const getOptionalRequestParams = ({ alertsIndexPattern?: string; allow?: string[]; allowReplacement?: string[]; - ragOnAlerts: boolean; replacements?: Record; size?: number; }): OptionalRequestParams => { @@ -119,10 +117,8 @@ export const getOptionalRequestParams = ({ const optionalReplacements = replacements ? { replacements } : undefined; const optionalSize = size ? { size } : undefined; - if ( - !ragOnAlerts || // the feature flag must be enabled - !alerts // the settings toggle must also be enabled - ) { + // the settings toggle must be enabled: + if (!alerts) { return {}; // don't send any optional params } @@ -138,9 +134,7 @@ export const getOptionalRequestParams = ({ export const hasParsableResponse = ({ alerts, assistantLangChain, - ragOnAlerts, }: { alerts: boolean; assistantLangChain: boolean; - ragOnAlerts: boolean; -}): boolean => assistantLangChain || (ragOnAlerts && alerts); +}): boolean => assistantLangChain || alerts; diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx index fcfbadb574bbd..fb973d492a651 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_send_messages/index.tsx @@ -37,7 +37,6 @@ export const useSendMessages = (): UseSendMessages => { assistantStreamingEnabled, defaultAllow, defaultAllowReplacement, - ragOnAlerts, knowledgeBase, } = useAssistantContext(); const [isLoading, setIsLoading] = useState(false); @@ -56,7 +55,6 @@ export const useSendMessages = (): UseSendMessages => { assistantLangChain: knowledgeBase.assistantLangChain, assistantStreamingEnabled, http, - ragOnAlerts, // feature flag replacements, messages, size: knowledgeBase.latestAlerts, @@ -74,7 +72,6 @@ export const useSendMessages = (): UseSendMessages => { knowledgeBase.alerts, knowledgeBase.assistantLangChain, knowledgeBase.latestAlerts, - ragOnAlerts, ] ); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx index afb785e2025bd..024a06fd7b314 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant_context/index.tsx @@ -89,7 +89,6 @@ export interface AssistantProviderProps { getInitialConversations: () => Record; modelEvaluatorEnabled?: boolean; nameSpace?: string; - ragOnAlerts?: boolean; setConversations: React.Dispatch>>; setDefaultAllow: React.Dispatch>; setDefaultAllowReplacement: React.Dispatch>; @@ -141,7 +140,6 @@ export interface UseAssistantContext { promptContexts: Record; modelEvaluatorEnabled: boolean; nameSpace: string; - ragOnAlerts: boolean; registerPromptContext: RegisterPromptContext; selectedSettingsTab: SettingsTabs; setAllQuickPrompts: React.Dispatch>; @@ -183,7 +181,6 @@ export const AssistantProvider: React.FC = ({ getInitialConversations, modelEvaluatorEnabled = false, nameSpace = DEFAULT_ASSISTANT_NAMESPACE, - ragOnAlerts = false, setConversations, setDefaultAllow, setDefaultAllowReplacement, @@ -328,7 +325,6 @@ export const AssistantProvider: React.FC = ({ modelEvaluatorEnabled, promptContexts, nameSpace, - ragOnAlerts, registerPromptContext, selectedSettingsTab, setAllQuickPrompts: setLocalStorageQuickPrompts, @@ -374,7 +370,6 @@ export const AssistantProvider: React.FC = ({ nameSpace, onConversationsUpdated, promptContexts, - ragOnAlerts, registerPromptContext, selectedSettingsTab, setDefaultAllow, diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.test.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.test.tsx index 06c1b33bfda85..6e5d7e7b1b174 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.test.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.test.tsx @@ -22,7 +22,6 @@ const mockUseAssistantContext = { prepend: jest.fn(), }, }, - ragOnAlerts: true, setAllSystemPrompts: jest.fn(), setConversations: jest.fn(), }; @@ -210,7 +209,7 @@ describe('Knowledge base settings', () => { expect(queryByTestId('knowledgeBaseActionButton')).not.toBeInTheDocument(); }); - it('renders the alerts settings when ragOnAlerts is true', () => { + it('renders the alerts settings', () => { const { getByTestId } = render( diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.tsx index bd41f5b888c93..5974bae6e5ab0 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/knowledge_base_settings.tsx @@ -47,7 +47,7 @@ interface Props { */ export const KnowledgeBaseSettings: React.FC = React.memo( ({ knowledgeBase, setUpdatedKnowledgeBaseSettings }) => { - const { http, ragOnAlerts } = useAssistantContext(); + const { http } = useAssistantContext(); const { data: kbStatus, isLoading, @@ -303,12 +303,10 @@ export const KnowledgeBaseSettings: React.FC = React.memo( - {ragOnAlerts && ( - - )} + ); } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/translations.ts b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/translations.ts index 03e989ab6a055..e1b176e9dcaa7 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/translations.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/knowledge_base/translations.ts @@ -21,25 +21,27 @@ export const ASK_QUESTIONS_ABOUT = i18n.translate( } ); -export const LATEST_AND_RISKIEST_OPEN_ALERTS = i18n.translate( - 'xpack.elasticAssistant.assistant.settings.knowledgeBaseSettings.latestAndRiskiestOpenAlertsLabel', - { - defaultMessage: 'latest and riskiest open and acknowledged alerts in your environment.', - } -); +export const LATEST_AND_RISKIEST_OPEN_ALERTS = (alertsCount: number) => + i18n.translate( + 'xpack.elasticAssistant.assistant.settings.knowledgeBaseSettings.latestAndRiskiestOpenAlertsLabel', + { + defaultMessage: + 'Send AI Assistant information about your {alertsCount} newest and riskiest open or acknowledged alerts.', + values: { alertsCount }, + } + ); export const YOUR_ANONYMIZATION_SETTINGS = i18n.translate( 'xpack.elasticAssistant.assistant.settings.knowledgeBaseSettings.yourAnonymizationSettingsLabel', { - defaultMessage: 'Your Anonymization settings will be applied to the alerts.', + defaultMessage: 'Your anonymization settings will apply to these alerts.', } ); export const SELECT_FEWER_ALERTS = i18n.translate( 'xpack.elasticAssistant.assistant.settings.knowledgeBaseSettings.selectFewerAlertsLabel', { - defaultMessage: - "Select fewer alerts if the model's maximum context length is frequently exceeded.", + defaultMessage: "Send fewer alerts if the model's context window is too small.", } ); diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 85525ff82bc1e..805192aed8a9f 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -99,11 +99,6 @@ export const allowedExperimentalValues = Object.freeze({ */ assistantModelEvaluation: false, - /** - * Enables Retrieval Augmented Generation (RAG) on Alerts in the assistant - */ - assistantRagOnAlerts: false, - /* * Enables the new user details flyout displayed on the Alerts page and timeline. * diff --git a/x-pack/plugins/security_solution/public/assistant/provider.tsx b/x-pack/plugins/security_solution/public/assistant/provider.tsx index 7a17a98bc0d6e..a9f9e14a8d3e0 100644 --- a/x-pack/plugins/security_solution/public/assistant/provider.tsx +++ b/x-pack/plugins/security_solution/public/assistant/provider.tsx @@ -57,7 +57,6 @@ export const AssistantProvider: React.FC = ({ children }) => { const { signalIndexName } = useSignalIndex(); const alertsIndexPattern = signalIndexName ?? undefined; - const ragOnAlerts = useIsExperimentalFeatureEnabled('assistantRagOnAlerts'); const toasts = useAppToasts() as unknown as IToasts; // useAppToasts is the current, non-deprecated method of getting the toasts service in the Security Solution, but it doesn't return the IToasts interface (defined by core) return ( @@ -82,7 +81,6 @@ export const AssistantProvider: React.FC = ({ children }) => { assistantStreamingEnabled={assistantStreamingEnabled} modelEvaluatorEnabled={isModelEvaluationEnabled} nameSpace={nameSpace} - ragOnAlerts={ragOnAlerts} setConversations={setConversations} setDefaultAllow={setDefaultAllow} setDefaultAllowReplacement={setDefaultAllowReplacement}