diff --git a/.buildkite/scripts/steps/cloud/deploy.json b/.buildkite/scripts/steps/cloud/deploy.json index da1918a2a6953..336fed24afbe3 100644 --- a/.buildkite/scripts/steps/cloud/deploy.json +++ b/.buildkite/scripts/steps/cloud/deploy.json @@ -141,7 +141,7 @@ ], "id": "ml", "size": { - "value": 1024, + "value": 4096, "resource": "memory" } } diff --git a/package.json b/package.json index e4e89ddceb856..2cf84a7c07b02 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,6 @@ "resolutions": { "**/@bazel/typescript/protobufjs": "6.11.4", "**/@hello-pangea/dnd": "16.6.0", - "**/@langchain/core": "0.2.3", "**/@types/node": "20.10.5", "**/@typescript-eslint/utils": "5.62.0", "**/chokidar": "^3.5.3", @@ -88,6 +87,7 @@ "**/globule/minimatch": "^3.1.2", "**/hoist-non-react-statics": "^3.3.2", "**/isomorphic-fetch/node-fetch": "^2.6.7", + "**/langchain": "0.2.8", "**/react-intl/**/@types/react": "^17.0.45", "**/remark-parse/trim": "1.0.1", "**/sharp": "0.32.6", @@ -932,10 +932,10 @@ "@kbn/watcher-plugin": "link:x-pack/plugins/watcher", "@kbn/xstate-utils": "link:packages/kbn-xstate-utils", "@kbn/zod-helpers": "link:packages/kbn-zod-helpers", - "@langchain/community": "^0.2.4", - "@langchain/core": "0.2.3", - "@langchain/langgraph": "^0.0.23", - "@langchain/openai": "^0.0.34", + "@langchain/community": "^0.2.16", + "@langchain/core": "^0.2.11", + "@langchain/langgraph": "^0.0.25", + "@langchain/openai": "^0.1.3", "@langtrase/trace-attributes": "^3.0.8", "@launchdarkly/node-server-sdk": "^9.4.6", "@loaders.gl/core": "^3.4.7", @@ -1072,7 +1072,7 @@ "jsonwebtoken": "^9.0.2", "jsts": "^1.6.2", "kea": "^2.6.0", - "langchain": "0.2.3", + "langchain": "^0.2.8", "langsmith": "^0.1.30", "launchdarkly-js-client-sdk": "^3.4.0", "launchdarkly-node-server-sdk": "^7.0.3", diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx index 44907d8b1fd00..bc0d484de49e7 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/assistant_overlay/index.tsx @@ -6,11 +6,10 @@ */ import React, { useCallback, useEffect, useRef, useState } from 'react'; -import { EuiModal, EuiFlyoutResizable, useEuiTheme } from '@elastic/eui'; - +import { EuiModal, EuiFlyoutResizable } from '@elastic/eui'; import useEvent from 'react-use/lib/useEvent'; // eslint-disable-next-line @kbn/eslint/module_migration -import styled from 'styled-components'; +import styled, { createGlobalStyle } from 'styled-components'; import { css } from '@emotion/react'; import { ShowAssistantOverlayProps, @@ -37,8 +36,13 @@ export interface Props { currentUserAvatar?: UserAvatar; } +export const UnifiedTimelineGlobalStyles = createGlobalStyle` + body:has(.timeline-portal-overlay-mask) .euiOverlayMask { + z-index: 1003 !important; + } +`; + export const AssistantOverlay = React.memo(({ isFlyoutMode, currentUserAvatar }) => { - const { euiTheme } = useEuiTheme(); const [isModalVisible, setIsModalVisible] = useState(false); const [conversationTitle, setConversationTitle] = useState( WELCOME_CONVERSATION_TITLE @@ -132,32 +136,33 @@ export const AssistantOverlay = React.memo(({ isFlyoutMode, currentUserAv if (isFlyoutMode) { return ( - div { - height: 100%; - } - `} - onClose={handleCloseModal} - data-test-subj="ai-assistant-flyout" - paddingSize="none" - hideCloseButton - // EUI TODO: This z-index override of EuiOverlayMask is a workaround, and ideally should be resolved with a cleaner UI/UX flow long-term - maskProps={{ style: `z-index: ${(euiTheme.levels.flyout as number) + 3}` }} // we need this flyout to be above the timeline flyout (which has a z-index of 1002) - > - - + <> + div { + height: 100%; + } + `} + onClose={handleCloseModal} + data-test-subj="ai-assistant-flyout" + paddingSize="none" + hideCloseButton + > + + + + ); } diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx index 830c5d2b7080a..7aa29cef29644 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/index.tsx @@ -519,14 +519,6 @@ const AssistantComponent: React.FC = ({ isFetchedAnonymizationFields, ]); - useEffect(() => {}, [ - areConnectorsFetched, - connectors, - conversationsLoaded, - currentConversation, - isLoading, - ]); - const createCodeBlockPortals = useCallback( () => messageCodeBlocks?.map((codeBlocks: CodeBlockDetails[], i: number) => { diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts index c8c8ab5ff7727..a88c9f8d5b985 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.test.ts @@ -88,11 +88,11 @@ describe('useConversation helpers', () => { describe('analyzeMarkdown', () => { it('should identify dsl Query successfully.', () => { - const result = analyzeMarkdown(markDownWithDSLQuery); + const result = analyzeMarkdown(markDownWithDSLQuery, new Date().toISOString()); expect(result[0].type).toBe('dsl'); }); it('should identify kql Query successfully.', () => { - const result = analyzeMarkdown(markDownWithKQLQuery); + const result = analyzeMarkdown(markDownWithKQLQuery, new Date().toISOString()); expect(result[0].type).toBe('kql'); }); }); diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.ts index 2d6c4075fba0e..d40ae2f6409ee 100644 --- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.ts +++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/use_conversation/helpers.ts @@ -14,6 +14,7 @@ import { getGenAiConfig } from '../../connectorland/helpers'; export interface CodeBlockDetails { type: QueryType; content: string; + timestamp: string; start: number; end: number; getControlContainer?: () => Element | undefined; @@ -22,6 +23,20 @@ export interface CodeBlockDetails { export type QueryType = 'eql' | 'esql' | 'kql' | 'dsl' | 'json' | 'no-type' | 'sql'; +// If your codeblocks aren't getting tagged with the right language, add keywords to the array. +export const MARKDOWN_TYPES = { + eql: ['Event Query Language', 'EQL sequence query', 'EQL'], + esql: ['Elasticsearch Query Language', 'ESQL', 'ES|QL', 'SQL'], + kql: ['Kibana Query Language', 'KQL Query', 'KQL'], + dsl: [ + 'Elasticsearch QueryDSL', + 'Elasticsearch Query DSL', + 'Elasticsearch DSL', + 'Query DSL', + 'DSL', + ], +}; + /** * `analyzeMarkdown` is a helper that enriches content returned from a query * with action buttons @@ -31,29 +46,16 @@ export type QueryType = 'eql' | 'esql' | 'kql' | 'dsl' | 'json' | 'no-type' | 's * * @param markdown */ -export const analyzeMarkdown = (markdown: string): CodeBlockDetails[] => { +export const analyzeMarkdown = (markdown: string, timestamp: string): CodeBlockDetails[] => { const codeBlockRegex = /```(\w+)?\s([\s\S]*?)```/g; const matches = [...markdown.matchAll(codeBlockRegex)]; - // If your codeblocks aren't getting tagged with the right language, add keywords to the array. - const types = { - eql: ['Event Query Language', 'EQL sequence query', 'EQL'], - esql: ['Elasticsearch Query Language', 'ESQL', 'ES|QL', 'SQL'], - kql: ['Kibana Query Language', 'KQL Query', 'KQL'], - dsl: [ - 'Elasticsearch QueryDSL', - 'Elasticsearch Query DSL', - 'Elasticsearch DSL', - 'Query DSL', - 'DSL', - ], - }; const result: CodeBlockDetails[] = matches.map((match) => { let type = match[1] || 'no-type'; if (type === 'no-type' || type === 'json') { const start = match.index || 0; const precedingText = markdown.slice(0, start); - for (const [typeKey, keywords] of Object.entries(types)) { + for (const [typeKey, keywords] of Object.entries(MARKDOWN_TYPES)) { if (keywords.some((kw) => precedingText.toLowerCase().includes(kw.toLowerCase()))) { type = typeKey; break; @@ -64,7 +66,7 @@ export const analyzeMarkdown = (markdown: string): CodeBlockDetails[] => { const content = match[2].trim(); const start = match.index || 0; const end = start + match[0].length; - return { type: type as QueryType, content, start, end }; + return { type: type as QueryType, content, start, end, timestamp }; }); return result; diff --git a/x-pack/packages/kbn-langchain/server/language_models/types.ts b/x-pack/packages/kbn-langchain/server/language_models/types.ts index df866bdf30eb7..43dcad34fda3c 100644 --- a/x-pack/packages/kbn-langchain/server/language_models/types.ts +++ b/x-pack/packages/kbn-langchain/server/language_models/types.ts @@ -16,7 +16,7 @@ export interface InvokeAIActionParamsSchema { function_call?: { arguments: string; name: string; - }; + } | null; tool_calls?: Array<{ id: string; diff --git a/x-pack/plugins/cases/public/components/__mock__/timeline.tsx b/x-pack/plugins/cases/public/components/__mock__/timeline.tsx index d576b0ef1732c..7bc6951eb1e25 100644 --- a/x-pack/plugins/cases/public/components/__mock__/timeline.tsx +++ b/x-pack/plugins/cases/public/components/__mock__/timeline.tsx @@ -12,8 +12,15 @@ const mockTimelineComponent = (name: string) => {nam export const timelineIntegrationMock = { editor_plugins: { - parsingPlugin: jest.fn(), - processingPluginRenderer: () => mockTimelineComponent('plugin-renderer'), + parsingPlugins: { + timeline: jest.fn(), + customCodeBlock: jest.fn(), + }, + processingPluginRenderer: { + timeline: () => mockTimelineComponent('plugin-renderer'), + esql: () => mockTimelineComponent('esql-plugin-renderer'), + customCodeBlock: () => mockTimelineComponent('custom-code-block-plugin-renderer'), + }, uiPlugin: { name: 'mock-timeline', button: { label: 'mock-timeline-button', iconType: 'mock-timeline-icon' }, diff --git a/x-pack/plugins/cases/public/components/markdown_editor/editor.tsx b/x-pack/plugins/cases/public/components/markdown_editor/editor.tsx index 92580b0e1da20..a1d7bc1cc0652 100644 --- a/x-pack/plugins/cases/public/components/markdown_editor/editor.tsx +++ b/x-pack/plugins/cases/public/components/markdown_editor/editor.tsx @@ -40,7 +40,9 @@ const MarkdownEditorComponent = forwardRef(null); useLensButtonToggle({ diff --git a/x-pack/plugins/cases/public/components/markdown_editor/renderer.tsx b/x-pack/plugins/cases/public/components/markdown_editor/renderer.tsx index f896da64cf021..b6ac0295c9cc0 100644 --- a/x-pack/plugins/cases/public/components/markdown_editor/renderer.tsx +++ b/x-pack/plugins/cases/public/components/markdown_editor/renderer.tsx @@ -16,6 +16,7 @@ interface Props { children: string; disableLinks?: boolean; textSize?: EuiMarkdownFormatProps['textSize']; + timestamp?: string; } const withDisabledLinks = (disableLinks?: boolean): React.FC => { @@ -28,8 +29,13 @@ const withDisabledLinks = (disableLinks?: boolean): React.FC return MarkdownLinkProcessingComponent; }; -const MarkdownRendererComponent: React.FC = ({ children, disableLinks, textSize }) => { - const { processingPlugins, parsingPlugins } = usePlugins(); +const MarkdownRendererComponent: React.FC = ({ + children, + disableLinks, + textSize, + timestamp, +}) => { + const { processingPlugins, parsingPlugins } = usePlugins({ timestamp }); // Deep clone of the processing plugins to prevent affecting the markdown editor. const processingPluginList = cloneDeep(processingPlugins); // This line of code is TS-compatible and it will break if [1][1] change in the future. diff --git a/x-pack/plugins/cases/public/components/markdown_editor/scrollable_markdown_renderer.tsx b/x-pack/plugins/cases/public/components/markdown_editor/scrollable_markdown_renderer.tsx index e4ca3d175cdf0..7db5a05e3acb1 100644 --- a/x-pack/plugins/cases/public/components/markdown_editor/scrollable_markdown_renderer.tsx +++ b/x-pack/plugins/cases/public/components/markdown_editor/scrollable_markdown_renderer.tsx @@ -20,7 +20,13 @@ export const getContentWrapperCss = (euiTheme: EuiThemeComputed<{}>) => css` -webkit-box-orient: vertical; `; -const ScrollableMarkdownRenderer = ({ content }: { content: string }) => { +const ScrollableMarkdownRenderer = ({ + content, + timestamp, +}: { + content: string; + timestamp?: string; +}) => { const { euiTheme } = useEuiTheme(); return (
{ css={getContentWrapperCss(euiTheme)} data-test-subj="scrollable-markdown" > - {content} + {content}
); }; diff --git a/x-pack/plugins/cases/public/components/markdown_editor/types.ts b/x-pack/plugins/cases/public/components/markdown_editor/types.ts index 82ff738a178bf..846bded0ed9f8 100644 --- a/x-pack/plugins/cases/public/components/markdown_editor/types.ts +++ b/x-pack/plugins/cases/public/components/markdown_editor/types.ts @@ -21,7 +21,13 @@ export type TemporaryProcessingPluginsType = [ [ typeof rehype2react, Parameters[0] & { - components: { a: FunctionComponent; lens: unknown; timeline: unknown }; + components: { + a: FunctionComponent; + lens: unknown; + timeline: unknown; + customCodeBlock: unknown; + esql: unknown; + }; } ], ...PluggableList diff --git a/x-pack/plugins/cases/public/components/markdown_editor/use_plugins.ts b/x-pack/plugins/cases/public/components/markdown_editor/use_plugins.ts index 2ed36feebcb4e..e2a74b3ad13ef 100644 --- a/x-pack/plugins/cases/public/components/markdown_editor/use_plugins.ts +++ b/x-pack/plugins/cases/public/components/markdown_editor/use_plugins.ts @@ -17,7 +17,13 @@ import { KibanaServices, useApplicationCapabilities } from '../../common/lib/kib import * as lensMarkdownPlugin from './plugins/lens'; import { ID as LensPluginId } from './plugins/lens/constants'; -export const usePlugins = (disabledPlugins?: string[]) => { +export const usePlugins = ({ + disabledPlugins, + timestamp, +}: { + disabledPlugins?: string[]; + timestamp?: string; +} = {}) => { const kibanaConfig = KibanaServices.getConfig(); const timelinePlugins = useTimelineContext()?.editor_plugins; const appCapabilities = useApplicationCapabilities(); @@ -31,10 +37,18 @@ export const usePlugins = (disabledPlugins?: string[]) => { if (timelinePlugins) { uiPlugins.push(timelinePlugins.uiPlugin); - parsingPlugins.push(timelinePlugins.parsingPlugin); + parsingPlugins.unshift(timelinePlugins.parsingPlugins.customCodeBlock); + parsingPlugins.push(timelinePlugins.parsingPlugins.timeline); // This line of code is TS-compatible and it will break if [1][1] change in the future. - processingPlugins[1][1].components.timeline = timelinePlugins.processingPluginRenderer; + processingPlugins[1][1].components.timeline = + timelinePlugins.processingPluginRenderer.timeline; + processingPlugins[1][1].components.esql = (props: { + value: string; + actionsDisabled: boolean; + }) => timelinePlugins.processingPluginRenderer.esql({ timestamp: timestamp ?? '', ...props }); + processingPlugins[1][1].components.customCodeBlock = + timelinePlugins.processingPluginRenderer.customCodeBlock; } if ( @@ -59,5 +73,6 @@ export const usePlugins = (disabledPlugins?: string[]) => { disabledPlugins, kibanaConfig?.markdownPlugins?.lens, timelinePlugins, + timestamp, ]); }; diff --git a/x-pack/plugins/cases/public/components/timeline_context/index.tsx b/x-pack/plugins/cases/public/components/timeline_context/index.tsx index 3b5e4949150c0..5a48d6f879824 100644 --- a/x-pack/plugins/cases/public/components/timeline_context/index.tsx +++ b/x-pack/plugins/cases/public/components/timeline_context/index.tsx @@ -30,10 +30,25 @@ interface TimelineProcessingPluginRendererProps { export interface CasesTimelineIntegration { editor_plugins: { - parsingPlugin: Plugin; - processingPluginRenderer: React.FC< - TimelineProcessingPluginRendererProps & { position: EuiMarkdownAstNodePosition } - >; + parsingPlugins: { + timeline: Plugin; + customCodeBlock: Plugin; + }; + processingPluginRenderer: { + timeline: React.FC< + TimelineProcessingPluginRendererProps & { position: EuiMarkdownAstNodePosition } + >; + esql: ({ + timestamp, + value, + actionsDisabled, + }: { + value: string; + timestamp: string; + actionsDisabled: boolean; + }) => JSX.Element; + customCodeBlock: ({ value }: { value: string }) => JSX.Element; + }; uiPlugin: EuiMarkdownEditorUiPlugin; }; hooks: { diff --git a/x-pack/plugins/cases/public/components/user_actions/comment/actions.tsx b/x-pack/plugins/cases/public/components/user_actions/comment/actions.tsx index 84f0ed768edf6..2b1cef6fc8ab8 100644 --- a/x-pack/plugins/cases/public/components/user_actions/comment/actions.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/comment/actions.tsx @@ -54,7 +54,7 @@ export const createActionAttachmentUserActionBuilder = ({ timelineAvatarAriaLabel: actionIconName, actions: , children: comment.comment.trim().length > 0 && ( - + ), }, ]; diff --git a/x-pack/plugins/cases/public/components/user_actions/comment/user.tsx b/x-pack/plugins/cases/public/components/user_actions/comment/user.tsx index 67366ee4a81d3..6bea462851c00 100644 --- a/x-pack/plugins/cases/public/components/user_actions/comment/user.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/comment/user.tsx @@ -99,6 +99,7 @@ export const createUserAttachmentUserActionBuilder = ({ ref={(element) => (commentRefs.current[comment.id] = element)} id={comment.id} content={comment.comment} + timestamp={comment.createdAt} isEditable={isEdit} caseId={caseId} onChangeEditable={handleManageMarkdownEditId} diff --git a/x-pack/plugins/cases/public/components/user_actions/markdown_form.test.tsx b/x-pack/plugins/cases/public/components/user_actions/markdown_form.test.tsx index 87bae9a4624a0..d8758363c29dd 100644 --- a/x-pack/plugins/cases/public/components/user_actions/markdown_form.test.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/markdown_form.test.tsx @@ -30,6 +30,7 @@ const defaultProps = { draftStorageKey, onChangeEditable, onSaveContent, + timestamp: new Date().toISOString(), }; describe('UserActionMarkdown ', () => { diff --git a/x-pack/plugins/cases/public/components/user_actions/markdown_form.tsx b/x-pack/plugins/cases/public/components/user_actions/markdown_form.tsx index 3866fe774ec14..75df5db9751f1 100644 --- a/x-pack/plugins/cases/public/components/user_actions/markdown_form.tsx +++ b/x-pack/plugins/cases/public/components/user_actions/markdown_form.tsx @@ -12,6 +12,7 @@ import { ScrollableMarkdown, EditableMarkdown } from '../markdown_editor'; interface UserActionMarkdownProps { content: string; + timestamp: string; id: string; caseId: string; isEditable: boolean; @@ -26,7 +27,7 @@ export interface UserActionMarkdownRefObject { const UserActionMarkdownComponent = forwardRef< UserActionMarkdownRefObject, UserActionMarkdownProps ->(({ id, content, caseId, isEditable, onChangeEditable, onSaveContent }, ref) => { +>(({ id, content, timestamp, caseId, isEditable, onChangeEditable, onSaveContent }, ref) => { const editorRef = useRef(); const fieldName = 'content'; @@ -43,7 +44,7 @@ const UserActionMarkdownComponent = forwardRef< formSchema={schema} /> ) : ( - + ); }); diff --git a/x-pack/plugins/elastic_assistant/kibana.jsonc b/x-pack/plugins/elastic_assistant/kibana.jsonc index 9879ba274d209..6db16861bf934 100644 --- a/x-pack/plugins/elastic_assistant/kibana.jsonc +++ b/x-pack/plugins/elastic_assistant/kibana.jsonc @@ -10,6 +10,7 @@ "requiredPlugins": [ "actions", "data", + "dataViews", "ml", "taskManager", "licensing", diff --git a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts index c6a9951e89e3e..78b2a35e53205 100644 --- a/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts +++ b/x-pack/plugins/elastic_assistant/server/__mocks__/request_context.ts @@ -9,6 +9,8 @@ import { licensingMock } from '@kbn/licensing-plugin/server/mocks'; import { actionsClientMock } from '@kbn/actions-plugin/server/actions_client/actions_client.mock'; import { MockedKeys } from '@kbn/utility-types-jest'; import { AwaitedProperties } from '@kbn/utility-types'; +import { dataViewsService as dataViewsServiceMock } from '@kbn/data-views-plugin/server/mocks'; +import { dataPluginMock } from '@kbn/data-plugin/server/mocks'; import { ElasticAssistantApiRequestHandlerContext, ElasticAssistantRequestHandlerContext, @@ -132,6 +134,8 @@ const createElasticAssistantRequestContextMock = ( getSpaceId: jest.fn(), core: clients.core, telemetry: clients.elasticAssistant.telemetry, + search: dataPluginMock.createRequestHandlerContext().search, + dataViews: dataViewsServiceMock, }; }; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.test.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.test.ts index 6f05edbed007b..b7d222d10eced 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.test.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.test.ts @@ -12,6 +12,8 @@ import { actionsClientMock } from '@kbn/actions-plugin/server/actions_client/act import { loggerMock } from '@kbn/logging-mocks'; import { initializeAgentExecutorWithOptions } from 'langchain/agents'; +import { dataViewsService as dataViewsServiceMock } from '@kbn/data-views-plugin/server/mocks'; +import { dataPluginMock } from '@kbn/data-plugin/server/mocks'; import { mockActionResponse } from '../../../__mocks__/action_result_data'; import { langChainMessages } from '../../../__mocks__/lang_chain_messages'; @@ -105,6 +107,8 @@ const defaultProps: AgentExecutorParams = { langChainMessages, logger: mockLogger, onNewReplacements: jest.fn(), + search: dataPluginMock.createRequestHandlerContext().search, + dataViews: dataViewsServiceMock, request: mockRequest, replacements: {}, }; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts index b6a624b368d82..b09d4d45bb48e 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/execute_custom_llm_chain/index.ts @@ -50,6 +50,8 @@ export const callAgentExecutor: AgentExecutor = async ({ request, size, traceOptions, + search, + dataViews, dataClients, }) => { const isOpenAI = llmType === 'openai'; @@ -105,6 +107,7 @@ export const callAgentExecutor: AgentExecutor = async ({ anonymizationFields, chain, esClient, + esStore, isEnabledKnowledgeBase, llm, logger, @@ -113,6 +116,8 @@ export const callAgentExecutor: AgentExecutor = async ({ replacements, request, size, + search, + dataViews, }; const tools: ToolInterface[] = assistantTools.flatMap( diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts index 7af0b459f4bc9..94d073ed075e2 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/executors/types.ts @@ -13,6 +13,8 @@ import { KibanaRequest, KibanaResponseFactory, ResponseHeaders } from '@kbn/core import type { LangChainTracer } from '@langchain/core/tracers/tracer_langchain'; import { ExecuteConnectorRequestBody, Message, Replacements } from '@kbn/elastic-assistant-common'; import { StreamResponseWithHeaders } from '@kbn/ml-response-stream/server'; +import { DataPluginStart } from '@kbn/data-plugin/server/plugin'; +import { DataViewsService } from '@kbn/data-views-plugin/server'; import { PublicMethodsOf } from '@kbn/utility-types'; import { ResponseBody } from '../types'; import type { AssistantTool } from '../../../types'; @@ -43,6 +45,8 @@ export interface AgentExecutorParams { conversationId?: string; dataClients?: AssistantDataClients; esClient: ElasticsearchClient; + search: ReturnType; + dataViews: DataViewsService; esStore: ElasticsearchStore; langChainMessages: BaseMessage[]; llmType?: string; diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/helpers.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/helpers.ts index 482c89c10e969..af02171532646 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/helpers.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/helpers.ts @@ -118,6 +118,11 @@ export const streamGraph = async ({ handleStreamEnd(finalMessage); } } + } else if (event.event === 'on_llm_end') { + const generations = event.data.output?.generations[0]; + if (generations?.[0]?.generationInfo?.finish_reason === 'stop') { + handleStreamEnd(finalMessage); + } } void processEvent(); diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts index 517ac10479461..d3637db5fd4d9 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/index.ts @@ -46,6 +46,8 @@ export const callAssistantGraph: AgentExecutor = async ({ request, size, traceOptions, + search, + dataViews, responseLanguage = 'English', }) => { const logger = parentLogger.get('defaultAssistantGraph'); @@ -93,6 +95,7 @@ export const callAssistantGraph: AgentExecutor = async ({ anonymizationFields, chain, esClient, + esStore, isEnabledKnowledgeBase, kbDataClient: dataClients?.kbDataClient, llm, @@ -102,6 +105,8 @@ export const callAssistantGraph: AgentExecutor = async ({ replacements, request, size, + search, + dataViews, }; const tools: StructuredTool[] = assistantTools.flatMap( @@ -137,7 +142,15 @@ export const callAssistantGraph: AgentExecutor = async ({ const inputs = { input: latestMessage[0]?.content as string }; if (isStream) { - return streamGraph({ apmTracer, assistantGraph, inputs, logger, onLlmResponse, request }); + return streamGraph({ + apmTracer, + assistantGraph, + inputs, + logger, + onLlmResponse, + request, + traceOptions, + }); } const graphResponse = await invokeGraph({ diff --git a/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/nodes/execute_tools.ts b/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/nodes/execute_tools.ts index b42455e14f6f1..47e201650a54d 100644 --- a/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/nodes/execute_tools.ts +++ b/x-pack/plugins/elastic_assistant/server/lib/langchain/graphs/default_assistant_graph/nodes/execute_tools.ts @@ -38,7 +38,15 @@ export const executeTools = async ({ config, logger, state, tools }: ExecuteTool if (!agentAction || 'returnValues' in agentAction) { throw new Error('Agent has not been run yet'); } - const out = await toolExecutor.invoke(agentAction, config); + + let out; + try { + out = await toolExecutor.invoke(agentAction, config); + } catch (err) { + return { + steps: [{ action: agentAction, observation: JSON.stringify(`Error: ${err}`, null, 2) }], + }; + } return { steps: [{ action: agentAction, observation: JSON.stringify(out, null, 2) }], }; diff --git a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts index 990417b799234..a7380a3c1d2e0 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts @@ -203,6 +203,8 @@ export const postEvaluateRoute = ( llmType: 'openai', logger, request: skeletonRequest, + search: assistantContext.search, + dataViews: assistantContext.dataViews, traceOptions: { exampleId, projectName, diff --git a/x-pack/plugins/elastic_assistant/server/routes/helpers.ts b/x-pack/plugins/elastic_assistant/server/routes/helpers.ts index aa060e24bc5df..907c2594cdeaf 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/helpers.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/helpers.ts @@ -445,6 +445,8 @@ export const langChainExecute = async ({ replacements, responseLanguage, size: request.body.size, + search: assistantContext.search, + dataViews: assistantContext.dataViews, traceOptions: { projectName: request.body.langSmithProject, tracers: getLangSmithTracer({ diff --git a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts index 75b05dced83ec..9b648ae7403b8 100644 --- a/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts +++ b/x-pack/plugins/elastic_assistant/server/routes/request_context_factory.ts @@ -63,6 +63,14 @@ export class RequestContextFactory implements IRequestContextFactory { actions: startPlugins.actions, + search: startPlugins.data.search.asScoped(request), + + dataViews: await startPlugins.dataViews.dataViewsServiceFactory( + coreContext.savedObjects.client, + coreContext.elasticsearch.client.asCurrentUser, + request + ), + logger: this.logger, getServerBasePath: () => core.http.basePath.serverBasePath, diff --git a/x-pack/plugins/elastic_assistant/server/types.ts b/x-pack/plugins/elastic_assistant/server/types.ts index 10bd2a3fe62c6..bf9c33d63d6a7 100755 --- a/x-pack/plugins/elastic_assistant/server/types.ts +++ b/x-pack/plugins/elastic_assistant/server/types.ts @@ -39,12 +39,15 @@ import { ActionsClientLlm, ActionsClientSimpleChatModel, } from '@kbn/langchain/server'; +import { DataPluginStart } from '@kbn/data-plugin/server/plugin'; +import { DataViewsServerPluginStart, DataViewsService } from '@kbn/data-views-plugin/server'; import { AttackDiscoveryDataClient } from './ai_assistant_data_clients/attack_discovery'; import { AIAssistantConversationsDataClient } from './ai_assistant_data_clients/conversations'; import type { GetRegisteredFeatures, GetRegisteredTools } from './services/app_context'; import { AIAssistantDataClient } from './ai_assistant_data_clients'; import { AIAssistantKnowledgeBaseDataClient } from './ai_assistant_data_clients/knowledge_base'; +import { ElasticsearchStore } from './lib/langchain/elasticsearch_store/elasticsearch_store'; export const PLUGIN_ID = 'elasticAssistant' as const; @@ -99,6 +102,8 @@ export interface ElasticAssistantPluginSetupDependencies { } export interface ElasticAssistantPluginStartDependencies { actions: ActionsPluginStart; + data: DataPluginStart; + dataViews: DataViewsServerPluginStart; spaces?: SpacesPluginStart; security: SecurityServiceStart; } @@ -106,6 +111,8 @@ export interface ElasticAssistantPluginStartDependencies { export interface ElasticAssistantApiRequestHandlerContext { core: CoreRequestHandlerContext; actions: ActionsPluginStart; + search: ReturnType; + dataViews: DataViewsService; getRegisteredFeatures: GetRegisteredFeatures; getRegisteredTools: GetRegisteredTools; logger: Logger; @@ -219,6 +226,7 @@ export interface AssistantToolParams { isEnabledKnowledgeBase: boolean; chain?: RetrievalQAChain; esClient: ElasticsearchClient; + esStore?: ElasticsearchStore; kbDataClient?: AIAssistantKnowledgeBaseDataClient; langChainTimeout?: number; llm?: ActionsClientLlm | ActionsClientChatOpenAI | ActionsClientSimpleChatModel; @@ -232,4 +240,6 @@ export interface AssistantToolParams { ExecuteConnectorRequestBody | AttackDiscoveryPostRequestBody >; size?: number; + dataViews?: DataViewsService; + search?: ReturnType; } diff --git a/x-pack/plugins/elastic_assistant/tsconfig.json b/x-pack/plugins/elastic_assistant/tsconfig.json index 8f546d6e5fe01..8ef0682918123 100644 --- a/x-pack/plugins/elastic_assistant/tsconfig.json +++ b/x-pack/plugins/elastic_assistant/tsconfig.json @@ -45,6 +45,7 @@ "@kbn/core-saved-objects-api-server", "@kbn/langchain", "@kbn/stack-connectors-plugin", + "@kbn/data-views-plugin", "@kbn/security-plugin", ], "exclude": [ diff --git a/x-pack/plugins/search_playground/server/lib/conversational_chain.test.ts b/x-pack/plugins/search_playground/server/lib/conversational_chain.test.ts index fa6ee8e6e1abd..d1533247048f2 100644 --- a/x-pack/plugins/search_playground/server/lib/conversational_chain.test.ts +++ b/x-pack/plugins/search_playground/server/lib/conversational_chain.test.ts @@ -433,7 +433,10 @@ describe('conversational chain', () => { expectedDocs: [ { documents: [ - { metadata: { _id: '1', _index: 'index' } }, + { + metadata: { _id: '1', _index: 'index' }, + pageContent: '', + }, { metadata: { _id: '1', _index: 'website' }, pageContent: expect.any(String), @@ -444,8 +447,8 @@ describe('conversational chain', () => { ], // Even with body_content of 1000, the token count should be below the model limit of 100 expectedTokens: [ - { type: 'context_token_count', count: 70 }, - { type: 'prompt_token_count', count: 97 }, + { type: 'context_token_count', count: 73 }, + { type: 'prompt_token_count', count: 100 }, ], expectedHasClipped: true, expectedSearchRequest: [ diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 0a7558515226f..27b19dc4e91a6 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -120,12 +120,12 @@ export const allowedExperimentalValues = Object.freeze({ /** * Enables the Assistant Model Evaluation advanced setting and API endpoint, introduced in `8.11.0`. */ - assistantModelEvaluation: false, + assistantModelEvaluation: true, /** * Enables the Assistant Knowledge Base by default, introduced in `8.15.0`. */ - assistantKnowledgeBaseByDefault: false, + assistantKnowledgeBaseByDefault: true, /** * Enables the Managed User section inside the new user details flyout. @@ -258,6 +258,11 @@ export const allowedExperimentalValues = Object.freeze({ * Adds a new option to filter descendants of a process for Management / Event Filters */ filterProcessDescendantsForEventFiltersEnabled: false, + + /** + * Adds a new langgraph-based ESQL generation tool + */ + aiAssistantGraphEsqlTool: false, }); type ExperimentalConfigKeys = Array; diff --git a/x-pack/plugins/security_solution/kibana.jsonc b/x-pack/plugins/security_solution/kibana.jsonc index f682ca478a17f..d6baee91d019c 100644 --- a/x-pack/plugins/security_solution/kibana.jsonc +++ b/x-pack/plugins/security_solution/kibana.jsonc @@ -25,6 +25,7 @@ "ecsDataQualityDashboard", "elasticAssistant", "embeddable", + "esqlDataGrid", "eventLog", "features", "fieldFormats", diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/custom_codeblock/custom_code_block.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/custom_codeblock/custom_code_block.tsx index 90e406f500ba8..eac01423340f0 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/custom_codeblock/custom_code_block.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/custom_codeblock/custom_code_block.tsx @@ -8,7 +8,10 @@ import { EuiCodeBlock, EuiFlexGroup, EuiFlexItem, EuiPanel, useEuiTheme } from ' import { css } from '@emotion/css'; import React from 'react'; -export const CustomCodeBlock = ({ value }: { value: string }) => { +export interface CustomCodeBlockProps { + value: string; +} +export const CustomCodeBlock = ({ value }: CustomCodeBlockProps) => { const theme = useEuiTheme(); return ( @@ -31,7 +34,7 @@ export const CustomCodeBlock = ({ value }: { value: string }) => { > - + {value} diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/custom_codeblock/custom_codeblock_markdown_plugin.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/custom_codeblock/custom_codeblock_markdown_plugin.tsx index 19f566537a2b6..8825974f9c9c6 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/custom_codeblock/custom_codeblock_markdown_plugin.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/custom_codeblock/custom_codeblock_markdown_plugin.tsx @@ -6,7 +6,8 @@ */ import type { Node } from 'unist'; -import type { Parent } from 'mdast'; +import type { Parent, Content, PhrasingContent } from 'mdast'; +import { MARKDOWN_TYPES } from '@kbn/elastic-assistant/impl/assistant/use_conversation/helpers'; export const customCodeBlockLanguagePlugin = () => { const visitor = (node: Node, parent?: Parent) => { @@ -17,14 +18,32 @@ export const customCodeBlockLanguagePlugin = () => { }); } - if ( - node.type === 'code' && - (node.lang === 'eql' || - node.lang === 'esql' || - node.lang === 'kql' || - node.lang === 'dsl' || - node.lang === 'json') - ) { + if (node.type === 'code' && !node.lang) { + try { + const index = parent?.children.indexOf(node as Content); + + if (index) { + const previousText = (parent?.children[index - 1]?.children as PhrasingContent[]) + ?.map((child) => child.value) + .join(' '); + for (const [typeKey, keywords] of Object.entries(MARKDOWN_TYPES)) { + if (keywords.some((kw) => previousText.toLowerCase().includes(kw.toLowerCase()))) { + node.lang = typeKey; + break; + } + } + } + } catch (e) { + /* empty */ + } + } + + if (node.type === 'code' && node.lang === 'esql') { + node.type = 'esql'; + return; + } + + if (node.type === 'code' && ['eql', 'kql', 'dsl', 'json'].includes(node.lang as string)) { node.type = 'customCodeBlock'; } }; diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/custom_codeblock/esql_code_block.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/custom_codeblock/esql_code_block.tsx new file mode 100644 index 0000000000000..f8623cdba3af0 --- /dev/null +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/custom_codeblock/esql_code_block.tsx @@ -0,0 +1,336 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { + EuiButtonEmpty, + EuiButtonIcon, + EuiCallOut, + EuiCodeBlock, + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiToolTip, + useEuiTheme, +} from '@elastic/eui'; +import { css } from '@emotion/css'; +import { i18n } from '@kbn/i18n'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { useQuery } from '@tanstack/react-query'; +import { v4 as uuidv4 } from 'uuid'; +import { + getIndexPatternFromESQLQuery, + getESQLQueryColumns, + getESQLAdHocDataview, + getESQLResults, +} from '@kbn/esql-utils'; +import useAsync from 'react-use/lib/useAsync'; +import { ESQLDataGrid } from '@kbn/esql-datagrid/public'; +import type { TypedLensByValueInput } from '@kbn/lens-plugin/public'; +import { getLensAttributesFromSuggestion } from '@kbn/visualization-utils'; +import { useKibana } from '../../../common/lib/kibana'; + +function generateId() { + return uuidv4(); +} + +const saveVisualizationLabel = i18n.translate('xpack.securitySolution.lensESQLFunction.save', { + defaultMessage: 'Save visualization', +}); + +export interface EsqlCodeBlockProps { + value: string; + timestamp: string; + actionsDisabled: boolean; +} + +const EsqlCodeBlockComponent = ({ value, actionsDisabled, timestamp }: EsqlCodeBlockProps) => { + const theme = useEuiTheme(); + const { lens, dataViews: dataViewService, data } = useKibana().services; + const [showVisualization, setShowVisualization] = useState(false); + + const { value: lensHelpersAsync } = useAsync(() => { + return lens.stateHelperApi(); + }, [lens]); + + const { data: queryResults, error: queryResultsError } = useQuery({ + queryKey: ['getESQLResults', value, timestamp], + enabled: showVisualization, + queryFn: async () => { + return getESQLResults({ + esqlQuery: value, + search: data.search.search, + filter: { + range: timestamp + ? { + '@timestamp': { + lte: timestamp, + format: 'strict_date_optional_time', + }, + } + : {}, + }, + }); + }, + select: (queryResultsData) => ({ + params: queryResultsData.params, + rows: queryResultsData.response.values, + columns: queryResultsData.response.columns, + }), + refetchOnWindowFocus: false, + keepPreviousData: true, + }); + + const indexPattern = useMemo(() => getIndexPatternFromESQLQuery(value), [value]); + const { value: formattedColumns, error: formattedColumnsError } = useAsync( + () => + getESQLQueryColumns({ + esqlQuery: value, + search: data.search.search, + }), + [value] + ); + + const { value: dataViewAsync, error: dataViewError } = useAsync(() => { + return getESQLAdHocDataview(indexPattern, dataViewService).then((dataView) => { + if (dataView.fields.getByName('@timestamp')?.type === 'date') { + dataView.timeFieldName = '@timestamp'; + } + return dataView; + }); + }, [indexPattern, dataViewService]); + + const [isSaveModalOpen, setIsSaveModalOpen] = useState(false); + const [isTableVisible, setIsTableVisible] = useState(false); + const [lensInput, setLensInput] = useState(undefined); + + const preferredChartType = undefined; // 'XY'; + + // initialization + useEffect(() => { + if (lensHelpersAsync && dataViewAsync && !lensInput && formattedColumns) { + const context = { + dataViewSpec: dataViewAsync?.toSpec(), + fieldName: '', + textBasedColumns: formattedColumns, + query: { + esql: value, + }, + }; + + const chartSuggestions = lensHelpersAsync.suggestions( + context, + dataViewAsync, + [], + preferredChartType + ); + + if (chartSuggestions?.length) { + const [suggestion] = chartSuggestions; + + const attrs = getLensAttributesFromSuggestion({ + filters: [], + query: { + esql: value, + }, + suggestion, + dataView: dataViewAsync, + }) as TypedLensByValueInput['attributes']; + + const lensEmbeddableInput = { + attributes: attrs, + id: generateId(), + }; + setLensInput(lensEmbeddableInput); + } + } + }, [dataViewAsync, formattedColumns, lensHelpersAsync, lensInput, preferredChartType, value]); + + // if the Lens suggestions api suggests a table then we want to render a Discover table instead + const isLensInputTable = lensInput?.attributes?.visualizationType === 'lnsDatatable'; + + const handleShowVisualization = useCallback(() => setShowVisualization(true), []); + + const handleToogleTableView = useCallback(() => setIsTableVisible((prev) => !prev), []); + + const handleCloseSaveModal = useCallback(() => setIsSaveModalOpen(false), []); + + return ( + <> + + + + + {value} + + + + {!showVisualization && ( + + + + + {i18n.translate('xpack.securitySolution.lensESQLFunction.visualizeThisQuery', { + defaultMessage: 'Generate Visualization', + })} + + + + + )} + + + {showVisualization && (queryResultsError || formattedColumnsError || dataViewError) && ( + + +

{`${queryResultsError || formattedColumnsError || dataViewError}`}

+ + + )} + {showVisualization && queryResults && formattedColumns && ( + + + {!isLensInputTable && ( + <> + + + + + + + + + + setIsSaveModalOpen(true)} + data-test-subj="securityAiAssistantLensESQLSaveButton" + aria-label={saveVisualizationLabel} + /> + + + + + + {isTableVisible ? ( + + ) : lensInput ? ( + + ) : null} + + + )} + {/* hide the grid in case of errors (as the user can't fix them) */} + {isLensInputTable && ( + + + + )} + + + )} + + {isSaveModalOpen ? ( + + ) : null} + + ); +}; + +export const EsqlCodeBlock = React.memo(EsqlCodeBlockComponent); diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx index 10d5a15c800ae..46b23998928c2 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/index.tsx @@ -193,6 +193,7 @@ export const getComments = ({ actionTypeId={actionTypeId} abortStream={abortStream} content={transformedMessage.content} + timestamp={transformedMessage.timestamp} index={index} isControlsEnabled={isControlsEnabled} isEnabledLangChain={isEnabledLangChain} diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx index 631d8b507ceed..367119c92486c 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/index.tsx @@ -17,6 +17,7 @@ import { MessageText } from './message_text'; interface Props { abortStream: () => void; content?: string; + timestamp?: string; isEnabledLangChain: boolean; isError?: boolean; isFetching?: boolean; @@ -33,6 +34,7 @@ interface Props { export const StreamComment = ({ abortStream, content, + timestamp, actionTypeId, index, isControlsEnabled = false, @@ -108,7 +110,14 @@ export const StreamComment = ({ return ( } + body={ + + } error={error ? new Error(error) : undefined} controls={controls} /> diff --git a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/message_text.tsx b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/message_text.tsx index c40b0c04043ad..2e93a32cca9e5 100644 --- a/x-pack/plugins/security_solution/public/assistant/get_comments/stream/message_text.tsx +++ b/x-pack/plugins/security_solution/public/assistant/get_comments/stream/message_text.tsx @@ -24,10 +24,14 @@ import { euiThemeVars } from '@kbn/ui-theme'; import type { Node } from 'unist'; import { customCodeBlockLanguagePlugin } from '../custom_codeblock/custom_codeblock_markdown_plugin'; +import type { CustomCodeBlockProps } from '../custom_codeblock/custom_code_block'; import { CustomCodeBlock } from '../custom_codeblock/custom_code_block'; +import type { EsqlCodeBlockProps } from '../custom_codeblock/esql_code_block'; +import { EsqlCodeBlock } from '../custom_codeblock/esql_code_block'; interface Props { content: string; + timestamp?: string; index: number; loading: boolean; } @@ -98,7 +102,21 @@ const loadingCursorPlugin = () => { }; }; -const getPluginDependencies = () => { +export const Esql = ({ timestamp, ...props }: EsqlCodeBlockProps) => ( + <> + + + +); + +export const CodeBlock = (props: CustomCodeBlockProps) => ( + <> + + + +); + +const getPluginDependencies = (timestamp?: string) => { const parsingPlugins = getDefaultEuiMarkdownParsingPlugins(); const processingPlugins = getDefaultEuiMarkdownProcessingPlugins(); @@ -108,14 +126,8 @@ const getPluginDependencies = () => { processingPlugins[1][1].components = { ...components, cursor: Cursor, - customCodeBlock: (props) => { - return ( - <> - - - - ); - }, + esql: (props) => , + customCodeBlock: CodeBlock, table: (props) => ( <> @@ -143,12 +155,12 @@ const getPluginDependencies = () => { }; }; -export function MessageText({ loading, content, index }: Props) { +export const MessageText = React.memo(({ loading, content, timestamp, index }: Props) => { const containerClassName = css` overflow-wrap: anywhere; `; - const { parsingPluginList, processingPluginList } = getPluginDependencies(); + const { parsingPluginList, processingPluginList } = getPluginDependencies(timestamp); return ( @@ -164,4 +176,5 @@ export function MessageText({ loading, content, index }: Props) { ); -} +}); +MessageText.displayName = 'MessageText'; diff --git a/x-pack/plugins/security_solution/public/assistant/helpers.tsx b/x-pack/plugins/security_solution/public/assistant/helpers.tsx index 61856ea1b17f3..32199234f5d89 100644 --- a/x-pack/plugins/security_solution/public/assistant/helpers.tsx +++ b/x-pack/plugins/security_solution/public/assistant/helpers.tsx @@ -65,14 +65,15 @@ export const augmentMessageCodeBlocks = ( currentConversation: Conversation, showAnonymizedValues: boolean ): CodeBlockDetails[][] => { - const cbd = currentConversation.messages.map(({ content }) => + const cbd = currentConversation.messages.map(({ content, timestamp }) => analyzeMarkdown( showAnonymizedValues ? content ?? '' : replaceAnonymizedValuesWithOriginalValues({ messageContent: content ?? '', replacements: currentConversation.replacements, - }) + }), + timestamp ) ); diff --git a/x-pack/plugins/security_solution/public/cases/pages/index.tsx b/x-pack/plugins/security_solution/public/cases/pages/index.tsx index 54ece0123738f..91b260e8940d9 100644 --- a/x-pack/plugins/security_solution/public/cases/pages/index.tsx +++ b/x-pack/plugins/security_solution/public/cases/pages/index.tsx @@ -35,6 +35,8 @@ import * as timelineMarkdownPlugin from '../../common/components/markdown_editor import { DetailsPanel } from '../../timelines/components/side_panel'; import { useFetchAlertData } from './use_fetch_alert_data'; import { useUpsellingMessage } from '../../common/hooks/use_upselling'; +import { Esql, CodeBlock } from '../../assistant/get_comments/stream/message_text'; +import { customCodeBlockLanguagePlugin } from '../../assistant/get_comments/custom_codeblock/custom_codeblock_markdown_plugin'; const TimelineDetailsPanel = () => { const { browserFields, runtimeMappings } = useSourcererDataView(SourcererScopeName.detections); @@ -183,8 +185,15 @@ const CaseContainerComponent: React.FC = () => { showAlertDetails, timelineIntegration: { editor_plugins: { - parsingPlugin: timelineMarkdownPlugin.parser, - processingPluginRenderer: timelineMarkdownPlugin.renderer, + parsingPlugins: { + timeline: timelineMarkdownPlugin.parser, + customCodeBlock: customCodeBlockLanguagePlugin, + }, + processingPluginRenderer: { + timeline: timelineMarkdownPlugin.renderer, + esql: Esql, + customCodeBlock: CodeBlock, + }, uiPlugin: timelineMarkdownPlugin.plugin({ interactionsUpsellingMessage }), }, hooks: { diff --git a/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/correct_common_esql_mistakes.test.ts b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/correct_common_esql_mistakes.test.ts new file mode 100644 index 0000000000000..40dc79851c052 --- /dev/null +++ b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/correct_common_esql_mistakes.test.ts @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// Origin: x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.test.ts + +import { correctCommonEsqlMistakes } from './correct_common_esql_mistakes'; + +describe('correctCommonEsqlMistakes', () => { + function normalize(input: string) { + return input.replaceAll(/[\t|\s]*\n[\t|\s]*/gms, '\n'); + } + + function expectQuery(input: string, expectedOutput: string) { + expect(normalize(correctCommonEsqlMistakes(input).output)).toEqual(normalize(expectedOutput)); + } + + it('replaces aliasing via the AS keyword with the = operator', () => { + expectQuery(`FROM logs-* | STATS COUNT() AS count`, 'FROM logs-*\n| STATS count = COUNT()'); + + expectQuery(`FROM logs-* | STATS COUNT() as count`, 'FROM logs-*\n| STATS count = COUNT()'); + + expectQuery( + `FROM logs-* | STATS AVG(transaction.duration.histogram) AS avg_request_latency, PERCENTILE(transaction.duration.histogram, 95) AS p95`, + `FROM logs-* + | STATS avg_request_latency = AVG(transaction.duration.histogram), p95 = PERCENTILE(transaction.duration.histogram, 95)` + ); + + expectQuery( + `FROM traces-apm* + | WHERE @timestamp >= NOW() - 24 hours + | STATS AVG(transaction.duration.us) AS avg_duration, SUM(success) AS total_successes, COUNT(*) AS total_requests BY service.name`, + `FROM traces-apm* + | WHERE @timestamp >= NOW() - 24 hours + | STATS avg_duration = AVG(transaction.duration.us), total_successes = SUM(success), total_requests = COUNT(*) BY service.name` + ); + }); + + it(`replaces " or ' escaping in FROM statements with backticks`, () => { + expectQuery(`FROM "logs-*" | LIMIT 10`, 'FROM logs-*\n| LIMIT 10'); + expectQuery(`FROM 'logs-*' | LIMIT 10`, 'FROM logs-*\n| LIMIT 10'); + expectQuery(`FROM logs-* | LIMIT 10`, 'FROM logs-*\n| LIMIT 10'); + }); + + it('replaces = as equal operator with ==', () => { + expectQuery( + `FROM logs-*\n| WHERE service.name = "foo"`, + `FROM logs-*\n| WHERE service.name == "foo"` + ); + + expectQuery( + `FROM logs-*\n| WHERE service.name = "foo" AND service.environment = "bar"`, + `FROM logs-*\n| WHERE service.name == "foo" AND service.environment == "bar"` + ); + + expectQuery( + `FROM logs-*\n| WHERE (service.name = "foo" AND service.environment = "bar") OR agent.name = "baz"`, + `FROM logs-*\n| WHERE (service.name == "foo" AND service.environment == "bar") OR agent.name == "baz"` + ); + + expectQuery( + `FROM logs-*\n| WHERE \`what=ever\` = "foo=bar"`, + `FROM logs-*\n| WHERE \`what=ever\` == "foo=bar"` + ); + }); + + it('replaces single-quote escaped strings with double-quote escaped strings', () => { + expectQuery( + `FROM nyc_taxis + | WHERE DATE_EXTRACT('hour', dropoff_datetime) >= 6 AND DATE_EXTRACT('hour', dropoff_datetime) < 10 + | LIMIT 10`, + `FROM nyc_taxis + | WHERE DATE_EXTRACT("hour", dropoff_datetime) >= 6 AND DATE_EXTRACT("hour", dropoff_datetime) < 10 + | LIMIT 10` + ); + expectQuery( + `FROM nyc_taxis + | WHERE DATE_EXTRACT('hour', "hh:mm a, 'of' d MMMM yyyy") >= 6 AND DATE_EXTRACT('hour', dropoff_datetime) < 10 + | LIMIT 10`, + `FROM nyc_taxis + | WHERE DATE_EXTRACT("hour", "hh:mm a, 'of' d MMMM yyyy") >= 6 AND DATE_EXTRACT("hour", dropoff_datetime) < 10 + | LIMIT 10` + ); + }); + + it(`verifies if the SORT key is in KEEP, and if it's not, it will include it`, () => { + expectQuery( + 'FROM logs-* \n| KEEP date \n| SORT @timestamp DESC', + 'FROM logs-*\n| KEEP date, @timestamp\n| SORT @timestamp DESC' + ); + + expectQuery( + `FROM logs-* | KEEP date, whatever | EVAL my_truncated_date_field = DATE_TRUNC(1 year, date) | SORT @timestamp, my_truncated_date_field DESC`, + 'FROM logs-*\n| KEEP date, whatever, @timestamp\n| EVAL my_truncated_date_field = DATE_TRUNC(1 year, date)\n| SORT @timestamp, my_truncated_date_field DESC' + ); + + expectQuery( + `FROM logs-* | KEEP date, whatever | RENAME whatever AS forever | SORT forever DESC`, + `FROM logs-*\n| KEEP date, whatever\n| RENAME whatever AS forever\n| SORT forever DESC` + ); + + expectQuery( + 'FROM employees\n| KEEP first_name, last_name\n| RENAME first_name AS fn, last_name AS ln', + 'FROM employees\n| KEEP first_name, last_name\n| RENAME first_name AS fn, last_name AS ln' + ); + }); + + it(`escapes the column name if SORT uses an expression`, () => { + expectQuery( + 'FROM logs-* \n| STATS COUNT(*) by service.name\n| SORT COUNT(*) DESC', + 'FROM logs-*\n| STATS COUNT(*) BY service.name\n| SORT `COUNT(*)` DESC' + ); + + expectQuery( + 'FROM logs-* \n| STATS COUNT(*) by service.name\n| SORT COUNT(*) DESC, @timestamp ASC', + 'FROM logs-*\n| STATS COUNT(*) BY service.name\n| SORT `COUNT(*)` DESC, @timestamp ASC' + ); + + expectQuery( + `FROM employees\n| KEEP first_name, last_name, height\n| SORT first_name ASC NULLS FIRST`, + `FROM employees\n| KEEP first_name, last_name, height\n| SORT first_name ASC NULLS FIRST` + ); + + expectQuery( + `FROM employees + | STATS my_count = COUNT() BY LEFT(last_name, 1) + | SORT \`LEFT(last_name, 1)\``, + `FROM employees + | STATS my_count = COUNT() BY LEFT(last_name, 1) + | SORT \`LEFT(last_name, 1)\`` + ); + }); + + it(`handles complicated queries correctly`, () => { + expectQuery( + `FROM "postgres-logs*" + | GROK message "%{TIMESTAMP_ISO8601:timestamp} %{TZ} \[%{NUMBER:process_id}\]: \[%{NUMBER:log_line}\] user=%{USER:user},db=%{USER:database},app=\[%{DATA:application}\],client=%{IP:client_ip} LOG: duration: %{NUMBER:duration:float} ms statement: %{GREEDYDATA:statement}" + | EVAL "@timestamp" = TO_DATETIME(timestamp) + | WHERE statement LIKE 'SELECT%' + | STATS avg_duration = AVG(duration)`, + `FROM postgres-logs* + | GROK message "%{TIMESTAMP_ISO8601:timestamp} %{TZ} \[%{NUMBER:process_id}\]: \[%{NUMBER:log_line}\] user=%{USER:user},db=%{USER:database},app=\[%{DATA:application}\],client=%{IP:client_ip} LOG: duration: %{NUMBER:duration:float} ms statement: %{GREEDYDATA:statement}" + | EVAL @timestamp = TO_DATETIME(timestamp) + | WHERE statement LIKE "SELECT%" + | STATS avg_duration = AVG(duration)` + ); + + expectQuery( + `FROM metrics-apm* + | WHERE metricset.name == "service_destination" AND @timestamp > NOW() - 24 hours + | EVAL total_events = span.destination.service.response_time.count + | EVAL total_latency = span.destination.service.response_time.sum.us + | EVAL is_failure = CASE(event.outcome == "failure", 1, 0) + | STATS + avg_throughput = AVG(total_events), + avg_latency_per_request = AVG(total_latency / total_events), + failure_rate = AVG(is_failure) + BY span.destination.service.resource`, + `FROM metrics-apm* + | WHERE metricset.name == "service_destination" AND @timestamp > NOW() - 24 hours + | EVAL total_events = span.destination.service.response_time.count + | EVAL total_latency = span.destination.service.response_time.sum.us + | EVAL is_failure = CASE(event.outcome == "failure", 1, 0) + | STATS avg_throughput = AVG(total_events), avg_latency_per_request = AVG(total_latency / total_events), failure_rate = AVG(is_failure) BY span.destination.service.resource` + ); + + expectQuery( + `FROM sample_data + | EVAL successful = CASE( + STARTS_WITH(message, "Connected to"), 1, + message == "Connection error", 0 + ) + | STATS success_rate = AVG(successful)`, + `FROM sample_data + | EVAL successful = CASE( + STARTS_WITH(message, "Connected to"), 1, + message == "Connection error", 0 + ) + | STATS success_rate = AVG(successful)` + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/correct_common_esql_mistakes.ts b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/correct_common_esql_mistakes.ts new file mode 100644 index 0000000000000..957338203479b --- /dev/null +++ b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/correct_common_esql_mistakes.ts @@ -0,0 +1,291 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// Origin: x-pack/plugins/observability_solution/observability_ai_assistant_app/server/functions/query/correct_common_esql_mistakes.ts + +const STRING_DELIMITER_TOKENS = ['`', "'", '"']; +const ESCAPE_TOKEN = '\\\\'; + +// this function splits statements by a certain token, +// and takes into account escaping, or function calls + +function split(value: string, splitToken: string) { + const statements: string[] = []; + + let delimiterToken: string | undefined; + + let groupingCount: number = 0; + + let currentStatement: string = ''; + + const trimmed = value.trim().split(''); + + for (let index = 0; index < trimmed.length; index++) { + const char = trimmed[index]; + + if ( + !delimiterToken && + groupingCount === 0 && + trimmed + .slice(index, index + splitToken.length) + .join('') + .toLowerCase() === splitToken.toLowerCase() + ) { + index += splitToken.length - 1; + statements.push(currentStatement.trim()); + currentStatement = ''; + // eslint-disable-next-line no-continue + continue; + } + + currentStatement += char; + + if (delimiterToken === char) { + // end identifier + delimiterToken = undefined; + } else if (!delimiterToken && trimmed[index - 1] !== ESCAPE_TOKEN) { + const applicableToken = STRING_DELIMITER_TOKENS.includes(char) ? char : undefined; + + if (applicableToken) { + // start identifier + delimiterToken = applicableToken; + // eslint-disable-next-line no-continue + continue; + } else if (char === '(') { + groupingCount++; + } else if (char === ')') { + groupingCount--; + } + } + } + + if (currentStatement) { + statements.push(currentStatement.trim()); + } + + return statements; +} + +export function splitIntoCommands(query: string) { + const commands: string[] = split(query, '|'); + + return commands.map((command) => { + const commandName = command.match(/^([A-Za-z]+)/)?.[1]; + + return { + name: commandName, + command, + }; + }); +} + +function replaceSingleQuotesWithDoubleQuotes(command: string) { + const regex = /'(?=(?:[^"]*"[^"]*")*[^"]*$)/g; + return command.replace(regex, '"'); +} + +function removeColumnQuotesAndEscape(column: string) { + const plainColumnIdentifier = column.replaceAll(/^"(.*)"$/g, `$1`).replaceAll(/^'(.*)'$/g, `$1`); + + if (isValidColumnName(plainColumnIdentifier)) { + return plainColumnIdentifier; + } + + return `\`${plainColumnIdentifier}\``; +} + +function replaceAsKeywordWithAssignments(command: string) { + return command.replaceAll(/^STATS\s*(.*)/g, (__, statsOperations: string) => { + return `STATS ${statsOperations.replaceAll( + /(,\s*)?(.*?)\s(AS|as)\s([`a-zA-Z0-9.\-_]+)/g, + '$1$4 = $2' + )}`; + }); +} + +function isValidColumnName(column: string) { + return Boolean(column.match(/^`.*`$/) || column.match(/^[@A-Za-z\._\-\d]+$/)); +} + +function escapeColumns(line: string) { + const [, command, body] = line.match(/^([A-Za-z_]+)(.*)$/ms) ?? ['', '', '']; + + const escapedBody = split(body.trim(), ',') + .map((statement) => { + const [lhs, rhs] = split(statement, '='); + if (!rhs) { + return lhs; + } + return `${removeColumnQuotesAndEscape(lhs)} = ${rhs}`; + }) + .join(', '); + + return `${command} ${escapedBody}`; +} + +function verifyKeepColumns( + keepCommand: string, + nextCommands: Array<{ name?: string; command: string }> +) { + const columnsInKeep = split(keepCommand.replace(/^KEEP\s*/, ''), ',').map((statement) => + split(statement, '=')?.[0].trim() + ); + + const availableColumns = columnsInKeep.concat(); + + for (const { name, command } of nextCommands) { + if (['STATS', 'KEEP', 'DROP', 'DISSECT', 'GROK', 'ENRICH', 'RENAME'].includes(name || '')) { + // these operations alter columns in a way that is hard to analyze, so we abort + break; + } + + const commandBody = command.replace(/^[A-Za-z]+\s*/, ''); + + if (name === 'EVAL') { + // EVAL creates new columns, make them available + const columnsInEval = split(commandBody, ',').map((column) => + split(column.trim(), '=')[0].trim() + ); + + columnsInEval.forEach((column) => { + availableColumns.push(column); + }); + } + + if (name === 'RENAME') { + // RENAME creates and removes columns + split(commandBody, ',').forEach((statement) => { + const [prevName, newName] = split(statement, 'AS').map((side) => side.trim()); + availableColumns.push(newName); + if (!availableColumns.includes(prevName)) { + columnsInKeep.push(prevName); + } + }); + } + + if (name === 'SORT') { + const columnsInSort = split(commandBody, ',').map((column) => + split(column.trim(), ' ')[0].trim() + ); + + columnsInSort.forEach((column) => { + if (isValidColumnName(column) && !availableColumns.includes(column)) { + columnsInKeep.push(column); + } + }); + } + } + + return `KEEP ${columnsInKeep.join(', ')}`; +} + +function escapeExpressionsInSort(sortCommand: string) { + const columnsInSort = split(sortCommand.replace(/^SORT\s*/, ''), ',') + .map((statement) => split(statement, '=')?.[0].trim()) + .map((columnAndSortOrder) => { + let [, column, sortOrder = ''] = + columnAndSortOrder.match(/^(.*?)\s+(ASC|DESC\s*([A-Z\s]+)?)?$/i) || []; + if (!column) { + return columnAndSortOrder; + } + + if (sortOrder) sortOrder = ` ${sortOrder}`; + + if (!column.match(/^`?[a-zA-Z0-9_\.@]+`?$/)) { + column = `\`${column}\``; + } + + return `${column}${sortOrder}`; + }); + + return `SORT ${columnsInSort.join(', ')}`; +} + +function ensureEqualityOperators(whereCommand: string) { + const body = whereCommand.split(/^WHERE /)[1]; + + const byChar = body.split(''); + + let next = ''; + let isColumnName = false; + byChar.forEach((char, index) => { + next += char; + + if (!isColumnName && char === '=' && byChar[index - 1] === ' ' && byChar[index + 1] === ' ') { + next += '='; + } + + if (!isColumnName && (char === '`' || char.match(/[a-z@]/i))) { + isColumnName = true; + } else if (isColumnName && (char === '`' || !char.match(/[a-z@0-9]/i))) { + isColumnName = false; + } + }); + + return `WHERE ${next}`; +} + +export function correctCommonEsqlMistakes(query: string): { + isCorrection: boolean; + input: string; + output: string; +} { + const commands = splitIntoCommands(query.trim()); + + const formattedCommands: string[] = commands.map(({ name, command }, index) => { + let formattedCommand = command; + + switch (name) { + case 'FROM': + formattedCommand = formattedCommand + .replaceAll(/FROM "(.*)"/g, 'FROM $1') + .replaceAll(/FROM '(.*)'/g, 'FROM $1') + .replaceAll(/FROM `(.*)`/g, 'FROM $1'); + break; + + case 'WHERE': + formattedCommand = replaceSingleQuotesWithDoubleQuotes(formattedCommand); + formattedCommand = ensureEqualityOperators(formattedCommand); + break; + + case 'EVAL': + formattedCommand = replaceSingleQuotesWithDoubleQuotes(formattedCommand); + formattedCommand = escapeColumns(formattedCommand); + break; + + case 'STATS': + formattedCommand = replaceAsKeywordWithAssignments(formattedCommand); + const [before, after] = split(formattedCommand, ' BY '); + formattedCommand = escapeColumns(before); + if (after) { + formattedCommand += ` BY ${after}`; + } + break; + + case 'KEEP': + formattedCommand = verifyKeepColumns(formattedCommand, commands.slice(index + 1)); + break; + + case 'SORT': + formattedCommand = escapeExpressionsInSort(formattedCommand); + break; + } + return formattedCommand; + }); + + const output = formattedCommands.join('\n| '); + + const originalFormattedQuery = commands.map((cmd) => cmd.command).join('\n| '); + + const isCorrection = output !== originalFormattedQuery; + + return { + input: query, + output, + isCorrection, + }; +} diff --git a/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts index 29b10e9fb0275..7907d834ce2f3 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.test.ts @@ -12,6 +12,7 @@ import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import type { KibanaRequest } from '@kbn/core-http-server'; import type { ExecuteConnectorRequestBody } from '@kbn/elastic-assistant-common/impl/schemas/actions_connector/post_actions_connector_execute_route.gen'; import { loggerMock } from '@kbn/logging-mocks'; +import { createSearchRequestHandlerContext } from '@kbn/data-plugin/server/search/mocks'; describe('EsqlLanguageKnowledgeBaseTool', () => { const chain = {} as RetrievalQAChain; @@ -28,12 +29,14 @@ describe('EsqlLanguageKnowledgeBaseTool', () => { size: 20, }, } as unknown as KibanaRequest; + const search = createSearchRequestHandlerContext(); const logger = loggerMock.create(); const rest = { chain, esClient, logger, request, + search, }; describe('isSupported', () => { diff --git a/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.ts b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.ts index 692753a22dea0..e5448280c5117 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/esql_language_knowledge_base_tool.ts @@ -9,6 +9,7 @@ import { DynamicStructuredTool } from '@langchain/core/tools'; import { z } from 'zod'; import type { AssistantTool, AssistantToolParams } from '@kbn/elastic-assistant-plugin/server'; import { APP_UI_ID } from '../../../../common'; +import type { LangchainZodAny } from '..'; export type EsqlKnowledgeBaseToolParams = AssistantToolParams; @@ -36,7 +37,7 @@ export const ESQL_KNOWLEDGE_BASE_TOOL: AssistantTool = { description: toolDetails.description, schema: z.object({ question: z.string().describe(`The user's exact question about ESQL`), - }), + }) as unknown as LangchainZodAny, func: async (input, _, cbManager) => { const result = await chain.invoke( { diff --git a/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/graph_esql_language_tool.ts b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/graph_esql_language_tool.ts new file mode 100644 index 0000000000000..bceddef76e8a1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/assistant/tools/esql_language_knowledge_base/graph_esql_language_tool.ts @@ -0,0 +1,320 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEmpty, map, without } from 'lodash'; +import { DynamicStructuredTool } from '@langchain/core/tools'; +import { z } from 'zod'; +import type { AssistantTool, AssistantToolParams } from '@kbn/elastic-assistant-plugin/server'; +import { StringOutputParser } from '@langchain/core/output_parsers'; +import type { StateGraphArgs } from '@langchain/langgraph'; +import { END, START, StateGraph } from '@langchain/langgraph'; +import type { BaseMessage } from '@langchain/core/messages'; +import type { RunnableConfig } from '@langchain/core/runnables'; +import { getESQLQueryColumns, getIndexPatternFromESQLQuery } from '@kbn/esql-utils'; +import { ChatPromptTemplate } from '@langchain/core/prompts'; +import { validateQuery } from '@kbn/esql-validation-autocomplete'; +import type { EditorError, ESQLMessage } from '@kbn/esql-ast'; +import { getAstAndSyntaxErrors } from '@kbn/esql-ast'; +import type { + ActionsClientChatOpenAI, + ActionsClientLlm, + ActionsClientSimpleChatModel, +} from '@kbn/langchain/server'; +import type { ElasticsearchStore } from '@kbn/elastic-assistant-plugin/server/lib/langchain/elasticsearch_store/elasticsearch_store'; +import { APP_UI_ID } from '../../../../common'; +import { correctCommonEsqlMistakes } from './correct_common_esql_mistakes'; +import type { LangchainZodAny } from '..'; + +export const INLINE_ESQL_QUERY_REGEX = /```esql\s*(.*?)\s*```/gms; + +export type GraphESQLToolParams = AssistantToolParams; + +const TOOL_NAME = 'GraphESQLTool'; + +const toolDetails = { + id: 'graph-esql-tool', + name: TOOL_NAME, + description: `You MUST use the "${TOOL_NAME}" function when the user wants to: + - visualize data + - run any arbitrary query + - breakdown or filter ES|QL queries that are displayed on the current page + - convert queries from another language to ES|QL + - asks general questions about ES|QL + + DO NOT UNDER ANY CIRCUMSTANCES generate ES|QL queries or explain anything about the ES|QL query language yourself. + DO NOT UNDER ANY CIRCUMSTANCES try to correct an ES|QL query yourself - always use the "${TOOL_NAME}" function for this. + + If the user asks for a query, and one of the dataset info functions was called and returned no results, you should still call the query function to generate an example query. + + Even if the "${TOOL_NAME}" function was used before that, follow it up with the "${TOOL_NAME}" function. If a query fails, do not attempt to correct it yourself. Again you should call the "${TOOL_NAME}" function, + even if it has been called before.`, +}; + +interface IState { + messages: BaseMessage[]; + esqlQuery: string; + errors: string[]; + availableDataViews: string[]; + dataViewFields: string[]; + invalidQueries: string[]; +} + +// This defines the agent state +const graphState: StateGraphArgs['channels'] = { + messages: { + value: (x: BaseMessage[], y: BaseMessage[]) => x.concat(y), + default: () => [], + }, + esqlQuery: { + value: (x: string, y?: string) => y ?? x, + default: () => '', + }, + errors: { + value: (x, y) => (y && !y.length ? y : x.concat(y)), + default: () => [], + }, + invalidQueries: { + value: (x, y) => x.concat(y), + default: () => [], + }, + availableDataViews: { + value: (x, y) => y ?? x, + default: () => [], + }, + dataViewFields: { + value: (x, y) => y ?? x, + default: () => [], + }, +}; + +const getDataStreams = + ({ dataViews }: { dataViews: AssistantToolParams['dataViews'] }) => + async () => { + const allDataStreams = await dataViews?.getTitles(); + + return { + availableDataViews: allDataStreams, + }; + }; + +const getGenerateQuery = + ({ + userQuery, + llm, + esStore, + }: { + userQuery: string; + llm: ActionsClientLlm | ActionsClientChatOpenAI | ActionsClientSimpleChatModel; + esStore?: ElasticsearchStore; + }) => + async (state: IState) => { + const knowledgeBaseDocs = (await esStore?.similaritySearch(userQuery)) ?? []; + const documentation = map(knowledgeBaseDocs, 'pageContent').join('\n'); + + const answerPrompt = await ChatPromptTemplate.fromMessages([ + [ + 'system', + `As an expert user of Elastic Security, please generate an accurate and valid ESQL query to detect the use case below. Your response should be formatted to be able to use immediately in an Elastic Security timeline or detection rule. Take your time with the answer, check your knowledge really well on all the functions I am asking for. For ES|QL answers specifically, you should only ever answer with what's available in your private knowledge. I cannot afford for queries to be inaccurate. Assume I am using the Elastic Common Schema and Elastic Agent. Under any circumstances wrap index in quotes. + + If multiple indices are matched please try to use wildcard to match all indices. If you are unsure about the index name, please refer to the context provided below. + + ES|QL documentation: + {documentation} + + Available indices: + {availableDataViews} + `, + ], + [ + 'user', + `Answer the user's question that was previously asked ("{query}..."). Take into account any previous errors: +{errors} + +and invalid ES|QL queries: +{invalidQueries} + +If errors were related to "Unknown column" make sure to check the Available index fields: +{dataViewFields} + +Format any ES|QL query as follows: + +\`\`\`esql + +\`\`\` + + +Respond in plain text. Do not attempt to use a function. + +DO NOT UNDER ANY CIRCUMSTANCES generate more than a single query. +If multiple queries are needed, do it as a follow-up step. Make this clear to the user. For example: + +Human: plot both yesterday's and today's data. + +Assistant: Here's how you can plot yesterday's data: + +\`\`\`esql + +\`\`\` + + +Let's see that first. We'll look at today's data next. + +Human: + +Assistant: Let's look at today's data: + + +\`\`\`esql + +\`\`\` + + +DO NOT UNDER ANY CIRCUMSTANCES use commands or functions that are not a capability of ES|QL +as mentioned in the system message. When converting queries from one language +to ES|QL, make sure that the functions are available and documented in ES|QL. +E.g., for SPL's LEN, use LENGTH. For IF, use CASE.`, + ], + ['ai', 'Please find the ESQL query below:'], + ]).partial({ + documentation, + availableDataViews: state.availableDataViews.join('\n'), + errors: state.errors.join('\n'), + dataViewFields: state.dataViewFields.join('\n'), + invalidQueries: state.invalidQueries.join('\n'), + }); + + const finalChain = answerPrompt + .pipe(llm as ActionsClientChatOpenAI | ActionsClientSimpleChatModel) + .pipe(new StringOutputParser()); + + const finalResult = await finalChain.invoke({ + query: userQuery, + }); + + const correctedResult = finalResult.replaceAll(INLINE_ESQL_QUERY_REGEX, (_match, query) => { + const correction = correctCommonEsqlMistakes(query); + + return `\`\`\`esql\n${correction.output}\n\`\`\``; + }); + + const esqlQuery = correctedResult.match(new RegExp(INLINE_ESQL_QUERY_REGEX, 'ms'))?.[1]; + + return { answer: correctedResult, esqlQuery }; + }; + +const getValidateQuery = + ({ + dataViews, + search, + }: { + dataViews: AssistantToolParams['dataViews']; + search: AssistantToolParams['search']; + }) => + async (state: IState, config?: RunnableConfig) => { + const { errors } = await validateQuery(state.esqlQuery, getAstAndSyntaxErrors, { + // setting this to true, we don't want to validate the index / fields existence + ignoreOnMissingCallbacks: true, + }); + + if (!isEmpty(errors)) { + return { + errors: map( + errors, + (error) => (error as ESQLMessage)?.text || (error as EditorError)?.message + ), + invalidQueries: [state.esqlQuery], + }; + } + + if (search?.search) { + try { + await getESQLQueryColumns({ + esqlQuery: state.esqlQuery, + search: search.search, + }); + return { errors: [] }; + } catch (e) { + let dataViewFields; + let availableDataViews = state.availableDataViews; + if (dataViews) { + const indexPattern = getIndexPatternFromESQLQuery(state.esqlQuery); + let indexFields; + try { + indexFields = await dataViews.getFieldsForWildcard({ pattern: indexPattern }); + } catch (err) { + availableDataViews = without(availableDataViews, indexPattern); + } + + dataViewFields = map(indexFields, (field) => `${field.name} (${field.type})`); + } + + return { + errors: e?.message.match(new RegExp(/Unknown column.*/)) ?? e?.message, + dataViewFields, + availableDataViews, + invalidQueries: [state.esqlQuery], + }; + } + } + + return {}; + }; + +const shouldRegenerate = (state: IState) => { + if (state.errors?.length) { + return 'generateQuery'; + } + + return END; +}; + +export const GRAPH_ESQL_TOOL: AssistantTool = { + ...toolDetails, + sourceRegister: APP_UI_ID, + isSupported: (params: AssistantToolParams): params is AssistantToolParams => { + const { kbDataClient, isEnabledKnowledgeBase, modelExists } = params; + return isEnabledKnowledgeBase && modelExists && kbDataClient != null; + }, + getTool(params: AssistantToolParams) { + if (!this.isSupported(params)) return null; + + const { dataViews, search, llm, esStore } = params; + if (!llm) return null; + + return new DynamicStructuredTool({ + name: toolDetails.name, + description: toolDetails.description, + schema: z.object({ + question: z.string().describe(`The user's exact question about ESQL`), + }) as unknown as LangchainZodAny, + func: async (input, _, cbManager) => { + const workflow = new StateGraph({ + channels: graphState, + }) + .addNode('getDataStreams', getDataStreams({ dataViews })) + .addNode('generateQuery', getGenerateQuery({ userQuery: input.question, llm, esStore })) + .addNode('validateQuery', getValidateQuery({ dataViews, search })) + .addEdge(START, 'getDataStreams') + .addEdge('getDataStreams', 'generateQuery') + .addEdge('generateQuery', 'validateQuery') + .addConditionalEdges('validateQuery', shouldRegenerate); + + const app = workflow.compile(); + + let query; + try { + query = await app.invoke({ question: input.question }); + } catch (e) { + return e; + } + + return query.esqlQuery; + }, + tags: ['esql', 'query-generation', 'knowledge-base'], + }); + }, +}; diff --git a/x-pack/plugins/security_solution/server/assistant/tools/index.ts b/x-pack/plugins/security_solution/server/assistant/tools/index.ts index 0e5ea3a8f69d1..b843999f6eaf1 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/index.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/index.ts @@ -7,17 +7,22 @@ import type { AssistantTool } from '@kbn/elastic-assistant-plugin/server'; +import type { z } from 'zod'; import { ALERT_COUNTS_TOOL } from './alert_counts/alert_counts_tool'; +import { GRAPH_ESQL_TOOL } from './esql_language_knowledge_base/graph_esql_language_tool'; import { ESQL_KNOWLEDGE_BASE_TOOL } from './esql_language_knowledge_base/esql_language_knowledge_base_tool'; import { OPEN_AND_ACKNOWLEDGED_ALERTS_TOOL } from './open_and_acknowledged_alerts/open_and_acknowledged_alerts_tool'; import { ATTACK_DISCOVERY_TOOL } from './attack_discovery/attack_discovery_tool'; import { KNOWLEDGE_BASE_RETRIEVAL_TOOL } from './knowledge_base/knowledge_base_retrieval_tool'; import { KNOWLEDGE_BASE_WRITE_TOOL } from './knowledge_base/knowledge_base_write_tool'; -export const getAssistantTools = (): AssistantTool[] => [ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type LangchainZodAny = z.ZodObject; + +export const getAssistantTools = (graphEsqlToolEnabled?: boolean): AssistantTool[] => [ ALERT_COUNTS_TOOL, ATTACK_DISCOVERY_TOOL, - ESQL_KNOWLEDGE_BASE_TOOL, + ...(graphEsqlToolEnabled ? [GRAPH_ESQL_TOOL] : [ESQL_KNOWLEDGE_BASE_TOOL]), KNOWLEDGE_BASE_RETRIEVAL_TOOL, KNOWLEDGE_BASE_WRITE_TOOL, OPEN_AND_ACKNOWLEDGED_ALERTS_TOOL, diff --git a/x-pack/plugins/security_solution/server/assistant/tools/knowledge_base/knowledge_base_retrieval_tool.ts b/x-pack/plugins/security_solution/server/assistant/tools/knowledge_base/knowledge_base_retrieval_tool.ts index 47cb35e244d51..6c9d644c02685 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/knowledge_base/knowledge_base_retrieval_tool.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/knowledge_base/knowledge_base_retrieval_tool.ts @@ -10,6 +10,7 @@ import { z } from 'zod'; import type { AssistantTool, AssistantToolParams } from '@kbn/elastic-assistant-plugin/server'; import type { AIAssistantKnowledgeBaseDataClient } from '@kbn/elastic-assistant-plugin/server/ai_assistant_data_clients/knowledge_base'; import { APP_UI_ID } from '../../../../common'; +import type { LangchainZodAny } from '..'; export interface KnowledgeBaseRetrievalToolParams extends AssistantToolParams { kbDataClient: AIAssistantKnowledgeBaseDataClient; @@ -39,7 +40,7 @@ export const KNOWLEDGE_BASE_RETRIEVAL_TOOL: AssistantTool = { description: toolDetails.description, schema: z.object({ query: z.string().describe(`Summary of items/things to search for in the knowledge base`), - }), + }) as unknown as LangchainZodAny, func: async (input, _, cbManager) => { logger.debug(`KnowledgeBaseRetrievalToolParams:input\n ${JSON.stringify(input, null, 2)}`); diff --git a/x-pack/plugins/security_solution/server/assistant/tools/knowledge_base/knowledge_base_write_tool.ts b/x-pack/plugins/security_solution/server/assistant/tools/knowledge_base/knowledge_base_write_tool.ts index addb2a5580dfc..7d5f7e968ba75 100644 --- a/x-pack/plugins/security_solution/server/assistant/tools/knowledge_base/knowledge_base_write_tool.ts +++ b/x-pack/plugins/security_solution/server/assistant/tools/knowledge_base/knowledge_base_write_tool.ts @@ -11,6 +11,7 @@ import type { AssistantTool, AssistantToolParams } from '@kbn/elastic-assistant- import type { AIAssistantKnowledgeBaseDataClient } from '@kbn/elastic-assistant-plugin/server/ai_assistant_data_clients/knowledge_base'; import type { KnowledgeBaseEntryCreateProps } from '@kbn/elastic-assistant-common'; import { APP_UI_ID } from '../../../../common'; +import type { LangchainZodAny } from '..'; export interface KnowledgeBaseWriteToolParams extends AssistantToolParams { kbDataClient: AIAssistantKnowledgeBaseDataClient; @@ -45,7 +46,7 @@ export const KNOWLEDGE_BASE_WRITE_TOOL: AssistantTool = { .describe( `Whether or not the entry is required to always be included in conversations. Is only true if the user explicitly asks for it to be required or always included in conversations, otherwise this is always false.` ), - }), + }) as unknown as LangchainZodAny, func: async (input, _, cbManager) => { logger.debug(`KnowledgeBaseWriteToolParams:input\n ${JSON.stringify(input, null, 2)}`); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 5b5b833dd2d4b..7da418e4c5869 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -565,7 +565,10 @@ export class Plugin implements ISecuritySolutionPlugin { this.licensing$ = plugins.licensing.license$; // Assistant Tool and Feature Registration - plugins.elasticAssistant.registerTools(APP_UI_ID, getAssistantTools()); + plugins.elasticAssistant.registerTools( + APP_UI_ID, + getAssistantTools(config.experimentalFeatures.aiAssistantGraphEsqlTool) + ); plugins.elasticAssistant.registerFeatures(APP_UI_ID, { assistantKnowledgeBaseByDefault: config.experimentalFeatures.assistantKnowledgeBaseByDefault, assistantModelEvaluation: config.experimentalFeatures.assistantModelEvaluation, diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index 400efb61ae0aa..ef7693a30d0c0 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -210,5 +210,9 @@ "@kbn/core-i18n-browser", "@kbn/core-theme-browser", "@kbn/integration-assistant-plugin", + "@kbn/esql-datagrid", + "@kbn/visualization-utils", + "@kbn/esql-validation-autocomplete", + "@kbn/esql-ast", ] } diff --git a/yarn.lock b/yarn.lock index 17882dae4ca64..9bb3677b5bd73 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6973,33 +6973,33 @@ resolved "https://registry.yarnpkg.com/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz#8ace5259254426ccef57f3175bc64ed7095ed919" integrity sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw== -"@langchain/community@^0.2.4": - version "0.2.4" - resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.2.4.tgz#fb5feb4f4a01a1b33adfd28ce7126d0dedb3e6d1" - integrity sha512-rwrPNQLyIe84TPqPYbYOfDA4G/ba1rdj7OtZg63dQmxIvNDOmUCh4xIQac2iuRUnM3o4Ben0Faa9qz+V5oPgIA== +"@langchain/community@^0.2.16": + version "0.2.16" + resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.2.16.tgz#5888baf7fc7ea272c5f91aaa0e71bc444167262d" + integrity sha512-dFDcMabKACvuRd0w6EIRLWf1ubPGZEeEwFt9v1jiEr4HCFxH0OF+iM1QUCcVRbB2fK5lqmKeTD1XAeZV8+AyXA== dependencies: - "@langchain/core" "~0.2.0" - "@langchain/openai" "~0.0.28" + "@langchain/core" "~0.2.11" + "@langchain/openai" "~0.1.0" binary-extensions "^2.2.0" expr-eval "^2.0.2" flat "^5.0.2" js-yaml "^4.1.0" langchain "0.2.3" - langsmith "~0.1.1" + langsmith "~0.1.30" uuid "^9.0.0" zod "^3.22.3" zod-to-json-schema "^3.22.5" -"@langchain/core@0.2.3", "@langchain/core@>0.1.0 <0.3.0", "@langchain/core@>0.1.56 <0.3.0", "@langchain/core@>0.1.61 <0.3.0", "@langchain/core@~0.2.0": - version "0.2.3" - resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.2.3.tgz#7faa82f92b0c7843506e827a38bfcbb60f009d13" - integrity sha512-mVuFHSLpPQ4yOHNXeoSA3LnmIMuFmUiit5rvbYcPZqM6SrB2zCNN2nD4Ty5+3H5X4tYItDoSqsTuUNUQySXRQw== +"@langchain/core@>0.1.0 <0.3.0", "@langchain/core@>0.1.61 <0.3.0", "@langchain/core@>=0.2.5 <0.3.0", "@langchain/core@>=0.2.9 <0.3.0", "@langchain/core@^0.2.11", "@langchain/core@~0.2.11": + version "0.2.11" + resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.2.11.tgz#5f47467e20e56b250831baef20083657c6facb4c" + integrity sha512-d4SNL7WI0c3oHrV4WxCRH1/TNqdePXEzYjYwIb4aEH6lW1aM0utGhLbNthX+aYkOL4Ynx2FoG4h91ECIipiKWQ== dependencies: ansi-styles "^5.0.0" camelcase "6" decamelize "1.2.0" js-tiktoken "^1.0.12" - langsmith "~0.1.7" + langsmith "~0.1.30" ml-distance "^4.0.0" mustache "^4.2.0" p-queue "^6.6.2" @@ -7008,22 +7008,22 @@ zod "^3.22.4" zod-to-json-schema "^3.22.3" -"@langchain/langgraph@^0.0.23": - version "0.0.23" - resolved "https://registry.yarnpkg.com/@langchain/langgraph/-/langgraph-0.0.23.tgz#34b5ad5dc9fe644ee96bcfcf11197ec1d7f9e0e2" - integrity sha512-pXlcsBOseT5xdf9enUqbLQ/59LaZxgMI2dL2vFJ+EpcoK7bQnlzzhRtRPp+vubMyMeEKRoAXlaA9ObwpVi93CA== +"@langchain/langgraph@^0.0.25": + version "0.0.25" + resolved "https://registry.yarnpkg.com/@langchain/langgraph/-/langgraph-0.0.25.tgz#2582d8652e2dda722f0c5043c1d0254a778e2486" + integrity sha512-DiTnB5Psm0y7TSgHdK4r/r+xzLohbN4zMQL+5Wk3EmOGX45ioBp98AqL8hYdyxKgHM6cjoIFHavHF7EhMg+ugQ== dependencies: "@langchain/core" ">0.1.61 <0.3.0" uuid "^9.0.1" -"@langchain/openai@^0.0.34", "@langchain/openai@~0.0.28": - version "0.0.34" - resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.0.34.tgz#36c9bca0721ab9f7e5d40927e7c0429cacbd5b56" - integrity sha512-M+CW4oXle5fdoz2T2SwdOef8pl3/1XmUx1vjn2mXUVM/128aO0l23FMF0SNBsAbRV6P+p/TuzjodchJbi0Ht/A== +"@langchain/openai@>=0.1.0 <0.3.0", "@langchain/openai@^0.1.3", "@langchain/openai@~0.1.0": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.1.3.tgz#6eb0994e970d85ffa9aaeafb94449024ccf6ca63" + integrity sha512-riv/JC9x2A8b7GcHu8sx+mlZJ8KAwSSi231IPTlcciYnKozmrQ5H0vrtiD31fxiDbaRsk7tyCpkSBIOQEo7CyQ== dependencies: - "@langchain/core" ">0.1.56 <0.3.0" + "@langchain/core" ">=0.2.5 <0.3.0" js-tiktoken "^1.0.12" - openai "^4.41.1" + openai "^4.49.1" zod "^3.22.4" zod-to-json-schema "^3.22.3" @@ -21774,20 +21774,20 @@ kuler@^2.0.0: resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A== -langchain@0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.2.3.tgz#c14bb05cf871b21bd63b84b3ab89580b1d62539f" - integrity sha512-T9xR7zd+Nj0oXy6WoYKmZLy0DlQiDLFPGYWdOXDxy+AvqlujoPdVQgDSpdqiOHvAjezrByAoKxoHCz5XMwTP/Q== +langchain@0.2.3, langchain@0.2.8, langchain@^0.2.8: + version "0.2.8" + resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.2.8.tgz#9bd77f5c12071d0ccb637c04fc33415e5369e5aa" + integrity sha512-kb2IOMA71xH8e6EXFg0l4S+QSMC/c796pj1+7mPBkR91HHwoyHZhFRrBaZv4tV+Td+Ba91J2uEDBmySklZLpNQ== dependencies: - "@langchain/core" "~0.2.0" - "@langchain/openai" "~0.0.28" + "@langchain/core" ">=0.2.9 <0.3.0" + "@langchain/openai" ">=0.1.0 <0.3.0" "@langchain/textsplitters" "~0.0.0" binary-extensions "^2.2.0" js-tiktoken "^1.0.12" js-yaml "^4.1.0" jsonpointer "^5.0.1" langchainhub "~0.0.8" - langsmith "~0.1.7" + langsmith "~0.1.30" ml-distance "^4.0.0" openapi-types "^12.1.3" p-retry "4" @@ -21801,10 +21801,10 @@ langchainhub@~0.0.8: resolved "https://registry.yarnpkg.com/langchainhub/-/langchainhub-0.0.8.tgz#fd4b96dc795e22e36c1a20bad31b61b0c33d3110" integrity sha512-Woyb8YDHgqqTOZvWIbm2CaFDGfZ4NTSyXV687AG4vXEfoNo7cGQp7nhl7wL3ehenKWmNEmcxCLgOZzW8jE6lOQ== -langsmith@^0.1.30, langsmith@~0.1.1, langsmith@~0.1.7: - version "0.1.30" - resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.1.30.tgz#3000e441605b26e15a87fb991a3929c944edbc0a" - integrity sha512-g8f10H1iiRjCweXJjgM3Y9xl6ApCa1OThDvc0BlSDLVrGVPy1on9wT39vAzYkeadC7oG48p7gfpGlYH3kLkJ9Q== +langsmith@^0.1.30, langsmith@~0.1.30: + version "0.1.32" + resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.1.32.tgz#38938b0e8685522087b697b8200c488c6490c137" + integrity sha512-EUWHIH6fiOCGRYdzgwGoXwJxCMyUrL+bmUcxoVmkXoXoAGDOVinz8bqJLKbxotsQWqM64NKKsW85OTIutgNaMQ== dependencies: "@types/uuid" "^9.0.1" commander "^10.0.1" @@ -24419,10 +24419,10 @@ open@^8.0.9, open@^8.4.0, open@~8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" -openai@^4.24.1, openai@^4.41.1: - version "4.47.1" - resolved "https://registry.yarnpkg.com/openai/-/openai-4.47.1.tgz#1d23c7a8eb3d7bcdc69709cd905f4c9af0181dba" - integrity sha512-WWSxhC/69ZhYWxH/OBsLEirIjUcfpQ5+ihkXKp06hmeYXgBBIUCa9IptMzYx6NdkiOCsSGYCnTIsxaic3AjRCQ== +openai@^4.24.1, openai@^4.49.1: + version "4.51.0" + resolved "https://registry.yarnpkg.com/openai/-/openai-4.51.0.tgz#8ab08bba2441375e8e4ce6161f9ac987d2b2c157" + integrity sha512-UKuWc3/qQyklqhHM8CbdXCv0Z0obap6T0ECdcO5oATQxAbKE5Ky3YCXFQY207z+eGG6ez4U9wvAcuMygxhmStg== dependencies: "@types/node" "^18.11.18" "@types/node-fetch" "^2.6.4"