diff --git a/src/dev/run_quick_checks.ts b/src/dev/run_quick_checks.ts index cdb59bdce3cb2..2fe4b712bdbb1 100644 --- a/src/dev/run_quick_checks.ts +++ b/src/dev/run_quick_checks.ts @@ -6,10 +6,10 @@ * Side Public License, v 1. */ -import { exec } from 'child_process'; +import { execFile } from 'child_process'; import { availableParallelism } from 'os'; -import { join, isAbsolute } from 'path'; -import { readdirSync, readFileSync } from 'fs'; +import { isAbsolute, join } from 'path'; +import { existsSync, readdirSync, readFileSync } from 'fs'; import { run, RunOptions } from '@kbn/dev-cli-runner'; import { REPO_ROOT } from '@kbn/repo-info'; @@ -54,7 +54,7 @@ void run(async ({ log, flagsReader }) => { targetFile: flagsReader.string('file'), targetDir: flagsReader.string('dir'), checks: flagsReader.string('checks'), - }); + }).map((script) => (isAbsolute(script) ? script : join(REPO_ROOT, script))); logger.write( `--- Running ${scriptsToRun.length} checks, with parallelism ${MAX_PARALLELISM}...`, @@ -108,7 +108,7 @@ function collectScriptsToRun(inputOptions: { } } -async function runAllChecks(scriptsToRun: string[]) { +async function runAllChecks(scriptsToRun: string[]): Promise { const checksRunning: Array> = []; const checksFinished: CheckResult[] = []; @@ -121,10 +121,20 @@ async function runAllChecks(scriptsToRun: string[]) { const check = runCheckAsync(script); checksRunning.push(check); - check.then((result) => { - checksRunning.splice(checksRunning.indexOf(check), 1); - checksFinished.push(result); - }); + check + .then((result) => { + checksRunning.splice(checksRunning.indexOf(check), 1); + checksFinished.push(result); + }) + .catch((error) => { + checksRunning.splice(checksRunning.indexOf(check), 1); + checksFinished.push({ + success: false, + script, + output: error.message, + durationMs: 0, + }); + }); } await sleep(1000); @@ -138,9 +148,10 @@ async function runCheckAsync(script: string): Promise { const startTime = Date.now(); return new Promise((resolve) => { - const scriptProcess = exec(script); + validateScriptPath(script); + const scriptProcess = execFile('bash', [script]); let output = ''; - const appendToOutput = (data: string | Buffer) => (output += data); + const appendToOutput = (data: string | Buffer) => (output += data.toString()); scriptProcess.stdout?.on('data', appendToOutput); scriptProcess.stderr?.on('data', appendToOutput); @@ -170,9 +181,10 @@ function printResults(startTimestamp: number, results: CheckResult[]) { logger.info(`- Total time: ${total}, effective: ${effective}`); results.forEach((result) => { - logger.write( - `--- ${result.success ? '✅' : '❌'} ${result.script}: ${humanizeTime(result.durationMs)}` - ); + const resultLabel = result.success ? '✅' : '❌'; + const scriptPath = stripRoot(result.script); + const runtime = humanizeTime(result.durationMs); + logger.write(`--- ${resultLabel} ${scriptPath}: ${runtime}`); if (result.success) { logger.debug(result.output); } else { @@ -194,3 +206,22 @@ function humanizeTime(ms: number) { return `${minutes}m ${seconds}s`; } } + +function validateScriptPath(scriptPath: string) { + if (!isAbsolute(scriptPath)) { + logger.error(`Invalid script path: ${scriptPath}`); + throw new Error('Invalid script path'); + } else if (!scriptPath.endsWith('.sh')) { + logger.error(`Invalid script extension: ${scriptPath}`); + throw new Error('Invalid script extension'); + } else if (!existsSync(scriptPath)) { + logger.error(`Script not found: ${scriptPath}`); + throw new Error('Script not found'); + } else { + return; + } +} + +function stripRoot(script: string) { + return script.replace(REPO_ROOT, ''); +} diff --git a/src/plugins/data/config.mock.ts b/src/plugins/data/config.mock.ts index ab1d02cb63b31..b7c41754dc24d 100644 --- a/src/plugins/data/config.mock.ts +++ b/src/plugins/data/config.mock.ts @@ -7,7 +7,7 @@ */ import moment from 'moment/moment'; -import { SearchConfigSchema, SearchSessionsConfigSchema } from './config'; +import type { SearchConfigSchema, SearchSessionsConfigSchema } from './server/config'; export const getMockSearchConfig = ({ sessions: { enabled = true, defaultExpiration = moment.duration(7, 'd') } = { diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index fa03dc95f564f..09f3226eef57b 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -7,7 +7,7 @@ */ import { PluginInitializerContext } from '@kbn/core/public'; -import { ConfigSchema } from '../config'; +import type { ConfigSchema } from '../server/config'; /* * Filters: diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 530cbe978c3d1..8e0eea12ed168 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -14,7 +14,7 @@ import { IStorageWrapper, createStartServicesGetter, } from '@kbn/kibana-utils-plugin/public'; -import { ConfigSchema } from '../config'; +import type { ConfigSchema } from '../server/config'; import type { DataPublicPluginSetup, DataPublicPluginStart, diff --git a/src/plugins/data/public/search/search_interceptor/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor/search_interceptor.ts index 7f54c63592b14..95573127fdc90 100644 --- a/src/plugins/data/public/search/search_interceptor/search_interceptor.ts +++ b/src/plugins/data/public/search/search_interceptor/search_interceptor.ts @@ -73,7 +73,7 @@ import { toPartialResponseAfterTimeout } from './to_partial_response'; import { ISessionService, SearchSessionState } from '../session'; import { SearchResponseCache } from './search_response_cache'; import { SearchAbortController } from './search_abort_controller'; -import { SearchConfigSchema } from '../../../config'; +import type { SearchConfigSchema } from '../../../server/config'; import type { SearchServiceStartDependencies } from '../search_service'; import { createRequestHash } from './create_request_hash'; diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index 0bdc33cb59303..07afdc0514c55 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -62,7 +62,7 @@ import { SHARD_DELAY_AGG_NAME, } from '../../common/search/aggs/buckets/shard_delay'; import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn'; -import { ConfigSchema } from '../../config'; +import type { ConfigSchema } from '../../server/config'; import { NowProviderInternalContract } from '../now_provider'; import { DataPublicPluginStart, DataStartDependencies } from '../types'; import { AggsService } from './aggs'; diff --git a/src/plugins/data/public/search/session/session_service.ts b/src/plugins/data/public/search/session/session_service.ts index 5b936fb72a88d..b3c3e94f18b54 100644 --- a/src/plugins/data/public/search/session/session_service.ts +++ b/src/plugins/data/public/search/session/session_service.ts @@ -39,7 +39,7 @@ import { i18n } from '@kbn/i18n'; import moment from 'moment'; import { ISearchOptions } from '@kbn/search-types'; import { SearchUsageCollector } from '../..'; -import { ConfigSchema } from '../../../config'; +import type { ConfigSchema } from '../../../server/config'; import type { SessionMeta, SessionStateContainer, diff --git a/src/plugins/data/public/search/session/sessions_mgmt/application/index.tsx b/src/plugins/data/public/search/session/sessions_mgmt/application/index.tsx index b0a15ca405743..b0c4330c35e5a 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/application/index.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/application/index.tsx @@ -17,7 +17,7 @@ import { APP } from '..'; import { SearchSessionsMgmtAPI } from '../lib/api'; import { AsyncSearchIntroDocumentation } from '../lib/documentation'; import { renderApp } from './render'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; export class SearchSessionsMgmtApp { constructor( diff --git a/src/plugins/data/public/search/session/sessions_mgmt/components/main.test.tsx b/src/plugins/data/public/search/session/sessions_mgmt/components/main.test.tsx index 38ccbb9646dee..97b446cae47c1 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/components/main.test.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/components/main.test.tsx @@ -20,7 +20,7 @@ import { LocaleWrapper } from '../__mocks__'; import { SearchSessionsMgmtMain } from './main'; import { SharePluginStart } from '@kbn/share-plugin/public'; import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; import { createSearchUsageCollectorMock } from '../../../collectors/mocks'; let mockCoreSetup: MockedKeys; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/components/main.tsx b/src/plugins/data/public/search/session/sessions_mgmt/components/main.tsx index 009e3ce7d9b07..af5e1c3bc9a8c 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/components/main.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/components/main.tsx @@ -14,7 +14,7 @@ import type { SearchSessionsMgmtAPI } from '../lib/api'; import type { AsyncSearchIntroDocumentation } from '../lib/documentation'; import { SearchSessionsMgmtTable } from './table'; import { SearchSessionsDeprecatedWarning } from '../../search_sessions_deprecation_message'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; import { SearchUsageCollector } from '../../../collectors'; interface Props { diff --git a/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.test.tsx b/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.test.tsx index 6394deeab843b..efe35f206dc5f 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.test.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.test.tsx @@ -20,7 +20,7 @@ import { LocaleWrapper } from '../../__mocks__'; import { SearchSessionsMgmtTable } from './table'; import { SharePluginStart } from '@kbn/share-plugin/public'; import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; -import { SearchSessionsConfigSchema } from '../../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../../server/config'; import { createSearchUsageCollectorMock } from '../../../../collectors/mocks'; let mockCoreSetup: MockedKeys; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.tsx b/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.tsx index 833ad3ec7e75e..649a3d4316a5e 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.tsx @@ -21,7 +21,7 @@ import { OnActionComplete } from '../actions'; import { getAppFilter } from './app_filter'; import { getStatusFilter } from './status_filter'; import { SearchUsageCollector } from '../../../../collectors'; -import { SearchSessionsConfigSchema } from '../../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../../server/config'; interface Props { core: CoreStart; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/index.ts b/src/plugins/data/public/search/session/sessions_mgmt/index.ts index 7b95305f6e568..59437bf27b6e4 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/index.ts +++ b/src/plugins/data/public/search/session/sessions_mgmt/index.ts @@ -15,7 +15,7 @@ import type { ISessionsClient, SearchUsageCollector } from '../../..'; import { SEARCH_SESSIONS_MANAGEMENT_ID } from '../constants'; import type { SearchSessionsMgmtAPI } from './lib/api'; import type { AsyncSearchIntroDocumentation } from './lib/documentation'; -import { SearchSessionsConfigSchema } from '../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../server/config'; export interface IManagementSectionsPluginsSetup { management: ManagementSetup; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/lib/api.test.ts b/src/plugins/data/public/search/session/sessions_mgmt/lib/api.test.ts index 52b72afd30ab3..a765bb8a0a2d4 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/lib/api.test.ts +++ b/src/plugins/data/public/search/session/sessions_mgmt/lib/api.test.ts @@ -16,7 +16,7 @@ import { SearchSessionStatus } from '../../../../../common'; import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; import { SharePluginStart } from '@kbn/share-plugin/public'; import { SearchSessionsMgmtAPI } from './api'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; let mockCoreSetup: MockedKeys; let mockCoreStart: MockedKeys; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/lib/api.ts b/src/plugins/data/public/search/session/sessions_mgmt/lib/api.ts index f5abbc4c77de8..5c426942034f6 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/lib/api.ts +++ b/src/plugins/data/public/search/session/sessions_mgmt/lib/api.ts @@ -22,7 +22,7 @@ import { import { ISessionsClient } from '../../sessions_client'; import { SearchUsageCollector } from '../../../collectors'; import { SearchSessionsFindResponse, SearchSessionStatus } from '../../../../../common'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; type LocatorsStart = SharePluginStart['url']['locators']; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.test.tsx b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.test.tsx index 3fd53ccfddec2..643cd6b6286f6 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.test.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.test.tsx @@ -21,7 +21,7 @@ import { SearchSessionsMgmtAPI } from './api'; import { getColumns } from './get_columns'; import { SharePluginStart } from '@kbn/share-plugin/public'; import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; import { createSearchUsageCollectorMock } from '../../../collectors/mocks'; let mockCoreSetup: MockedKeys; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.tsx b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.tsx index 371ffac111e84..07862651f60ab 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.tsx @@ -30,7 +30,7 @@ import { SearchSessionsMgmtAPI } from './api'; import { getExpirationStatus } from './get_expiration_status'; import { UISession } from '../types'; import { SearchUsageCollector } from '../../../collectors'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; // Helper function: translate an app string to EuiIcon-friendly string const appToIcon = (app: string) => { diff --git a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_expiration_status.ts b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_expiration_status.ts index cde183f569f93..b80b315aaea8c 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_expiration_status.ts +++ b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_expiration_status.ts @@ -8,7 +8,7 @@ import { i18n } from '@kbn/i18n'; import moment from 'moment'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; export const getExpirationStatus = (config: SearchSessionsConfigSchema, expires: string | null) => { const tNow = moment.utc().valueOf(); diff --git a/src/plugins/data/config.ts b/src/plugins/data/server/config.ts similarity index 100% rename from src/plugins/data/config.ts rename to src/plugins/data/server/config.ts diff --git a/src/plugins/data/server/index.ts b/src/plugins/data/server/index.ts index 7a41c20094d6e..e9292342f917e 100644 --- a/src/plugins/data/server/index.ts +++ b/src/plugins/data/server/index.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ import { PluginConfigDescriptor, PluginInitializerContext } from '@kbn/core/server'; -import { ConfigSchema, configSchema } from '../config'; +import { ConfigSchema, configSchema } from './config'; import type { DataServerPlugin, DataPluginSetup, DataPluginStart } from './plugin'; export { getEsQueryConfig, DEFAULT_QUERY_LANGUAGE } from '../common'; diff --git a/src/plugins/data/server/plugin.ts b/src/plugins/data/server/plugin.ts index 77f90393a6164..296d59dc2f632 100644 --- a/src/plugins/data/server/plugin.ts +++ b/src/plugins/data/server/plugin.ts @@ -12,7 +12,7 @@ import { BfetchServerSetup } from '@kbn/bfetch-plugin/server'; import { PluginStart as DataViewsServerPluginStart } from '@kbn/data-views-plugin/server'; import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; import { FieldFormatsSetup, FieldFormatsStart } from '@kbn/field-formats-plugin/server'; -import { ConfigSchema } from '../config'; +import { ConfigSchema } from './config'; import type { ISearchSetup, ISearchStart } from './search'; import { DatatableUtilitiesService } from './datatable_utilities'; import { SearchService } from './search/search_service'; diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index 09bee9bb8bab8..b9449b7f5da11 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -90,7 +90,7 @@ import { SHARD_DELAY_AGG_NAME, } from '../../common/search/aggs/buckets/shard_delay'; import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn'; -import { ConfigSchema } from '../../config'; +import { ConfigSchema } from '../config'; import { SearchSessionService } from './session'; import { registerBsearchRoute } from './routes/bsearch'; import { enhancedEsSearchStrategyProvider } from './strategies/ese_search'; diff --git a/src/plugins/data/server/search/session/get_session_status.test.ts b/src/plugins/data/server/search/session/get_session_status.test.ts index 1d59fd11c471e..128c19220c2f3 100644 --- a/src/plugins/data/server/search/session/get_session_status.test.ts +++ b/src/plugins/data/server/search/session/get_session_status.test.ts @@ -10,7 +10,7 @@ import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { getSessionStatus } from './get_session_status'; import { SearchSessionSavedObjectAttributes, SearchSessionStatus } from '../../../common'; import moment from 'moment'; -import { SearchSessionsConfigSchema } from '../../../config'; +import { SearchSessionsConfigSchema } from '../../config'; const mockInProgressSearchResponse = { body: { diff --git a/src/plugins/data/server/search/session/get_session_status.ts b/src/plugins/data/server/search/session/get_session_status.ts index e1cb50a6c4cd4..57bae3c38e9a7 100644 --- a/src/plugins/data/server/search/session/get_session_status.ts +++ b/src/plugins/data/server/search/session/get_session_status.ts @@ -10,7 +10,7 @@ import moment from 'moment'; import { ElasticsearchClient } from '@kbn/core/server'; import { SearchSessionSavedObjectAttributes, SearchSessionStatus } from '../../../common'; import { SearchStatus } from './types'; -import { SearchSessionsConfigSchema } from '../../../config'; +import { SearchSessionsConfigSchema } from '../../config'; import { getSearchStatus } from './get_search_status'; export async function getSessionStatus( diff --git a/src/plugins/data/server/search/session/mocks.ts b/src/plugins/data/server/search/session/mocks.ts index 339ef628356db..a5bbb06a5621a 100644 --- a/src/plugins/data/server/search/session/mocks.ts +++ b/src/plugins/data/server/search/session/mocks.ts @@ -8,7 +8,7 @@ import moment from 'moment'; import type { IScopedSearchSessionsClient } from './types'; -import { SearchSessionsConfigSchema } from '../../../config'; +import { SearchSessionsConfigSchema } from '../../config'; export function createSearchSessionsClientMock(): jest.Mocked { return { diff --git a/src/plugins/data/server/search/session/session_service.test.ts b/src/plugins/data/server/search/session/session_service.test.ts index 5a31cefe7b998..5f596da374703 100644 --- a/src/plugins/data/server/search/session/session_service.test.ts +++ b/src/plugins/data/server/search/session/session_service.test.ts @@ -17,7 +17,7 @@ import { SearchSessionService } from './session_service'; import { createRequestHash } from './utils'; import moment from 'moment'; import { coreMock } from '@kbn/core/server/mocks'; -import { ConfigSchema } from '../../../config'; +import { ConfigSchema } from '../../config'; import type { AuthenticatedUser } from '@kbn/core/server'; import { SEARCH_SESSION_TYPE, SearchSessionStatus } from '../../../common'; import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; diff --git a/src/plugins/data/server/search/session/session_service.ts b/src/plugins/data/server/search/session/session_service.ts index efd41990493b3..ef367b552983d 100644 --- a/src/plugins/data/server/search/session/session_service.ts +++ b/src/plugins/data/server/search/session/session_service.ts @@ -32,7 +32,7 @@ import { } from '../../../common'; import { ISearchSessionService, NoSearchIdInSessionError } from '../..'; import { createRequestHash } from './utils'; -import { ConfigSchema, SearchSessionsConfigSchema } from '../../../config'; +import { ConfigSchema, SearchSessionsConfigSchema } from '../../config'; import { getSessionStatus } from './get_session_status'; export interface SearchSessionDependencies { diff --git a/src/plugins/data/server/search/session/types.ts b/src/plugins/data/server/search/session/types.ts index cc3d604f50efc..02cb9cad55bb7 100644 --- a/src/plugins/data/server/search/session/types.ts +++ b/src/plugins/data/server/search/session/types.ts @@ -19,7 +19,7 @@ import { SearchSessionSavedObjectAttributes, SearchSessionStatusResponse, } from '../../../common/search'; -import { SearchSessionsConfigSchema } from '../../../config'; +import { SearchSessionsConfigSchema } from '../../config'; export { SearchStatus } from '../../../common/search'; diff --git a/src/plugins/data/server/search/strategies/common/async_utils.ts b/src/plugins/data/server/search/strategies/common/async_utils.ts index ca33a01ac8064..6c2d4b98bc5a1 100644 --- a/src/plugins/data/server/search/strategies/common/async_utils.ts +++ b/src/plugins/data/server/search/strategies/common/async_utils.ts @@ -11,7 +11,7 @@ import { AsyncSearchGetRequest, } from '@elastic/elasticsearch/lib/api/types'; import { ISearchOptions } from '@kbn/search-types'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; /** @internal diff --git a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts index 9db01189ff66d..2bf6ff4f68601 100644 --- a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts @@ -10,7 +10,7 @@ import type { TransportResult } from '@elastic/elasticsearch'; import { tap } from 'rxjs'; import type { IScopedClusterClient, Logger } from '@kbn/core/server'; import { getKbnServerError } from '@kbn/kibana-utils-plugin/server'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; import { EqlSearchStrategyRequest, EqlSearchStrategyResponse, diff --git a/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts b/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts index 8c5a8ad204a72..d4933357e0334 100644 --- a/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts @@ -31,7 +31,7 @@ import { getTotalLoaded, shimHitsTotal, } from '../es_search'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; import { sanitizeRequestParams } from '../../sanitize_request_params'; export const enhancedEsSearchStrategyProvider = ( diff --git a/src/plugins/data/server/search/strategies/ese_search/request_utils.ts b/src/plugins/data/server/search/strategies/ese_search/request_utils.ts index ec182534e089f..d3341b192660c 100644 --- a/src/plugins/data/server/search/strategies/ese_search/request_utils.ts +++ b/src/plugins/data/server/search/strategies/ese_search/request_utils.ts @@ -12,7 +12,7 @@ import { AsyncSearchSubmitRequest } from '@elastic/elasticsearch/lib/api/types'; import { ISearchOptions } from '@kbn/search-types'; import { UI_SETTINGS } from '../../../../common'; import { getDefaultSearchParams } from '../es_search'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; import { getCommonDefaultAsyncGetParams, getCommonDefaultAsyncSubmitParams, diff --git a/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts b/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts index a204b6ca69cca..a8623d99e1632 100644 --- a/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts @@ -22,7 +22,7 @@ import { getKbnSearchError } from '../../report_search_error'; import type { ISearchStrategy, SearchStrategyDependencies } from '../../types'; import type { IAsyncSearchOptions } from '../../../../common'; import { toAsyncKibanaSearchResponse } from './response_utils'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; // `drop_null_columns` is going to change the response // now we get `all_columns` and `columns` diff --git a/src/plugins/data/server/search/strategies/sql_search/request_utils.ts b/src/plugins/data/server/search/strategies/sql_search/request_utils.ts index 1a2540796d690..a608409467824 100644 --- a/src/plugins/data/server/search/strategies/sql_search/request_utils.ts +++ b/src/plugins/data/server/search/strategies/sql_search/request_utils.ts @@ -8,7 +8,7 @@ import { SqlGetAsyncRequest, SqlQueryRequest } from '@elastic/elasticsearch/lib/api/types'; import { ISearchOptions } from '@kbn/search-types'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; import { getCommonDefaultAsyncGetParams, getCommonDefaultAsyncSubmitParams, diff --git a/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts b/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts index 55ed1bfecb995..c71e926d764db 100644 --- a/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts @@ -22,7 +22,7 @@ import type { import { pollSearch } from '../../../../common'; import { getDefaultAsyncGetParams, getDefaultAsyncSubmitParams } from './request_utils'; import { toAsyncKibanaSearchResponse } from './response_utils'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; export const sqlSearchStrategyProvider = ( searchConfig: SearchConfigSchema, diff --git a/src/plugins/data/tsconfig.json b/src/plugins/data/tsconfig.json index cc6ced2bef611..1b2fc9ed85e93 100644 --- a/src/plugins/data/tsconfig.json +++ b/src/plugins/data/tsconfig.json @@ -7,7 +7,6 @@ "common/**/*", "public/**/*", "server/**/*", - "config.ts", "config.mock.ts", "common/**/*.json", "public/**/*.json", diff --git a/src/plugins/no_data_page/public/plugin.ts b/src/plugins/no_data_page/public/plugin.ts index 910208f0f94be..dc7dff0e0781e 100644 --- a/src/plugins/no_data_page/public/plugin.ts +++ b/src/plugins/no_data_page/public/plugin.ts @@ -8,7 +8,7 @@ import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; import type { NoDataPagePublicSetup, NoDataPagePublicStart } from './types'; -import type { NoDataPageConfig } from '../config'; +import type { NoDataPageConfig } from '../server/config'; export class NoDataPagePlugin implements Plugin { constructor(private initializerContext: PluginInitializerContext) {} diff --git a/src/plugins/no_data_page/config.ts b/src/plugins/no_data_page/server/config.ts similarity index 100% rename from src/plugins/no_data_page/config.ts rename to src/plugins/no_data_page/server/config.ts diff --git a/src/plugins/no_data_page/server/index.ts b/src/plugins/no_data_page/server/index.ts index ba02a016a9676..cabe47bf65a33 100644 --- a/src/plugins/no_data_page/server/index.ts +++ b/src/plugins/no_data_page/server/index.ts @@ -8,7 +8,7 @@ import { PluginConfigDescriptor } from '@kbn/core-plugins-server'; -import { configSchema, NoDataPageConfig } from '../config'; +import { configSchema, NoDataPageConfig } from './config'; export const config: PluginConfigDescriptor = { exposeToBrowser: { diff --git a/src/plugins/no_data_page/tsconfig.json b/src/plugins/no_data_page/tsconfig.json index bab1c8c23edfb..e92a0c1560380 100644 --- a/src/plugins/no_data_page/tsconfig.json +++ b/src/plugins/no_data_page/tsconfig.json @@ -3,7 +3,7 @@ "compilerOptions": { "outDir": "target/types" }, - "include": ["common/**/*", "public/**/*", "server/**/*", "config.ts"], + "include": ["common/**/*", "public/**/*", "server/**/*"], "kbn_references": [ "@kbn/core", "@kbn/core-plugins-browser", diff --git a/src/plugins/share/public/index.ts b/src/plugins/share/public/index.ts index f212081133b2d..842f31bf11dc9 100644 --- a/src/plugins/share/public/index.ts +++ b/src/plugins/share/public/index.ts @@ -8,7 +8,7 @@ import type { PluginInitializerContext } from '@kbn/core/public'; -export type { ConfigSchema } from '../common/config'; +export type { ConfigSchema } from '../server/config'; export { CSV_QUOTE_VALUES_SETTING, CSV_SEPARATOR_SETTING } from '../common/constants'; diff --git a/src/plugins/share/common/config.ts b/src/plugins/share/server/config.ts similarity index 100% rename from src/plugins/share/common/config.ts rename to src/plugins/share/server/config.ts diff --git a/src/plugins/share/server/index.ts b/src/plugins/share/server/index.ts index 5f61ba8693814..c3bac70cc6e25 100644 --- a/src/plugins/share/server/index.ts +++ b/src/plugins/share/server/index.ts @@ -7,7 +7,7 @@ */ import { PluginConfigDescriptor, PluginInitializerContext } from '@kbn/core/server'; -import { ConfigSchema, configSchema } from '../common/config'; +import { ConfigSchema, configSchema } from './config'; export type { SharePublicSetup as SharePluginSetup, diff --git a/x-pack/plugins/actions/server/integration_tests/__snapshots__/connector_types.test.ts.snap b/x-pack/plugins/actions/server/integration_tests/__snapshots__/connector_types.test.ts.snap index ae1289b2a9e2e..6e4392e3f737e 100644 --- a/x-pack/plugins/actions/server/integration_tests/__snapshots__/connector_types.test.ts.snap +++ b/x-pack/plugins/actions/server/integration_tests/__snapshots__/connector_types.test.ts.snap @@ -273,8 +273,15 @@ Object { "keys": Object { "content": Object { "flags": Object { + "default": [Function], "error": [Function], + "presence": "optional", }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], "rules": Array [ Object { "args": Object { @@ -285,6 +292,33 @@ Object { ], "type": "string", }, + "rawContent": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "items": Array [ + Object { + "flags": Object { + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-any-type": true, + }, + ], + "type": "any", + }, + ], + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "array", + }, "role": Object { "flags": Object { "error": [Function], @@ -300,6 +334,14 @@ Object { "type": "string", }, }, + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], "type": "object", }, ], @@ -419,6 +461,86 @@ Object { ], "type": "number", }, + "toolChoice": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "keys": Object { + "name": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], + "type": "string", + }, + "type": Object { + "flags": Object { + "error": [Function], + }, + "matches": Array [ + Object { + "schema": Object { + "allow": Array [ + "auto", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "any", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "tool", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + ], + "type": "alternatives", + }, + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "object", + }, "tools": Object { "flags": Object { "default": [Function], @@ -556,8 +678,15 @@ Object { "keys": Object { "content": Object { "flags": Object { + "default": [Function], "error": [Function], + "presence": "optional", }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], "rules": Array [ Object { "args": Object { @@ -568,6 +697,33 @@ Object { ], "type": "string", }, + "rawContent": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "items": Array [ + Object { + "flags": Object { + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-any-type": true, + }, + ], + "type": "any", + }, + ], + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "array", + }, "role": Object { "flags": Object { "error": [Function], @@ -583,6 +739,14 @@ Object { "type": "string", }, }, + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], "type": "object", }, ], @@ -702,6 +866,86 @@ Object { ], "type": "number", }, + "toolChoice": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "keys": Object { + "name": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], + "type": "string", + }, + "type": Object { + "flags": Object { + "error": [Function], + }, + "matches": Array [ + Object { + "schema": Object { + "allow": Array [ + "auto", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "any", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "tool", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + ], + "type": "alternatives", + }, + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "object", + }, "tools": Object { "flags": Object { "default": [Function], @@ -839,14 +1083,51 @@ Object { "keys": Object { "content": Object { "flags": Object { + "default": [Function], "error": [Function], + "presence": "optional", }, "metas": Array [ Object { - "x-oas-any-type": true, + "x-oas-optional": true, }, ], - "type": "any", + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], + "type": "string", + }, + "rawContent": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "items": Array [ + Object { + "flags": Object { + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-any-type": true, + }, + ], + "type": "any", + }, + ], + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "array", }, "role": Object { "flags": Object { @@ -863,6 +1144,14 @@ Object { "type": "string", }, }, + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], "type": "object", }, ], @@ -982,6 +1271,86 @@ Object { ], "type": "number", }, + "toolChoice": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "keys": Object { + "name": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], + "type": "string", + }, + "type": Object { + "flags": Object { + "error": [Function], + }, + "matches": Array [ + Object { + "schema": Object { + "allow": Array [ + "auto", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "any", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "tool", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + ], + "type": "alternatives", + }, + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "object", + }, "tools": Object { "flags": Object { "default": [Function], diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.test.ts index d89126f549a1b..43bea033a2e3d 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.test.ts @@ -5,14 +5,12 @@ * 2.0. */ -import { KibanaRequest } from '@kbn/core/server'; -import { securityMock } from '@kbn/security-plugin/server/mocks'; +import { securityServiceMock } from '@kbn/core-security-server-mocks'; import { createApiKey } from './create_api_key'; describe('createApiKey lib function', () => { - const security = securityMock.createStart(); - const request = {} as KibanaRequest; + const security = securityServiceMock.createRequestHandlerContext(); const indexName = 'my-index'; const keyName = '{indexName}-key'; @@ -31,11 +29,9 @@ describe('createApiKey lib function', () => { }); it('should create an api key via the security plugin', async () => { - await expect(createApiKey(request, security, indexName, keyName)).resolves.toEqual( - createResponse - ); + await expect(createApiKey(security, indexName, keyName)).resolves.toEqual(createResponse); - expect(security.authc.apiKeys.create).toHaveBeenCalledWith(request, { + expect(security.authc.apiKeys.create).toHaveBeenCalledWith({ name: keyName, role_descriptors: { [`${indexName}-key-role`]: { diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.ts b/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.ts index 0c1a62c4d30db..b5d7fb7cf22e9 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.ts @@ -5,19 +5,16 @@ * 2.0. */ -import { KibanaRequest } from '@kbn/core/server'; - -import { SecurityPluginStart } from '@kbn/security-plugin/server'; +import type { SecurityRequestHandlerContext } from '@kbn/core-security-server'; import { toAlphanumeric } from '../../../common/utils/to_alphanumeric'; export const createApiKey = async ( - request: KibanaRequest, - security: SecurityPluginStart, + security: SecurityRequestHandlerContext, indexName: string, keyName: string ) => { - return await security.authc.apiKeys.create(request, { + return await security.authc.apiKeys.create({ name: keyName, role_descriptors: { [`${toAlphanumeric(indexName)}-key-role`]: { diff --git a/x-pack/plugins/enterprise_search/server/plugin.ts b/x-pack/plugins/enterprise_search/server/plugin.ts index a03429729bf2f..d021b1b3cd634 100644 --- a/x-pack/plugins/enterprise_search/server/plugin.ts +++ b/x-pack/plugins/enterprise_search/server/plugin.ts @@ -25,7 +25,7 @@ import type { LicensingPluginStart } from '@kbn/licensing-plugin/public'; import { LogsSharedPluginSetup } from '@kbn/logs-shared-plugin/server'; import type { MlPluginSetup } from '@kbn/ml-plugin/server'; import { SearchConnectorsPluginSetup } from '@kbn/search-connectors-plugin/server'; -import { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/server'; +import { SecurityPluginSetup } from '@kbn/security-plugin/server'; import { SpacesPluginStart } from '@kbn/spaces-plugin/server'; import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; @@ -104,7 +104,6 @@ interface PluginsSetup { export interface PluginsStart { data: DataPluginStart; - security: SecurityPluginStart; spaces?: SpacesPluginStart; } @@ -284,9 +283,7 @@ export class EnterpriseSearchPlugin implements Plugin { registerAnalyticsRoutes({ ...dependencies, data, savedObjects: coreStart.savedObjects }); }); - void getStartServices().then(([, { security: securityStart }]) => { - registerApiKeysRoutes(dependencies, securityStart); - }); + registerApiKeysRoutes(dependencies); /** * Bootstrap the routes, saved objects, and collector for telemetry diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/api_keys.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/api_keys.ts index 8b94de5e6955c..71e00bed27910 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/api_keys.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/api_keys.ts @@ -7,16 +7,11 @@ import { schema } from '@kbn/config-schema'; -import { SecurityPluginStart } from '@kbn/security-plugin/server'; - import { createApiKey } from '../../lib/indices/create_api_key'; import { RouteDependencies } from '../../plugin'; import { elasticsearchErrorHandler } from '../../utils/elasticsearch_error_handler'; -export function registerApiKeysRoutes( - { log, router }: RouteDependencies, - security: SecurityPluginStart -) { +export function registerApiKeysRoutes({ log, router }: RouteDependencies) { router.post( { path: '/internal/enterprise_search/{indexName}/api_keys', @@ -32,8 +27,9 @@ export function registerApiKeysRoutes( elasticsearchErrorHandler(log, async (context, request, response) => { const indexName = decodeURIComponent(request.params.indexName); const { keyName } = request.body; + const { security: coreSecurity } = await context.core; - const createResponse = await createApiKey(request, security, indexName, keyName); + const createResponse = await createApiKey(coreSecurity, indexName, keyName); if (!createResponse) { throw new Error('Unable to create API Key'); @@ -118,7 +114,8 @@ export function registerApiKeysRoutes( }, }, async (context, request, response) => { - const result = await security.authc.apiKeys.create(request, request.body); + const { security: coreSecurity } = await context.core; + const result = await coreSecurity.authc.apiKeys.create(request.body); if (result) { const apiKey = { ...result, beats_logstash_format: `${result.id}:${result.api_key}` }; return response.ok({ body: apiKey }); diff --git a/x-pack/plugins/enterprise_search/tsconfig.json b/x-pack/plugins/enterprise_search/tsconfig.json index 86cf6c3968005..58b1526e14baf 100644 --- a/x-pack/plugins/enterprise_search/tsconfig.json +++ b/x-pack/plugins/enterprise_search/tsconfig.json @@ -81,6 +81,8 @@ "@kbn/core-chrome-browser", "@kbn/navigation-plugin", "@kbn/search-homepage", - "@kbn/security-plugin-types-common" + "@kbn/security-plugin-types-common", + "@kbn/core-security-server", + "@kbn/core-security-server-mocks" ] } diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.test.ts new file mode 100644 index 0000000000000..1d25f09dce3bc --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.test.ts @@ -0,0 +1,310 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PassThrough } from 'stream'; +import type { InferenceExecutor } from '../../utils/inference_executor'; +import { MessageRole } from '../../../../common/chat_complete'; +import { ToolChoiceType } from '../../../../common/chat_complete/tools'; +import { bedrockClaudeAdapter } from './bedrock_claude_adapter'; +import { addNoToolUsageDirective } from './prompts'; + +describe('bedrockClaudeAdapter', () => { + const executorMock = { + invoke: jest.fn(), + } as InferenceExecutor & { invoke: jest.MockedFn }; + + beforeEach(() => { + executorMock.invoke.mockReset(); + executorMock.invoke.mockImplementation(async () => { + return { + actionId: '', + status: 'ok', + data: new PassThrough(), + }; + }); + }); + + function getCallParams() { + const params = executorMock.invoke.mock.calls[0][0].subActionParams as Record; + return { + system: params.system, + messages: params.messages, + tools: params.tools, + toolChoice: params.toolChoice, + }; + } + + describe('#chatComplete()', () => { + it('calls `executor.invoke` with the right fixed parameters', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + expect(executorMock.invoke).toHaveBeenCalledWith({ + subAction: 'invokeStream', + subActionParams: { + messages: [ + { + role: 'user', + rawContent: [{ type: 'text', text: 'question' }], + }, + ], + temperature: 0, + stopSequences: ['\n\nHuman:'], + }, + }); + }); + + it('correctly format tools', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + tools: { + myFunction: { + description: 'myFunction', + }, + myFunctionWithArgs: { + description: 'myFunctionWithArgs', + schema: { + type: 'object', + properties: { + foo: { + type: 'string', + description: 'foo', + }, + }, + required: ['foo'], + }, + }, + }, + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { tools } = getCallParams(); + expect(tools).toEqual([ + { + name: 'myFunction', + description: 'myFunction', + input_schema: { + properties: {}, + type: 'object', + }, + }, + { + name: 'myFunctionWithArgs', + description: 'myFunctionWithArgs', + input_schema: { + properties: { + foo: { + description: 'foo', + type: 'string', + }, + }, + required: ['foo'], + type: 'object', + }, + }, + ]); + }); + + it('correctly format messages', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + { + role: MessageRole.Assistant, + content: 'answer', + }, + { + role: MessageRole.User, + content: 'another question', + }, + { + role: MessageRole.Assistant, + content: null, + toolCalls: [ + { + function: { + name: 'my_function', + arguments: { + foo: 'bar', + }, + }, + toolCallId: '0', + }, + ], + }, + { + role: MessageRole.Tool, + toolCallId: '0', + response: { + bar: 'foo', + }, + }, + ], + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { messages } = getCallParams(); + expect(messages).toEqual([ + { + rawContent: [ + { + text: 'question', + type: 'text', + }, + ], + role: 'user', + }, + { + rawContent: [ + { + text: 'answer', + type: 'text', + }, + ], + role: 'assistant', + }, + { + rawContent: [ + { + text: 'another question', + type: 'text', + }, + ], + role: 'user', + }, + { + rawContent: [ + { + id: '0', + input: { + foo: 'bar', + }, + name: 'my_function', + type: 'tool_use', + }, + ], + role: 'assistant', + }, + { + rawContent: [ + { + content: '{"bar":"foo"}', + tool_use_id: '0', + type: 'tool_result', + }, + ], + role: 'user', + }, + ]); + }); + + it('correctly format system message', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + system: 'Some system message', + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { system } = getCallParams(); + expect(system).toEqual('Some system message'); + }); + + it('correctly format tool choice', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + toolChoice: ToolChoiceType.required, + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { toolChoice } = getCallParams(); + expect(toolChoice).toEqual({ + type: 'any', + }); + }); + + it('correctly format tool choice for named function', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + toolChoice: { function: 'foobar' }, + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { toolChoice } = getCallParams(); + expect(toolChoice).toEqual({ + type: 'tool', + name: 'foobar', + }); + }); + + it('correctly adapt the request for ToolChoiceType.None', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + system: 'some system instruction', + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + tools: { + myFunction: { + description: 'myFunction', + }, + }, + toolChoice: ToolChoiceType.none, + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { toolChoice, tools, system } = getCallParams(); + expect(toolChoice).toBeUndefined(); + expect(tools).toEqual([]); + expect(system).toEqual(addNoToolUsageDirective('some system instruction')); + }); + }); +}); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.ts new file mode 100644 index 0000000000000..5a03dc04347b1 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { filter, from, map, switchMap, tap } from 'rxjs'; +import { Readable } from 'stream'; +import type { InvokeAIActionParams } from '@kbn/stack-connectors-plugin/common/bedrock/types'; +import { parseSerdeChunkMessage } from './serde_utils'; +import { Message, MessageRole } from '../../../../common/chat_complete'; +import { createInferenceInternalError } from '../../../../common/errors'; +import { ToolChoiceType, type ToolOptions } from '../../../../common/chat_complete/tools'; +import { InferenceConnectorAdapter } from '../../types'; +import type { BedRockMessage, BedrockToolChoice } from './types'; +import { + BedrockChunkMember, + serdeEventstreamIntoObservable, +} from './serde_eventstream_into_observable'; +import { processCompletionChunks } from './process_completion_chunks'; +import { addNoToolUsageDirective } from './prompts'; + +export const bedrockClaudeAdapter: InferenceConnectorAdapter = { + chatComplete: ({ executor, system, messages, toolChoice, tools }) => { + const noToolUsage = toolChoice === ToolChoiceType.none; + + const connectorInvokeRequest: InvokeAIActionParams = { + system: noToolUsage ? addNoToolUsageDirective(system) : system, + messages: messagesToBedrock(messages), + tools: noToolUsage ? [] : toolsToBedrock(tools), + toolChoice: toolChoiceToBedrock(toolChoice), + temperature: 0, + stopSequences: ['\n\nHuman:'], + }; + + return from( + executor.invoke({ + subAction: 'invokeStream', + subActionParams: connectorInvokeRequest, + }) + ).pipe( + switchMap((response) => { + const readable = response.data as Readable; + return serdeEventstreamIntoObservable(readable); + }), + tap((eventData) => { + if ('modelStreamErrorException' in eventData) { + throw createInferenceInternalError(eventData.modelStreamErrorException.originalMessage); + } + }), + filter((value): value is BedrockChunkMember => { + return 'chunk' in value && value.chunk?.headers?.[':event-type']?.value === 'chunk'; + }), + map((message) => { + return parseSerdeChunkMessage(message.chunk); + }), + processCompletionChunks() + ); + }, +}; + +const toolChoiceToBedrock = ( + toolChoice: ToolOptions['toolChoice'] +): BedrockToolChoice | undefined => { + if (toolChoice === ToolChoiceType.required) { + return { + type: 'any', + }; + } else if (toolChoice === ToolChoiceType.auto) { + return { + type: 'auto', + }; + } else if (typeof toolChoice === 'object') { + return { + type: 'tool', + name: toolChoice.function, + }; + } + // ToolChoiceType.none is not supported by claude + // we are adding a directive to the system instructions instead in that case. + return undefined; +}; + +const toolsToBedrock = (tools: ToolOptions['tools']) => { + return tools + ? Object.entries(tools).map(([toolName, toolDef]) => { + return { + name: toolName, + description: toolDef.description, + input_schema: toolDef.schema ?? { + type: 'object' as const, + properties: {}, + }, + }; + }) + : undefined; +}; + +const messagesToBedrock = (messages: Message[]): BedRockMessage[] => { + return messages.map((message) => { + switch (message.role) { + case MessageRole.User: + return { + role: 'user' as const, + rawContent: [{ type: 'text' as const, text: message.content }], + }; + case MessageRole.Assistant: + return { + role: 'assistant' as const, + rawContent: [ + ...(message.content ? [{ type: 'text' as const, text: message.content }] : []), + ...(message.toolCalls + ? message.toolCalls.map((toolCall) => { + return { + type: 'tool_use' as const, + id: toolCall.toolCallId, + name: toolCall.function.name, + input: ('arguments' in toolCall.function + ? toolCall.function.arguments + : {}) as Record, + }; + }) + : []), + ], + }; + case MessageRole.Tool: + return { + role: 'user' as const, + rawContent: [ + { + type: 'tool_result' as const, + tool_use_id: message.toolCallId, + content: JSON.stringify(message.response), + }, + ], + }; + } + }); +}; diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/index.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/index.ts new file mode 100644 index 0000000000000..01d849e1ea9af --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { bedrockClaudeAdapter } from './bedrock_claude_adapter'; diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.test.ts new file mode 100644 index 0000000000000..6307aecaeefc4 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.test.ts @@ -0,0 +1,336 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { lastValueFrom, of, toArray } from 'rxjs'; +import { processCompletionChunks } from './process_completion_chunks'; +import type { CompletionChunk } from './types'; + +describe('processCompletionChunks', () => { + it('does not emit for a message_start event', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'message_start', + message: 'foo', + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual( + [] + ); + }); + + it('emits the correct value for a content_block_start event with text content ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: 'foo' }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionChunk', + content: 'foo', + tool_calls: [], + }, + ]); + }); + + it('emits the correct value for a content_block_start event with tool_use content ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_start', + index: 0, + content_block: { type: 'tool_use', id: 'id', name: 'name', input: '{}' }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionChunk', + content: '', + tool_calls: [ + { + toolCallId: 'id', + index: 0, + function: { + arguments: '', + name: 'name', + }, + }, + ], + }, + ]); + }); + + it('emits the correct value for a content_block_delta event with text content ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'delta' }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionChunk', + content: 'delta', + tool_calls: [], + }, + ]); + }); + + it('emits the correct value for a content_block_delta event with tool_use content ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_delta', + index: 0, + delta: { type: 'input_json_delta', partial_json: '{ "param' }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionChunk', + content: '', + tool_calls: [ + { + index: 0, + toolCallId: '', + function: { + arguments: '{ "param', + name: '', + }, + }, + ], + }, + ]); + }); + + it('does not emit for a content_block_stop event', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_stop', + index: 0, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual( + [] + ); + }); + + it('emits the correct value for a message_delta event with tool_use content ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'message_delta', + delta: { stop_reason: 'end_turn', stop_sequence: 'stop_seq', usage: { output_tokens: 42 } }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionChunk', + content: 'stop_seq', + tool_calls: [], + }, + ]); + }); + + it('emits a token count for a message_stop event ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'message_stop', + 'amazon-bedrock-invocationMetrics': { + inputTokenCount: 1, + outputTokenCount: 2, + invocationLatency: 3, + firstByteLatency: 4, + }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionTokenCount', + tokens: { + completion: 2, + prompt: 1, + total: 3, + }, + }, + ]); + }); + + it('emits the correct values for a text response scenario', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'message_start', + message: 'foo', + }, + { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: 'foo' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'delta1' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'delta2' }, + }, + { + type: 'content_block_stop', + index: 0, + }, + { + type: 'message_delta', + delta: { stop_reason: 'end_turn', stop_sequence: 'stop_seq', usage: { output_tokens: 42 } }, + }, + { + type: 'message_stop', + 'amazon-bedrock-invocationMetrics': { + inputTokenCount: 1, + outputTokenCount: 2, + invocationLatency: 3, + firstByteLatency: 4, + }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + content: 'foo', + tool_calls: [], + type: 'chatCompletionChunk', + }, + { + content: 'delta1', + tool_calls: [], + type: 'chatCompletionChunk', + }, + { + content: 'delta2', + tool_calls: [], + type: 'chatCompletionChunk', + }, + { + content: 'stop_seq', + tool_calls: [], + type: 'chatCompletionChunk', + }, + { + tokens: { + completion: 2, + prompt: 1, + total: 3, + }, + type: 'chatCompletionTokenCount', + }, + ]); + }); + + it('emits the correct values for a tool_use response scenario', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'message_start', + message: 'foo', + }, + { + type: 'content_block_start', + index: 0, + content_block: { type: 'tool_use', id: 'id', name: 'name', input: '{}' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'input_json_delta', partial_json: '{ "param' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'input_json_delta', partial_json: '": 12 }' }, + }, + { + type: 'content_block_stop', + index: 0, + }, + { + type: 'message_delta', + delta: { stop_reason: 'tool_use', stop_sequence: null, usage: { output_tokens: 42 } }, + }, + { + type: 'message_stop', + 'amazon-bedrock-invocationMetrics': { + inputTokenCount: 1, + outputTokenCount: 2, + invocationLatency: 3, + firstByteLatency: 4, + }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + content: '', + tool_calls: [ + { + function: { + arguments: '', + name: 'name', + }, + index: 0, + toolCallId: 'id', + }, + ], + type: 'chatCompletionChunk', + }, + { + content: '', + tool_calls: [ + { + function: { + arguments: '{ "param', + name: '', + }, + index: 0, + toolCallId: '', + }, + ], + type: 'chatCompletionChunk', + }, + { + content: '', + tool_calls: [ + { + function: { + arguments: '": 12 }', + name: '', + }, + index: 0, + toolCallId: '', + }, + ], + type: 'chatCompletionChunk', + }, + { + tokens: { + completion: 2, + prompt: 1, + total: 3, + }, + type: 'chatCompletionTokenCount', + }, + ]); + }); +}); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.ts new file mode 100644 index 0000000000000..5513cc9028ac9 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.ts @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Observable, Subscriber } from 'rxjs'; +import { + ChatCompletionChunkEvent, + ChatCompletionTokenCountEvent, + ChatCompletionChunkToolCall, + ChatCompletionEventType, +} from '../../../../common/chat_complete'; +import type { CompletionChunk, MessageStopChunk } from './types'; + +export function processCompletionChunks() { + return (source: Observable) => + new Observable((subscriber) => { + function handleNext(chunkBody: CompletionChunk) { + if (isTokenCountCompletionChunk(chunkBody)) { + return emitTokenCountEvent(subscriber, chunkBody); + } + + let completionChunk = ''; + let toolCallChunk: ChatCompletionChunkToolCall | undefined; + + switch (chunkBody.type) { + case 'content_block_start': + if (chunkBody.content_block.type === 'text') { + completionChunk = chunkBody.content_block.text || ''; + } else if (chunkBody.content_block.type === 'tool_use') { + toolCallChunk = { + index: chunkBody.index, + toolCallId: chunkBody.content_block.id, + function: { + name: chunkBody.content_block.name, + // the API returns '{}' here, which can't be merged with the deltas... + arguments: '', + }, + }; + } + break; + + case 'content_block_delta': + if (chunkBody.delta.type === 'text_delta') { + completionChunk = chunkBody.delta.text || ''; + } else if (chunkBody.delta.type === 'input_json_delta') { + toolCallChunk = { + index: chunkBody.index, + toolCallId: '', + function: { + name: '', + arguments: chunkBody.delta.partial_json, + }, + }; + } + break; + + case 'message_delta': + completionChunk = chunkBody.delta.stop_sequence || ''; + break; + + default: + break; + } + + if (completionChunk || toolCallChunk) { + subscriber.next({ + type: ChatCompletionEventType.ChatCompletionChunk, + content: completionChunk, + tool_calls: toolCallChunk ? [toolCallChunk] : [], + }); + } + } + + source.subscribe({ + next: (value) => { + try { + handleNext(value); + } catch (error) { + subscriber.error(error); + } + }, + error: (err) => { + subscriber.error(err); + }, + complete: () => { + subscriber.complete(); + }, + }); + }); +} + +function isTokenCountCompletionChunk(value: CompletionChunk): value is MessageStopChunk { + return value.type === 'message_stop' && 'amazon-bedrock-invocationMetrics' in value; +} + +function emitTokenCountEvent( + subscriber: Subscriber, + chunk: MessageStopChunk +) { + const { inputTokenCount, outputTokenCount } = chunk['amazon-bedrock-invocationMetrics']; + + subscriber.next({ + type: ChatCompletionEventType.ChatCompletionTokenCount, + tokens: { + completion: outputTokenCount, + prompt: inputTokenCount, + total: inputTokenCount + outputTokenCount, + }, + }); +} diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/prompts.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/prompts.ts new file mode 100644 index 0000000000000..ed8387bf75252 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/prompts.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const noToolUsageDirective = ` +Please answer with text. You should NOT call or use a tool, even if tools might be available and even if +the user explicitly asks for it. DO NOT UNDER ANY CIRCUMSTANCES call a tool. Instead, ALWAYS reply with text. +`; + +export const addNoToolUsageDirective = (systemMessage: string | undefined): string => { + return systemMessage ? `${systemMessage}\n\n${noToolUsageDirective}` : noToolUsageDirective; +}; diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.test.ts new file mode 100644 index 0000000000000..bed6458a94dc7 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.test.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Readable } from 'stream'; +import { Observable, toArray, firstValueFrom, map, filter } from 'rxjs'; +import { + BedrockChunkMember, + BedrockStreamMember, + serdeEventstreamIntoObservable, +} from './serde_eventstream_into_observable'; +import { EventStreamMarshaller } from '@smithy/eventstream-serde-node'; +import { fromUtf8, toUtf8 } from '@smithy/util-utf8'; +import type { CompletionChunk } from './types'; +import { parseSerdeChunkMessage, serializeSerdeChunkMessage } from './serde_utils'; + +describe('serdeEventstreamIntoObservable', () => { + const marshaller = new EventStreamMarshaller({ + utf8Encoder: toUtf8, + utf8Decoder: fromUtf8, + }); + + const getSerdeEventStream = (chunks: CompletionChunk[]) => { + const input = Readable.from(chunks); + return marshaller.serialize(input, serializeSerdeChunkMessage); + }; + + const getChunks = async (serde$: Observable) => { + return await firstValueFrom( + serde$.pipe( + filter((value): value is BedrockChunkMember => { + return 'chunk' in value && value.chunk?.headers?.[':event-type']?.value === 'chunk'; + }), + map((message) => { + return parseSerdeChunkMessage(message.chunk); + }), + toArray() + ) + ); + }; + + it('converts a single chunk', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'Hello' }, + }, + ]; + + const inputStream = getSerdeEventStream(chunks); + const serde$ = serdeEventstreamIntoObservable(inputStream); + + const result = await getChunks(serde$); + + expect(result).toEqual(chunks); + }); + + it('converts multiple chunks', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: 'start' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'Hello' }, + }, + { + type: 'content_block_stop', + index: 0, + }, + ]; + + const inputStream = getSerdeEventStream(chunks); + const serde$ = serdeEventstreamIntoObservable(inputStream); + + const result = await getChunks(serde$); + + expect(result).toEqual(chunks); + }); +}); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.ts new file mode 100644 index 0000000000000..24a245ab2efcc --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EventStreamMarshaller } from '@smithy/eventstream-serde-node'; +import { fromUtf8, toUtf8 } from '@smithy/util-utf8'; +import { identity } from 'lodash'; +import { Observable } from 'rxjs'; +import { Readable } from 'stream'; +import { Message } from '@smithy/types'; +import { createInferenceInternalError } from '../../../../common/errors'; + +interface ModelStreamErrorException { + name: 'ModelStreamErrorException'; + originalStatusCode?: number; + originalMessage?: string; +} + +export interface BedrockChunkMember { + chunk: Message; +} + +export interface ModelStreamErrorExceptionMember { + modelStreamErrorException: ModelStreamErrorException; +} + +export type BedrockStreamMember = BedrockChunkMember | ModelStreamErrorExceptionMember; + +// AWS uses SerDe to send over serialized data, so we use their +// @smithy library to parse the stream data + +export function serdeEventstreamIntoObservable( + readable: Readable +): Observable { + return new Observable((subscriber) => { + const marshaller = new EventStreamMarshaller({ + utf8Encoder: toUtf8, + utf8Decoder: fromUtf8, + }); + + async function processStream() { + for await (const chunk of marshaller.deserialize(readable, identity)) { + if (chunk) { + subscriber.next(chunk); + } + } + } + + processStream().then( + () => { + subscriber.complete(); + }, + (error) => { + if (!(error instanceof Error)) { + try { + const exceptionType = error.headers[':exception-type'].value; + const body = toUtf8(error.body); + let message = `Encountered error in Bedrock stream of type ${exceptionType}`; + try { + message += '\n' + JSON.parse(body).message; + } catch (parseError) { + // trap + } + error = createInferenceInternalError(message); + } catch (decodeError) { + error = createInferenceInternalError(decodeError.message); + } + } + subscriber.error(error); + } + ); + }); +} diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.test.ts new file mode 100644 index 0000000000000..c763fd8c9daf3 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.test.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CompletionChunk } from './types'; +import { serializeSerdeChunkMessage, parseSerdeChunkMessage } from './serde_utils'; + +describe('parseSerdeChunkMessage', () => { + it('parses a serde chunk message', () => { + const chunk: CompletionChunk = { + type: 'content_block_stop', + index: 0, + }; + + expect(parseSerdeChunkMessage(serializeSerdeChunkMessage(chunk))).toEqual(chunk); + }); +}); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.ts new file mode 100644 index 0000000000000..d7050b7744940 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { toUtf8, fromUtf8 } from '@smithy/util-utf8'; +import type { Message } from '@smithy/types'; +import type { CompletionChunk } from './types'; + +/** + * Extract the completion chunk from a chunk message + */ +export function parseSerdeChunkMessage(chunk: Message): CompletionChunk { + return JSON.parse(Buffer.from(JSON.parse(toUtf8(chunk.body)).bytes, 'base64').toString('utf-8')); +} + +/** + * Reverse `parseSerdeChunkMessage` + */ +export const serializeSerdeChunkMessage = (input: CompletionChunk): Message => { + const b64 = Buffer.from(JSON.stringify(input), 'utf-8').toString('base64'); + const body = fromUtf8(JSON.stringify({ bytes: b64 })); + return { + headers: { + ':event-type': { type: 'string', value: 'chunk' }, + ':content-type': { type: 'string', value: 'application/json' }, + ':message-type': { type: 'string', value: 'event' }, + }, + body, + }; +}; diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/types.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/types.ts new file mode 100644 index 0000000000000..f0937a8d8ec18 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/types.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * BedRock message as expected by the bedrock connector + */ +export interface BedRockMessage { + role: 'user' | 'assistant'; + content?: string; + rawContent?: BedRockMessagePart[]; +} + +/** + * Bedrock message parts + */ +export type BedRockMessagePart = + | { type: 'text'; text: string } + | { + type: 'tool_use'; + id: string; + name: string; + input: Record; + } + | { type: 'tool_result'; tool_use_id: string; content: string }; + +export type BedrockToolChoice = { type: 'auto' } | { type: 'any' } | { type: 'tool'; name: string }; + +interface CompletionChunkBase { + type: string; +} + +export interface MessageStartChunk extends CompletionChunkBase { + type: 'message_start'; + message: unknown; +} + +export interface ContentBlockStartChunk extends CompletionChunkBase { + type: 'content_block_start'; + index: number; + content_block: + | { + type: 'text'; + text: string; + } + | { type: 'tool_use'; id: string; name: string; input: string }; +} + +export interface ContentBlockDeltaChunk extends CompletionChunkBase { + type: 'content_block_delta'; + index: number; + delta: + | { + type: 'text_delta'; + text: string; + } + | { + type: 'input_json_delta'; + partial_json: string; + }; +} + +export interface ContentBlockStopChunk extends CompletionChunkBase { + type: 'content_block_stop'; + index: number; +} + +export interface MessageDeltaChunk extends CompletionChunkBase { + type: 'message_delta'; + delta: { + stop_reason: string; + stop_sequence: null | string; + usage: { + output_tokens: number; + }; + }; +} + +export interface MessageStopChunk extends CompletionChunkBase { + type: 'message_stop'; + 'amazon-bedrock-invocationMetrics': { + inputTokenCount: number; + outputTokenCount: number; + invocationLatency: number; + firstByteLatency: number; + }; +} + +export type CompletionChunk = + | MessageStartChunk + | ContentBlockStartChunk + | ContentBlockDeltaChunk + | ContentBlockStopChunk + | MessageDeltaChunk + | MessageStopChunk; diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.test.ts index 9e0b0da6d5894..558e0cd06ef91 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.test.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.test.ts @@ -9,6 +9,7 @@ import { InferenceConnectorType } from '../../../common/connectors'; import { getInferenceAdapter } from './get_inference_adapter'; import { openAIAdapter } from './openai'; import { geminiAdapter } from './gemini'; +import { bedrockClaudeAdapter } from './bedrock'; describe('getInferenceAdapter', () => { it('returns the openAI adapter for OpenAI type', () => { @@ -19,7 +20,7 @@ describe('getInferenceAdapter', () => { expect(getInferenceAdapter(InferenceConnectorType.Gemini)).toBe(geminiAdapter); }); - it('returns undefined for Bedrock type', () => { - expect(getInferenceAdapter(InferenceConnectorType.Bedrock)).toBe(undefined); + it('returns the bedrock adapter for Bedrock type', () => { + expect(getInferenceAdapter(InferenceConnectorType.Bedrock)).toBe(bedrockClaudeAdapter); }); }); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.ts b/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.ts index 0538d828a473a..f34b0c27a339f 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.ts @@ -9,6 +9,7 @@ import { InferenceConnectorType } from '../../../common/connectors'; import type { InferenceConnectorAdapter } from '../types'; import { openAIAdapter } from './openai'; import { geminiAdapter } from './gemini'; +import { bedrockClaudeAdapter } from './bedrock'; export const getInferenceAdapter = ( connectorType: InferenceConnectorType @@ -21,8 +22,7 @@ export const getInferenceAdapter = ( return geminiAdapter; case InferenceConnectorType.Bedrock: - // not implemented yet - break; + return bedrockClaudeAdapter; } return undefined; diff --git a/x-pack/plugins/inference/tsconfig.json b/x-pack/plugins/inference/tsconfig.json index 16d7ca041582c..593556c8f39c8 100644 --- a/x-pack/plugins/inference/tsconfig.json +++ b/x-pack/plugins/inference/tsconfig.json @@ -22,6 +22,7 @@ "@kbn/logging", "@kbn/core-http-server", "@kbn/actions-plugin", - "@kbn/config-schema" + "@kbn/config-schema", + "@kbn/stack-connectors-plugin" ] } diff --git a/x-pack/plugins/ml/server/routes/schemas/modules.ts b/x-pack/plugins/ml/server/routes/schemas/modules.ts index 50fe45b653e42..8cb6bfdc96fd8 100644 --- a/x-pack/plugins/ml/server/routes/schemas/modules.ts +++ b/x-pack/plugins/ml/server/routes/schemas/modules.ts @@ -138,11 +138,11 @@ const moduleSchema = schema.object({ type: schema.string(), logo: schema.maybe(schema.any()), logoFile: schema.maybe(schema.string()), - defaultIndexPattern: schema.string(), - query: schema.any(), + defaultIndexPattern: schema.maybe(schema.string()), + query: schema.maybe(schema.any()), jobs: schema.arrayOf(schema.any()), datafeeds: schema.arrayOf(schema.any()), - kibana: schema.any(), + kibana: schema.maybe(schema.any()), tags: schema.maybe(schema.arrayOf(schema.string())), }); @@ -157,7 +157,7 @@ export const dataRecognizerConfigResponse = () => schema.object({ datafeeds: schema.arrayOf(schema.any()), jobs: schema.arrayOf(schema.any()), - kibana: schema.any(), + kibana: schema.maybe(schema.any()), }); export const jobExistsResponse = () => diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts new file mode 100644 index 0000000000000..7fd0bb3c5ee18 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition } from '@kbn/entities-schema'; +import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { installEntityDefinition } from './entities/install_entity_definition'; +import { startTransform } from './entities/start_transform'; +import { findEntityDefinitions } from './entities/find_entity_definition'; +import { uninstallEntityDefinition } from './entities/uninstall_entity_definition'; +import { EntityDefinitionNotFound } from './entities/errors/entity_not_found'; + +export class EntityClient { + constructor( + private options: { + esClient: ElasticsearchClient; + soClient: SavedObjectsClientContract; + logger: Logger; + } + ) {} + + async createEntityDefinition({ + definition, + installOnly = false, + }: { + definition: EntityDefinition; + installOnly?: boolean; + }) { + const installedDefinition = await installEntityDefinition({ + definition, + soClient: this.options.soClient, + esClient: this.options.esClient, + logger: this.options.logger, + }); + + if (!installOnly) { + await startTransform(this.options.esClient, definition, this.options.logger); + } + + return installedDefinition; + } + + async deleteEntityDefinition({ id, deleteData = false }: { id: string; deleteData?: boolean }) { + const [definition] = await findEntityDefinitions({ + id, + perPage: 1, + soClient: this.options.soClient, + esClient: this.options.esClient, + }); + + if (!definition) { + const message = `Unable to find entity definition with [${id}]`; + this.options.logger.error(message); + throw new EntityDefinitionNotFound(message); + } + + await uninstallEntityDefinition({ + definition, + deleteData, + soClient: this.options.soClient, + esClient: this.options.esClient, + logger: this.options.logger, + }); + } + + async getEntityDefinitions({ page = 1, perPage = 10 }: { page?: number; perPage?: number }) { + const definitions = await findEntityDefinitions({ + esClient: this.options.esClient, + soClient: this.options.soClient, + page, + perPage, + }); + + return { definitions }; + } +} diff --git a/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts b/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts index de9e8bec2826f..d65a2a9e186f3 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts @@ -14,6 +14,7 @@ import { PluginInitializerContext, PluginConfigDescriptor, Logger, + KibanaRequest, } from '@kbn/core/server'; import { installEntityManagerTemplates } from './lib/manage_index_templates'; import { setupRoutes } from './routes'; @@ -26,6 +27,7 @@ import { EntityManagerConfig, configSchema, exposeToBrowserConfig } from '../com import { entityDefinition, EntityDiscoveryApiKeyType } from './saved_objects'; import { upgradeBuiltInEntityDefinitions } from './lib/entities/upgrade_entity_definition'; import { builtInDefinitions } from './lib/entities/built_in'; +import { EntityClient } from './lib/entity_client'; export type EntityManagerServerPluginSetup = ReturnType; export type EntityManagerServerPluginStart = ReturnType; @@ -73,6 +75,12 @@ export class EntityManagerServerPlugin router, logger: this.logger, server: this.server, + getScopedClient: async ({ request }: { request: KibanaRequest }) => { + const [coreStart] = await core.getStartServices(); + const esClient = coreStart.elasticsearch.client.asScoped(request).asCurrentUser; + const soClient = coreStart.savedObjects.getScopedClient(request); + return new EntityClient({ esClient, soClient, logger: this.logger }); + }, }); return {}; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/create.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/create.ts index 973ba3507c455..62a2b88cd99f8 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/create.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/create.ts @@ -16,8 +16,6 @@ import { SetupRouteOptions } from '../types'; import { EntityIdConflict } from '../../lib/entities/errors/entity_id_conflict_error'; import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception'; import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; -import { startTransform } from '../../lib/entities/start_transform'; -import { installEntityDefinition } from '../../lib/entities/install_entity_definition'; import { EntityDefinitionIdInvalid } from '../../lib/entities/errors/entity_definition_id_invalid'; /** @@ -56,7 +54,8 @@ import { EntityDefinitionIdInvalid } from '../../lib/entities/errors/entity_defi */ export function createEntityDefinitionRoute({ router, - server, + getScopedClient, + logger, }: SetupRouteOptions) { router.post( { @@ -66,24 +65,14 @@ export function createEntityDefinitionRoute({ query: createEntityDefinitionQuerySchema, }, }, - async (context, req, res) => { - const { logger } = server; - const core = await context.core; - const soClient = core.savedObjects.client; - const esClient = core.elasticsearch.client.asCurrentUser; - + async (context, request, res) => { try { - const definition = await installEntityDefinition({ - soClient, - esClient, - logger, - definition: req.body, + const client = await getScopedClient({ request }); + const definition = await client.createEntityDefinition({ + definition: request.body, + installOnly: request.query.installOnly, }); - if (!req.query.installOnly) { - await startTransform(esClient, definition, logger); - } - return res.ok({ body: definition }); } catch (e) { logger.error(e); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/delete.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/delete.ts index d0748e5b52e67..c2798aef9eb14 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/delete.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/delete.ts @@ -13,9 +13,7 @@ import { import { SetupRouteOptions } from '../types'; import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception'; import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; -import { readEntityDefinition } from '../../lib/entities/read_entity_definition'; import { EntityDefinitionNotFound } from '../../lib/entities/errors/entity_not_found'; -import { uninstallEntityDefinition } from '../../lib/entities/uninstall_entity_definition'; /** * @openapi @@ -53,7 +51,7 @@ import { uninstallEntityDefinition } from '../../lib/entities/uninstall_entity_d */ export function deleteEntityDefinitionRoute({ router, - server, + getScopedClient, logger, }: SetupRouteOptions) { router.delete<{ id: string }, { deleteData?: boolean }, unknown>( @@ -64,18 +62,12 @@ export function deleteEntityDefinitionRoute({ query: deleteEntityDefinitionQuerySchema.strict(), }, }, - async (context, req, res) => { + async (context, request, res) => { try { - const soClient = (await context.core).savedObjects.client; - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - - const definition = await readEntityDefinition(soClient, req.params.id, logger); - await uninstallEntityDefinition({ - definition, - soClient, - esClient, - logger, - deleteData: req.query.deleteData, + const client = await getScopedClient({ request }); + await client.deleteEntityDefinition({ + id: request.params.id, + deleteData: request.query.deleteData, }); return res.ok({ body: { acknowledged: true } }); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts index 8039ee176a9b1..454679779c6a9 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts @@ -9,7 +9,6 @@ import { z } from '@kbn/zod'; import { RequestHandlerContext } from '@kbn/core/server'; import { getEntityDefinitionQuerySchema } from '@kbn/entities-schema'; import { SetupRouteOptions } from '../types'; -import { findEntityDefinitions } from '../../lib/entities/find_entity_definition'; /** * @openapi @@ -52,6 +51,7 @@ import { findEntityDefinitions } from '../../lib/entities/find_entity_definition */ export function getEntityDefinitionRoute({ router, + getScopedClient, logger, }: SetupRouteOptions) { router.get<{ id?: string }, { page?: number; perPage?: number }, unknown>( @@ -62,18 +62,15 @@ export function getEntityDefinitionRoute({ params: z.object({ id: z.optional(z.string()) }), }, }, - async (context, req, res) => { + async (context, request, res) => { try { - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const soClient = (await context.core).savedObjects.client; - const definitions = await findEntityDefinitions({ - esClient, - soClient, - page: req.query.page ?? 1, - perPage: req.query.perPage ?? 10, - id: req.params.id, + const client = await getScopedClient({ request }); + const result = await client.getEntityDefinitions({ + page: request.query.page, + perPage: request.query.perPage, }); - return res.ok({ body: { definitions } }); + + return res.ok({ body: result }); } catch (e) { logger.error(e); return res.customError({ body: e, statusCode: 500 }); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/types.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/types.ts index 8e3dc7111298d..d4d8cfba815ae 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/types.ts @@ -5,12 +5,14 @@ * 2.0. */ -import { IRouter, RequestHandlerContextBase } from '@kbn/core-http-server'; +import { IRouter, KibanaRequest, RequestHandlerContextBase } from '@kbn/core-http-server'; import { Logger } from '@kbn/core/server'; import { EntityManagerServerSetup } from '../types'; +import { EntityClient } from '../lib/entity_client'; export interface SetupRouteOptions { router: IRouter; server: EntityManagerServerSetup; logger: Logger; + getScopedClient: ({ request }: { request: KibanaRequest }) => Promise; } diff --git a/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx b/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx index f1e0b9073c036..49dfd3d42d5c4 100644 --- a/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx +++ b/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx @@ -90,7 +90,7 @@ export const AddMetadataFilterButton = ({ item }: AddMetadataFilterButtonProps) } return ( - + { + it('returns empty string if no values are provided', () => { + expect(getSavedObjectKqlFilter({ field: 'tags' })).toBe(''); + }); + + it('returns KQL string if values are provided', () => { + expect(getSavedObjectKqlFilter({ field: 'tags', values: 'apm' })).toBe( + 'synthetics-monitor.attributes.tags:"apm"' + ); + }); + + it('searches at root when specified', () => { + expect(getSavedObjectKqlFilter({ field: 'tags', values: 'apm', searchAtRoot: true })).toBe( + 'tags:"apm"' + ); + }); + + it('handles array values', () => { + expect(getSavedObjectKqlFilter({ field: 'tags', values: ['apm', 'synthetics'] })).toBe( + 'synthetics-monitor.attributes.tags:("apm" OR "synthetics")' + ); + }); + + it('escapes quotes', () => { + expect(getSavedObjectKqlFilter({ field: 'tags', values: ['"apm', 'synthetics'] })).toBe( + 'synthetics-monitor.attributes.tags:("\\"apm" OR "synthetics")' + ); + }); +}); diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/common.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/common.ts index 491b67160677e..bc58a866bef83 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/common.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/common.ts @@ -7,6 +7,7 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { SavedObjectsFindResponse } from '@kbn/core/server'; +import { escapeQuotes } from '@kbn/es-query'; import { RouteContext } from './types'; import { MonitorSortFieldSchema } from '../../common/runtime_types/monitor_management/sort_field'; import { getAllLocations } from '../synthetics_service/get_all_locations'; @@ -132,19 +133,19 @@ export const getMonitorFilters = async ({ const filtersStr = [ filter, - getKqlFilter({ field: 'tags', values: tags }), - getKqlFilter({ field: 'project_id', values: projects }), - getKqlFilter({ field: 'type', values: monitorTypes }), - getKqlFilter({ field: 'locations.id', values: locationFilter }), - getKqlFilter({ field: 'schedule.number', values: schedules }), - getKqlFilter({ field: 'id', values: monitorQueryIds }), + getSavedObjectKqlFilter({ field: 'tags', values: tags }), + getSavedObjectKqlFilter({ field: 'project_id', values: projects }), + getSavedObjectKqlFilter({ field: 'type', values: monitorTypes }), + getSavedObjectKqlFilter({ field: 'locations.id', values: locationFilter }), + getSavedObjectKqlFilter({ field: 'schedule.number', values: schedules }), + getSavedObjectKqlFilter({ field: 'id', values: monitorQueryIds }), ] .filter((f) => !!f) .join(' AND '); return { filtersStr, locationFilter }; }; -export const getKqlFilter = ({ +export const getSavedObjectKqlFilter = ({ field, values, operator = 'OR', @@ -166,10 +167,12 @@ export const getKqlFilter = ({ } if (Array.isArray(values)) { - return ` (${fieldKey}:"${values.join(`" ${operator} ${fieldKey}:"`)}" )`; + return `${fieldKey}:(${values + .map((value) => `"${escapeQuotes(value)}"`) + .join(` ${operator} `)})`; } - return `${fieldKey}:"${values}"`; + return `${fieldKey}:"${escapeQuotes(values)}"`; }; const parseLocationFilter = async (context: RouteContext, locations?: string | string[]) => { diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor/add_monitor_api.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor/add_monitor_api.ts index 1c38e093237e1..359f3373cfd35 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor/add_monitor_api.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor/add_monitor_api.ts @@ -12,7 +12,7 @@ import { isValidNamespace } from '@kbn/fleet-plugin/common'; import { i18n } from '@kbn/i18n'; import { parseMonitorLocations } from './utils'; import { MonitorValidationError } from '../monitor_validation'; -import { getKqlFilter } from '../../common'; +import { getSavedObjectKqlFilter } from '../../common'; import { deleteMonitor } from '../delete_monitor'; import { monitorAttributes, syntheticsMonitorType } from '../../../../common/types/saved_objects'; import { PrivateLocationAttributes } from '../../../runtime_types/private_locations'; @@ -238,7 +238,7 @@ export class AddEditMonitorAPI { async validateUniqueMonitorName(name: string, id?: string) { const { savedObjectsClient } = this.routeContext; - const kqlFilter = getKqlFilter({ field: 'name.keyword', values: name }); + const kqlFilter = getSavedObjectKqlFilter({ field: 'name.keyword', values: name }); const { total } = await savedObjectsClient.find({ perPage: 0, type: syntheticsMonitorType, diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts index b5c8dc2b4f0b1..2136634be7ef7 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts @@ -10,7 +10,7 @@ import { SyntheticsRestApiRouteFactory } from '../types'; import { syntheticsMonitorType } from '../../../common/types/saved_objects'; import { ConfigKey } from '../../../common/runtime_types'; import { SYNTHETICS_API_URLS } from '../../../common/constants'; -import { getMonitors, getKqlFilter } from '../common'; +import { getMonitors, getSavedObjectKqlFilter } from '../common'; import { deleteMonitorBulk } from './bulk_cruds/delete_monitor_bulk'; export const deleteSyntheticsMonitorProjectRoute: SyntheticsRestApiRouteFactory = () => ({ @@ -39,7 +39,7 @@ export const deleteSyntheticsMonitorProjectRoute: SyntheticsRestApiRouteFactory const deleteFilter = `${syntheticsMonitorType}.attributes.${ ConfigKey.PROJECT_ID - }: "${decodedProjectName}" AND ${getKqlFilter({ + }: "${decodedProjectName}" AND ${getSavedObjectKqlFilter({ field: 'journey_id', values: monitorsToDelete.map((id: string) => `${id}`), })}`; diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/settings/private_locations/get_location_monitors.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/settings/private_locations/get_location_monitors.ts index de96ba8eb9438..6701946c8a6d6 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/settings/private_locations/get_location_monitors.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/settings/private_locations/get_location_monitors.ts @@ -6,7 +6,7 @@ */ import { ALL_SPACES_ID } from '@kbn/spaces-plugin/common/constants'; -import { getKqlFilter } from '../../common'; +import { getSavedObjectKqlFilter } from '../../common'; import { SyntheticsRestApiRouteFactory } from '../../types'; import { SYNTHETICS_API_URLS } from '../../../../common/constants'; import { monitorAttributes, syntheticsMonitorType } from '../../../../common/types/saved_objects'; @@ -47,7 +47,7 @@ export const getLocationMonitors: SyntheticsRestApiRouteFactory = () => export const getMonitorsByLocation = async (server: SyntheticsServerSetup, locationId?: string) => { const soClient = server.coreStart.savedObjects.createInternalRepository(); - const locationFilter = getKqlFilter({ field: 'locations.id', values: locationId }); + const locationFilter = getSavedObjectKqlFilter({ field: 'locations.id', values: locationId }); const locationMonitors = await soClient.find({ type: syntheticsMonitorType, diff --git a/x-pack/plugins/observability_solution/synthetics/server/synthetics_service/project_monitor/project_monitor_formatter.ts b/x-pack/plugins/observability_solution/synthetics/server/synthetics_service/project_monitor/project_monitor_formatter.ts index fd59e7a0ec40a..6069076e375a2 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/synthetics_service/project_monitor/project_monitor_formatter.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/synthetics_service/project_monitor/project_monitor_formatter.ts @@ -11,7 +11,7 @@ import { } from '@kbn/core/server'; import { i18n } from '@kbn/i18n'; import { EncryptedSavedObjectsClient } from '@kbn/encrypted-saved-objects-plugin/server'; -import { getKqlFilter } from '../../routes/common'; +import { getSavedObjectKqlFilter } from '../../routes/common'; import { InvalidLocationError } from './normalizers/common_fields'; import { SyntheticsServerSetup } from '../../types'; import { RouteContext } from '../../routes/types'; @@ -337,7 +337,10 @@ export class ProjectMonitorFormatter { monitors: Array> ) => { const configIds = monitors.map((monitor) => monitor.attributes[ConfigKey.CONFIG_ID]); - const monitorFilter = getKqlFilter({ field: ConfigKey.CONFIG_ID, values: configIds }); + const monitorFilter = getSavedObjectKqlFilter({ + field: ConfigKey.CONFIG_ID, + values: configIds, + }); const finder = await this.encryptedSavedObjectsClient.createPointInTimeFinderDecryptedAsInternalUser( { diff --git a/x-pack/plugins/osquery/common/api/live_query/live_queries.schema.yaml b/x-pack/plugins/osquery/common/api/live_query/live_queries.schema.yaml index 964d0938676a9..fa9fc90742bbb 100644 --- a/x-pack/plugins/osquery/common/api/live_query/live_queries.schema.yaml +++ b/x-pack/plugins/osquery/common/api/live_query/live_queries.schema.yaml @@ -5,7 +5,8 @@ info: paths: /api/osquery/live_queries: get: - summary: Find live queries + summary: Get live queries + description: Get a list of all live queries. operationId: OsqueryFindLiveQueries x-codegen-enabled: true x-labels: [serverless, ess] @@ -25,6 +26,7 @@ paths: post: summary: Create a live query + description: Create and run a live query. operationId: OsqueryCreateLiveQuery x-codegen-enabled: true x-labels: [serverless, ess] @@ -45,6 +47,7 @@ paths: /api/osquery/live_queries/{id}: get: summary: Get live query details + description: Get the details of a live query using the query ID. operationId: OsqueryGetLiveQueryDetails x-codegen-enabled: true x-labels: [serverless, ess] @@ -70,6 +73,7 @@ paths: /api/osquery/live_queries/{id}/results/{actionId}: get: summary: Get live query results + description: Get the results of a live query using the query action ID. operationId: OsqueryGetLiveQueryResults x-codegen-enabled: true x-labels: [serverless, ess] diff --git a/x-pack/plugins/osquery/common/api/packs/packs.schema.yaml b/x-pack/plugins/osquery/common/api/packs/packs.schema.yaml index bc9afa2ff0964..7bdb90151b73d 100644 --- a/x-pack/plugins/osquery/common/api/packs/packs.schema.yaml +++ b/x-pack/plugins/osquery/common/api/packs/packs.schema.yaml @@ -5,7 +5,8 @@ info: paths: /api/osquery/packs: get: - summary: Find packs + summary: Get packs + description: Get a list of all query packs. operationId: OsqueryFindPacks x-codegen-enabled: true x-labels: [serverless, ess] @@ -23,7 +24,8 @@ paths: schema: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' post: - summary: Create a packs + summary: Create a pack + description: Create a query pack. operationId: OsqueryCreatePacks x-codegen-enabled: true x-labels: [serverless, ess] @@ -42,7 +44,8 @@ paths: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' /api/osquery/packs/{id}: get: - summary: Get packs details + summary: Get pack details + description: Get the details of a query pack using the pack ID. operationId: OsqueryGetPacksDetails x-codegen-enabled: true x-labels: [serverless, ess] @@ -60,7 +63,8 @@ paths: schema: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' delete: - summary: Delete packs + summary: Delete a pack + description: Delete a query pack using the pack ID. operationId: OsqueryDeletePacks x-codegen-enabled: true x-labels: [serverless, ess] @@ -78,7 +82,11 @@ paths: schema: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' put: - summary: Update packs + summary: Update a pack + description: | + Update a query pack using the pack ID. + > info + > You cannot update a prebuilt pack. operationId: OsqueryUpdatePacks x-codegen-enabled: true x-labels: [serverless, ess] diff --git a/x-pack/plugins/osquery/common/api/saved_query/saved_query.schema.yaml b/x-pack/plugins/osquery/common/api/saved_query/saved_query.schema.yaml index 181dbb6350b56..359770016fb82 100644 --- a/x-pack/plugins/osquery/common/api/saved_query/saved_query.schema.yaml +++ b/x-pack/plugins/osquery/common/api/saved_query/saved_query.schema.yaml @@ -5,7 +5,8 @@ info: paths: /api/osquery/saved_queries: get: - summary: Find saved queries + summary: Get saved queries + description: Get a list of all saved queries. operationId: OsqueryFindSavedQueries x-codegen-enabled: true x-labels: [serverless, ess] @@ -24,6 +25,7 @@ paths: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' post: summary: Create a saved query + description: Create and run a saved query. operationId: OsqueryCreateSavedQuery x-codegen-enabled: true x-labels: [serverless, ess] @@ -43,6 +45,7 @@ paths: /api/osquery/saved_queries/{id}: get: summary: Get saved query details + description: Get the details of a saved query using the query ID. operationId: OsqueryGetSavedQueryDetails x-codegen-enabled: true x-labels: [serverless, ess] @@ -60,7 +63,8 @@ paths: schema: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' delete: - summary: Delete saved query + summary: Delete a saved query + description: Delete a saved query using the query ID. operationId: OsqueryDeleteSavedQuery x-codegen-enabled: true x-labels: [serverless, ess] @@ -78,7 +82,11 @@ paths: schema: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' put: - summary: Update saved query + summary: Update a saved query + description: | + Update a saved query using the query ID. + > info + > You cannot update a prebuilt saved query. operationId: OsqueryUpdateSavedQuery x-codegen-enabled: true x-labels: [serverless, ess] diff --git a/x-pack/plugins/osquery/docs/openapi/ess/osquery_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/osquery/docs/openapi/ess/osquery_api_2023_10_31.bundled.schema.yaml index e8635fbc478e4..4f6933aef5f2f 100644 --- a/x-pack/plugins/osquery/docs/openapi/ess/osquery_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/osquery/docs/openapi/ess/osquery_api_2023_10_31.bundled.schema.yaml @@ -13,6 +13,7 @@ servers: paths: /api/osquery/live_queries: get: + description: Get a list of all live queries. operationId: OsqueryFindLiveQueries parameters: - in: query @@ -27,10 +28,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find live queries + summary: Get live queries tags: - Security Solution Osquery API post: + description: Create and run a live query. operationId: OsqueryCreateLiveQuery requestBody: content: @@ -50,6 +52,7 @@ paths: - Security Solution Osquery API '/api/osquery/live_queries/{id}': get: + description: Get the details of a live query using the query ID. operationId: OsqueryGetLiveQueryDetails parameters: - in: path @@ -74,6 +77,7 @@ paths: - Security Solution Osquery API '/api/osquery/live_queries/{id}/results/{actionId}': get: + description: Get the results of a live query using the query action ID. operationId: OsqueryGetLiveQueryResults parameters: - in: path @@ -103,6 +107,7 @@ paths: - Security Solution Osquery API /api/osquery/packs: get: + description: Get a list of all query packs. operationId: OsqueryFindPacks parameters: - in: query @@ -117,10 +122,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find packs + summary: Get packs tags: - Security Solution Osquery API post: + description: Create a query pack. operationId: OsqueryCreatePacks requestBody: content: @@ -135,11 +141,12 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Create a packs + summary: Create a pack tags: - Security Solution Osquery API '/api/osquery/packs/{id}': delete: + description: Delete a query pack using the pack ID. operationId: OsqueryDeletePacks parameters: - in: path @@ -154,10 +161,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Delete packs + summary: Delete a pack tags: - Security Solution Osquery API get: + description: Get the details of a query pack using the pack ID. operationId: OsqueryGetPacksDetails parameters: - in: path @@ -172,10 +180,14 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Get packs details + summary: Get pack details tags: - Security Solution Osquery API put: + description: | + Update a query pack using the pack ID. + > info + > You cannot update a prebuilt pack. operationId: OsqueryUpdatePacks parameters: - in: path @@ -196,11 +208,12 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Update packs + summary: Update a pack tags: - Security Solution Osquery API /api/osquery/saved_queries: get: + description: Get a list of all saved queries. operationId: OsqueryFindSavedQueries parameters: - in: query @@ -215,10 +228,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find saved queries + summary: Get saved queries tags: - Security Solution Osquery API post: + description: Create and run a saved query. operationId: OsqueryCreateSavedQuery requestBody: content: @@ -238,6 +252,7 @@ paths: - Security Solution Osquery API '/api/osquery/saved_queries/{id}': delete: + description: Delete a saved query using the query ID. operationId: OsqueryDeleteSavedQuery parameters: - in: path @@ -252,10 +267,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Delete saved query + summary: Delete a saved query tags: - Security Solution Osquery API get: + description: Get the details of a saved query using the query ID. operationId: OsqueryGetSavedQueryDetails parameters: - in: path @@ -274,6 +290,10 @@ paths: tags: - Security Solution Osquery API put: + description: | + Update a saved query using the query ID. + > info + > You cannot update a prebuilt saved query. operationId: OsqueryUpdateSavedQuery parameters: - in: path @@ -294,7 +314,7 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Update saved query + summary: Update a saved query tags: - Security Solution Osquery API components: diff --git a/x-pack/plugins/osquery/docs/openapi/serverless/osquery_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/osquery/docs/openapi/serverless/osquery_api_2023_10_31.bundled.schema.yaml index 5ee7cc382c480..836298b2e7cba 100644 --- a/x-pack/plugins/osquery/docs/openapi/serverless/osquery_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/osquery/docs/openapi/serverless/osquery_api_2023_10_31.bundled.schema.yaml @@ -13,6 +13,7 @@ servers: paths: /api/osquery/live_queries: get: + description: Get a list of all live queries. operationId: OsqueryFindLiveQueries parameters: - in: query @@ -27,10 +28,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find live queries + summary: Get live queries tags: - Security Solution Osquery API post: + description: Create and run a live query. operationId: OsqueryCreateLiveQuery requestBody: content: @@ -50,6 +52,7 @@ paths: - Security Solution Osquery API '/api/osquery/live_queries/{id}': get: + description: Get the details of a live query using the query ID. operationId: OsqueryGetLiveQueryDetails parameters: - in: path @@ -74,6 +77,7 @@ paths: - Security Solution Osquery API '/api/osquery/live_queries/{id}/results/{actionId}': get: + description: Get the results of a live query using the query action ID. operationId: OsqueryGetLiveQueryResults parameters: - in: path @@ -103,6 +107,7 @@ paths: - Security Solution Osquery API /api/osquery/packs: get: + description: Get a list of all query packs. operationId: OsqueryFindPacks parameters: - in: query @@ -117,10 +122,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find packs + summary: Get packs tags: - Security Solution Osquery API post: + description: Create a query pack. operationId: OsqueryCreatePacks requestBody: content: @@ -135,11 +141,12 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Create a packs + summary: Create a pack tags: - Security Solution Osquery API '/api/osquery/packs/{id}': delete: + description: Delete a query pack using the pack ID. operationId: OsqueryDeletePacks parameters: - in: path @@ -154,10 +161,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Delete packs + summary: Delete a pack tags: - Security Solution Osquery API get: + description: Get the details of a query pack using the pack ID. operationId: OsqueryGetPacksDetails parameters: - in: path @@ -172,10 +180,14 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Get packs details + summary: Get pack details tags: - Security Solution Osquery API put: + description: | + Update a query pack using the pack ID. + > info + > You cannot update a prebuilt pack. operationId: OsqueryUpdatePacks parameters: - in: path @@ -196,11 +208,12 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Update packs + summary: Update a pack tags: - Security Solution Osquery API /api/osquery/saved_queries: get: + description: Get a list of all saved queries. operationId: OsqueryFindSavedQueries parameters: - in: query @@ -215,10 +228,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find saved queries + summary: Get saved queries tags: - Security Solution Osquery API post: + description: Create and run a saved query. operationId: OsqueryCreateSavedQuery requestBody: content: @@ -238,6 +252,7 @@ paths: - Security Solution Osquery API '/api/osquery/saved_queries/{id}': delete: + description: Delete a saved query using the query ID. operationId: OsqueryDeleteSavedQuery parameters: - in: path @@ -252,10 +267,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Delete saved query + summary: Delete a saved query tags: - Security Solution Osquery API get: + description: Get the details of a saved query using the query ID. operationId: OsqueryGetSavedQueryDetails parameters: - in: path @@ -274,6 +290,10 @@ paths: tags: - Security Solution Osquery API put: + description: | + Update a saved query using the query ID. + > info + > You cannot update a prebuilt saved query. operationId: OsqueryUpdateSavedQuery parameters: - in: path @@ -294,7 +314,7 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Update saved query + summary: Update a saved query tags: - Security Solution Osquery API components: diff --git a/x-pack/plugins/stack_connectors/common/bedrock/schema.ts b/x-pack/plugins/stack_connectors/common/bedrock/schema.ts index 093a5f9b11518..03f4f5cc01735 100644 --- a/x-pack/plugins/stack_connectors/common/bedrock/schema.ts +++ b/x-pack/plugins/stack_connectors/common/bedrock/schema.ts @@ -28,13 +28,30 @@ export const RunActionParamsSchema = schema.object({ raw: schema.maybe(schema.boolean()), }); +export const BedrockMessageSchema = schema.object( + { + role: schema.string(), + content: schema.maybe(schema.string()), + rawContent: schema.maybe(schema.arrayOf(schema.any())), + }, + { + validate: (value) => { + if (value.content === undefined && value.rawContent === undefined) { + return 'Must specify either content or rawContent'; + } else if (value.content !== undefined && value.rawContent !== undefined) { + return 'content and rawContent can not be used at the same time'; + } + }, + } +); + +export const BedrockToolChoiceSchema = schema.object({ + type: schema.oneOf([schema.literal('auto'), schema.literal('any'), schema.literal('tool')]), + name: schema.maybe(schema.string()), +}); + export const InvokeAIActionParamsSchema = schema.object({ - messages: schema.arrayOf( - schema.object({ - role: schema.string(), - content: schema.string(), - }) - ), + messages: schema.arrayOf(BedrockMessageSchema), model: schema.maybe(schema.string()), temperature: schema.maybe(schema.number()), stopSequences: schema.maybe(schema.arrayOf(schema.string())), @@ -53,6 +70,7 @@ export const InvokeAIActionParamsSchema = schema.object({ }) ) ), + toolChoice: schema.maybe(BedrockToolChoiceSchema), }); export const InvokeAIActionResponseSchema = schema.object({ @@ -60,12 +78,7 @@ export const InvokeAIActionResponseSchema = schema.object({ }); export const InvokeAIRawActionParamsSchema = schema.object({ - messages: schema.arrayOf( - schema.object({ - role: schema.string(), - content: schema.any(), - }) - ), + messages: schema.arrayOf(BedrockMessageSchema), model: schema.maybe(schema.string()), temperature: schema.maybe(schema.number()), stopSequences: schema.maybe(schema.arrayOf(schema.string())), @@ -84,6 +97,7 @@ export const InvokeAIRawActionParamsSchema = schema.object({ }) ) ), + toolChoice: schema.maybe(BedrockToolChoiceSchema), }); export const InvokeAIRawActionResponseSchema = schema.object({}, { unknowns: 'allow' }); diff --git a/x-pack/plugins/stack_connectors/common/bedrock/types.ts b/x-pack/plugins/stack_connectors/common/bedrock/types.ts index b144f78b91edd..3b02f40d2de62 100644 --- a/x-pack/plugins/stack_connectors/common/bedrock/types.ts +++ b/x-pack/plugins/stack_connectors/common/bedrock/types.ts @@ -19,6 +19,8 @@ import { InvokeAIRawActionResponseSchema, StreamingResponseSchema, RunApiLatestResponseSchema, + BedrockMessageSchema, + BedrockToolChoiceSchema, } from './schema'; export type Config = TypeOf; @@ -33,3 +35,5 @@ export type RunActionResponse = TypeOf; export type StreamingResponse = TypeOf; export type DashboardActionParams = TypeOf; export type DashboardActionResponse = TypeOf; +export type BedRockMessage = TypeOf; +export type BedrockToolChoice = TypeOf; diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts index b5ec114a9c456..c2c773bdeaf87 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts @@ -22,7 +22,7 @@ import { RunActionResponseSchema, RunApiLatestResponseSchema, } from '../../../common/bedrock/schema'; -import { +import type { Config, Secrets, RunActionParams, @@ -32,6 +32,8 @@ import { InvokeAIRawActionParams, InvokeAIRawActionResponse, RunApiLatestResponse, + BedRockMessage, + BedrockToolChoice, } from '../../../common/bedrock/types'; import { SUB_ACTION, @@ -309,13 +311,14 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B signal, timeout, tools, + toolChoice, }: InvokeAIActionParams | InvokeAIRawActionParams, connectorUsageCollector: ConnectorUsageCollector ): Promise { const res = (await this.streamApi( { body: JSON.stringify( - formatBedrockBody({ messages, stopSequences, system, temperature, tools }) + formatBedrockBody({ messages, stopSequences, system, temperature, tools, toolChoice }) ), model, signal, @@ -344,13 +347,23 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B maxTokens, signal, timeout, + tools, + toolChoice, }: InvokeAIActionParams, connectorUsageCollector: ConnectorUsageCollector ): Promise { const res = (await this.runApi( { body: JSON.stringify( - formatBedrockBody({ messages, stopSequences, system, temperature, maxTokens }) + formatBedrockBody({ + messages, + stopSequences, + system, + temperature, + maxTokens, + tools, + toolChoice, + }) ), model, signal, @@ -372,6 +385,7 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B signal, timeout, tools, + toolChoice, anthropicVersion, }: InvokeAIRawActionParams, connectorUsageCollector: ConnectorUsageCollector @@ -385,6 +399,7 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B temperature, max_tokens: maxTokens, tools, + tool_choice: toolChoice, anthropic_version: anthropicVersion, }), model, @@ -405,14 +420,16 @@ const formatBedrockBody = ({ system, maxTokens = DEFAULT_TOKEN_LIMIT, tools, + toolChoice, }: { - messages: Array<{ role: string; content?: string }>; + messages: BedRockMessage[]; stopSequences?: string[]; temperature?: number; maxTokens?: number; // optional system message to be sent to the API system?: string; tools?: Array<{ name: string; description: string }>; + toolChoice?: BedrockToolChoice; }) => ({ anthropic_version: 'bedrock-2023-05-31', ...ensureMessageFormat(messages, system), @@ -420,8 +437,14 @@ const formatBedrockBody = ({ stop_sequences: stopSequences, temperature, tools, + tool_choice: toolChoice, }); +interface FormattedBedRockMessage { + role: string; + content: string | BedRockMessage['rawContent']; +} + /** * Ensures that the messages are in the correct format for the Bedrock API * If 2 user or 2 assistant messages are sent in a row, Bedrock throws an error @@ -429,19 +452,32 @@ const formatBedrockBody = ({ * @param messages */ const ensureMessageFormat = ( - messages: Array<{ role: string; content?: string }>, + messages: BedRockMessage[], systemPrompt?: string -): { messages: Array<{ role: string; content?: string }>; system?: string } => { +): { + messages: FormattedBedRockMessage[]; + system?: string; +} => { let system = systemPrompt ? systemPrompt : ''; - const newMessages = messages.reduce((acc: Array<{ role: string; content?: string }>, m) => { - const lastMessage = acc[acc.length - 1]; + const newMessages = messages.reduce((acc, m) => { if (m.role === 'system') { system = `${system.length ? `${system}\n` : ''}${m.content}`; return acc; } - if (lastMessage && lastMessage.role === m.role) { + const messageRole = () => (['assistant', 'ai'].includes(m.role) ? 'assistant' : 'user'); + + if (m.rawContent) { + acc.push({ + role: messageRole(), + content: m.rawContent, + }); + return acc; + } + + const lastMessage = acc[acc.length - 1]; + if (lastMessage && lastMessage.role === m.role && typeof lastMessage.content === 'string') { // Bedrock only accepts assistant and user roles. // If 2 user or 2 assistant messages are sent in a row, combine the messages into a single message return [ @@ -451,11 +487,9 @@ const ensureMessageFormat = ( } // force role outside of system to ensure it is either assistant or user - return [ - ...acc, - { content: m.content, role: ['assistant', 'ai'].includes(m.role) ? 'assistant' : 'user' }, - ]; + return [...acc, { content: m.content, role: messageRole() }]; }, []); + return system.length ? { system, messages: newMessages } : { messages: newMessages }; }; diff --git a/x-pack/test/api_integration/services/security_solution_osquery_api.gen.ts b/x-pack/test/api_integration/services/security_solution_osquery_api.gen.ts index 1093c30a5d357..c133ebce15c90 100644 --- a/x-pack/test/api_integration/services/security_solution_osquery_api.gen.ts +++ b/x-pack/test/api_integration/services/security_solution_osquery_api.gen.ts @@ -93,6 +93,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Create and run a live query. + */ osqueryCreateLiveQuery(props: OsqueryCreateLiveQueryProps) { return supertest .post('/api/osquery/live_queries') @@ -101,6 +104,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Create a query pack. + */ osqueryCreatePacks(props: OsqueryCreatePacksProps) { return supertest .post('/api/osquery/packs') @@ -109,6 +115,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Create and run a saved query. + */ osqueryCreateSavedQuery(props: OsqueryCreateSavedQueryProps) { return supertest .post('/api/osquery/saved_queries') @@ -117,6 +126,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Delete a query pack using the pack ID. + */ osqueryDeletePacks(props: OsqueryDeletePacksProps) { return supertest .delete(replaceParams('/api/osquery/packs/{id}', props.params)) @@ -124,6 +136,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Delete a saved query using the query ID. + */ osqueryDeleteSavedQuery(props: OsqueryDeleteSavedQueryProps) { return supertest .delete(replaceParams('/api/osquery/saved_queries/{id}', props.params)) @@ -131,6 +146,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Get a list of all live queries. + */ osqueryFindLiveQueries(props: OsqueryFindLiveQueriesProps) { return supertest .get('/api/osquery/live_queries') @@ -139,6 +157,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get a list of all query packs. + */ osqueryFindPacks(props: OsqueryFindPacksProps) { return supertest .get('/api/osquery/packs') @@ -147,6 +168,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get a list of all saved queries. + */ osqueryFindSavedQueries(props: OsqueryFindSavedQueriesProps) { return supertest .get('/api/osquery/saved_queries') @@ -155,6 +179,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get the details of a live query using the query ID. + */ osqueryGetLiveQueryDetails(props: OsqueryGetLiveQueryDetailsProps) { return supertest .get(replaceParams('/api/osquery/live_queries/{id}', props.params)) @@ -163,6 +190,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get the results of a live query using the query action ID. + */ osqueryGetLiveQueryResults(props: OsqueryGetLiveQueryResultsProps) { return supertest .get(replaceParams('/api/osquery/live_queries/{id}/results/{actionId}', props.params)) @@ -171,6 +201,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get the details of a query pack using the pack ID. + */ osqueryGetPacksDetails(props: OsqueryGetPacksDetailsProps) { return supertest .get(replaceParams('/api/osquery/packs/{id}', props.params)) @@ -178,6 +211,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Get the details of a saved query using the query ID. + */ osqueryGetSavedQueryDetails(props: OsqueryGetSavedQueryDetailsProps) { return supertest .get(replaceParams('/api/osquery/saved_queries/{id}', props.params)) @@ -185,6 +221,12 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Update a query pack using the pack ID. +> info +> You cannot update a prebuilt pack. + + */ osqueryUpdatePacks(props: OsqueryUpdatePacksProps) { return supertest .put(replaceParams('/api/osquery/packs/{id}', props.params)) @@ -193,6 +235,12 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Update a saved query using the query ID. +> info +> You cannot update a prebuilt saved query. + + */ osqueryUpdateSavedQuery(props: OsqueryUpdateSavedQueryProps) { return supertest .put(replaceParams('/api/osquery/saved_queries/{id}', props.params)) diff --git a/x-pack/test/functional/apps/infra/hosts_view.ts b/x-pack/test/functional/apps/infra/hosts_view.ts index ccea5affc9202..5f2dd01bb980e 100644 --- a/x-pack/test/functional/apps/infra/hosts_view.ts +++ b/x-pack/test/functional/apps/infra/hosts_view.ts @@ -298,7 +298,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { (await pageObjects.infraHostsView.isKPIChartsLoaded()) ); - describe('Hosts View', function () { + // Failing: See https://github.com/elastic/kibana/issues/191806 + describe.skip('Hosts View', function () { let synthEsInfraClient: InfraSynthtraceEsClient; let syntEsLogsClient: LogsSynthtraceEsClient; diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/index.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/index.ts index 8a2bc1449cd7e..53b21e013c06a 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/index.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/index.ts @@ -12,7 +12,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { const ml = getService('ml'); describe('machine learning - data frame analytics', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); before(async () => { await ml.securityCommon.createMlRoles(); diff --git a/x-pack/test/functional/apps/ml/permissions/index.ts b/x-pack/test/functional/apps/ml/permissions/index.ts index 8b28c9e6ccda4..224544a015d8a 100644 --- a/x-pack/test/functional/apps/ml/permissions/index.ts +++ b/x-pack/test/functional/apps/ml/permissions/index.ts @@ -12,7 +12,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { const ml = getService('ml'); describe('machine learning - permissions', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); before(async () => { await ml.securityCommon.createMlRoles(); diff --git a/x-pack/test/functional/apps/ml/short_tests/model_management/index.ts b/x-pack/test/functional/apps/ml/short_tests/model_management/index.ts index c20957beb1ea5..55cb854c69efd 100644 --- a/x-pack/test/functional/apps/ml/short_tests/model_management/index.ts +++ b/x-pack/test/functional/apps/ml/short_tests/model_management/index.ts @@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('model management', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); loadTestFile(require.resolve('./model_list')); }); diff --git a/x-pack/test/functional/apps/ml/short_tests/notifications/index.ts b/x-pack/test/functional/apps/ml/short_tests/notifications/index.ts index e026d44a67af2..d7756a75a66de 100644 --- a/x-pack/test/functional/apps/ml/short_tests/notifications/index.ts +++ b/x-pack/test/functional/apps/ml/short_tests/notifications/index.ts @@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('Notifcations', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); loadTestFile(require.resolve('./notification_list')); }); diff --git a/x-pack/test/functional/apps/ml/short_tests/settings/index.ts b/x-pack/test/functional/apps/ml/short_tests/settings/index.ts index d3f7000918a8e..2f46e75038ff9 100644 --- a/x-pack/test/functional/apps/ml/short_tests/settings/index.ts +++ b/x-pack/test/functional/apps/ml/short_tests/settings/index.ts @@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('settings', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); loadTestFile(require.resolve('./calendar_creation')); loadTestFile(require.resolve('./calendar_edit')); diff --git a/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts b/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts index 37f238dbeecc9..53f4b7cbf943e 100644 --- a/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts +++ b/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts @@ -12,7 +12,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { const ml = getService('ml'); describe('machine learning - stack management jobs', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); before(async () => { await ml.securityCommon.createMlRoles(); await ml.securityCommon.createMlUsers(); diff --git a/x-pack/test/functional/page_objects/asset_details.ts b/x-pack/test/functional/page_objects/asset_details.ts index 95a7819fb11a6..7ce90d213d234 100644 --- a/x-pack/test/functional/page_objects/asset_details.ts +++ b/x-pack/test/functional/page_objects/asset_details.ts @@ -180,7 +180,9 @@ export function AssetDetailsProvider({ getService }: FtrProviderContext) { }, async clickAddMetadataFilter() { - return testSubjects.click('infraAssetDetailsMetadataAddFilterButton'); + // Make this selector tied to the field to avoid flakiness + // https://github.com/elastic/kibana/issues/191565 + return testSubjects.click('infraAssetDetailsMetadataField.host.name'); }, async clickRemoveMetadataFilter() { diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/ml/index.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/ml/index.ts index 7a898b63c3f22..ea21b37e86a66 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/ml/index.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/ml/index.ts @@ -12,7 +12,7 @@ export default ({ loadTestFile, getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver'); describe('ML app', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); before(async () => { await ml.securityCommon.createMlRoles();