From ca594f32e16fd7054d761103ec3c5dd4d3e689eb Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Fri, 30 Aug 2024 09:47:09 +0200 Subject: [PATCH 01/12] [ML] Fix schema definition for the module endpoint (#191633) ## Summary Fixes an issue introduced in https://github.com/elastic/kibana/pull/190840 where the module endpoint had a wrong type definition for validation the response causing the Rules page in Security plugin to crash. --- x-pack/plugins/ml/server/routes/schemas/modules.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/ml/server/routes/schemas/modules.ts b/x-pack/plugins/ml/server/routes/schemas/modules.ts index 50fe45b653e42..8cb6bfdc96fd8 100644 --- a/x-pack/plugins/ml/server/routes/schemas/modules.ts +++ b/x-pack/plugins/ml/server/routes/schemas/modules.ts @@ -138,11 +138,11 @@ const moduleSchema = schema.object({ type: schema.string(), logo: schema.maybe(schema.any()), logoFile: schema.maybe(schema.string()), - defaultIndexPattern: schema.string(), - query: schema.any(), + defaultIndexPattern: schema.maybe(schema.string()), + query: schema.maybe(schema.any()), jobs: schema.arrayOf(schema.any()), datafeeds: schema.arrayOf(schema.any()), - kibana: schema.any(), + kibana: schema.maybe(schema.any()), tags: schema.maybe(schema.arrayOf(schema.string())), }); @@ -157,7 +157,7 @@ export const dataRecognizerConfigResponse = () => schema.object({ datafeeds: schema.arrayOf(schema.any()), jobs: schema.arrayOf(schema.any()), - kibana: schema.any(), + kibana: schema.maybe(schema.any()), }); export const jobExistsResponse = () => From 79fb4f22e369bd63e014efecbd27f6acd4e209fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Fri, 30 Aug 2024 10:00:20 +0200 Subject: [PATCH 02/12] [Move `@kbn/config-schema` to server] `data` (#191778) --- src/plugins/data/config.mock.ts | 2 +- src/plugins/data/public/index.ts | 2 +- src/plugins/data/public/plugin.ts | 2 +- .../data/public/search/search_interceptor/search_interceptor.ts | 2 +- src/plugins/data/public/search/search_service.ts | 2 +- src/plugins/data/public/search/session/session_service.ts | 2 +- .../public/search/session/sessions_mgmt/application/index.tsx | 2 +- .../search/session/sessions_mgmt/components/main.test.tsx | 2 +- .../public/search/session/sessions_mgmt/components/main.tsx | 2 +- .../session/sessions_mgmt/components/table/table.test.tsx | 2 +- .../search/session/sessions_mgmt/components/table/table.tsx | 2 +- src/plugins/data/public/search/session/sessions_mgmt/index.ts | 2 +- .../data/public/search/session/sessions_mgmt/lib/api.test.ts | 2 +- src/plugins/data/public/search/session/sessions_mgmt/lib/api.ts | 2 +- .../search/session/sessions_mgmt/lib/get_columns.test.tsx | 2 +- .../public/search/session/sessions_mgmt/lib/get_columns.tsx | 2 +- .../search/session/sessions_mgmt/lib/get_expiration_status.ts | 2 +- src/plugins/data/{ => server}/config.ts | 0 src/plugins/data/server/index.ts | 2 +- src/plugins/data/server/plugin.ts | 2 +- src/plugins/data/server/search/search_service.ts | 2 +- .../data/server/search/session/get_session_status.test.ts | 2 +- src/plugins/data/server/search/session/get_session_status.ts | 2 +- src/plugins/data/server/search/session/mocks.ts | 2 +- src/plugins/data/server/search/session/session_service.test.ts | 2 +- src/plugins/data/server/search/session/session_service.ts | 2 +- src/plugins/data/server/search/session/types.ts | 2 +- src/plugins/data/server/search/strategies/common/async_utils.ts | 2 +- .../server/search/strategies/eql_search/eql_search_strategy.ts | 2 +- .../server/search/strategies/ese_search/ese_search_strategy.ts | 2 +- .../data/server/search/strategies/ese_search/request_utils.ts | 2 +- .../strategies/esql_async_search/esql_async_search_strategy.ts | 2 +- .../data/server/search/strategies/sql_search/request_utils.ts | 2 +- .../server/search/strategies/sql_search/sql_search_strategy.ts | 2 +- src/plugins/data/tsconfig.json | 1 - 35 files changed, 33 insertions(+), 34 deletions(-) rename src/plugins/data/{ => server}/config.ts (100%) diff --git a/src/plugins/data/config.mock.ts b/src/plugins/data/config.mock.ts index ab1d02cb63b31..b7c41754dc24d 100644 --- a/src/plugins/data/config.mock.ts +++ b/src/plugins/data/config.mock.ts @@ -7,7 +7,7 @@ */ import moment from 'moment/moment'; -import { SearchConfigSchema, SearchSessionsConfigSchema } from './config'; +import type { SearchConfigSchema, SearchSessionsConfigSchema } from './server/config'; export const getMockSearchConfig = ({ sessions: { enabled = true, defaultExpiration = moment.duration(7, 'd') } = { diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index fa03dc95f564f..09f3226eef57b 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -7,7 +7,7 @@ */ import { PluginInitializerContext } from '@kbn/core/public'; -import { ConfigSchema } from '../config'; +import type { ConfigSchema } from '../server/config'; /* * Filters: diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 530cbe978c3d1..8e0eea12ed168 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -14,7 +14,7 @@ import { IStorageWrapper, createStartServicesGetter, } from '@kbn/kibana-utils-plugin/public'; -import { ConfigSchema } from '../config'; +import type { ConfigSchema } from '../server/config'; import type { DataPublicPluginSetup, DataPublicPluginStart, diff --git a/src/plugins/data/public/search/search_interceptor/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor/search_interceptor.ts index 7f54c63592b14..95573127fdc90 100644 --- a/src/plugins/data/public/search/search_interceptor/search_interceptor.ts +++ b/src/plugins/data/public/search/search_interceptor/search_interceptor.ts @@ -73,7 +73,7 @@ import { toPartialResponseAfterTimeout } from './to_partial_response'; import { ISessionService, SearchSessionState } from '../session'; import { SearchResponseCache } from './search_response_cache'; import { SearchAbortController } from './search_abort_controller'; -import { SearchConfigSchema } from '../../../config'; +import type { SearchConfigSchema } from '../../../server/config'; import type { SearchServiceStartDependencies } from '../search_service'; import { createRequestHash } from './create_request_hash'; diff --git a/src/plugins/data/public/search/search_service.ts b/src/plugins/data/public/search/search_service.ts index 0bdc33cb59303..07afdc0514c55 100644 --- a/src/plugins/data/public/search/search_service.ts +++ b/src/plugins/data/public/search/search_service.ts @@ -62,7 +62,7 @@ import { SHARD_DELAY_AGG_NAME, } from '../../common/search/aggs/buckets/shard_delay'; import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn'; -import { ConfigSchema } from '../../config'; +import type { ConfigSchema } from '../../server/config'; import { NowProviderInternalContract } from '../now_provider'; import { DataPublicPluginStart, DataStartDependencies } from '../types'; import { AggsService } from './aggs'; diff --git a/src/plugins/data/public/search/session/session_service.ts b/src/plugins/data/public/search/session/session_service.ts index 5b936fb72a88d..b3c3e94f18b54 100644 --- a/src/plugins/data/public/search/session/session_service.ts +++ b/src/plugins/data/public/search/session/session_service.ts @@ -39,7 +39,7 @@ import { i18n } from '@kbn/i18n'; import moment from 'moment'; import { ISearchOptions } from '@kbn/search-types'; import { SearchUsageCollector } from '../..'; -import { ConfigSchema } from '../../../config'; +import type { ConfigSchema } from '../../../server/config'; import type { SessionMeta, SessionStateContainer, diff --git a/src/plugins/data/public/search/session/sessions_mgmt/application/index.tsx b/src/plugins/data/public/search/session/sessions_mgmt/application/index.tsx index b0a15ca405743..b0c4330c35e5a 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/application/index.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/application/index.tsx @@ -17,7 +17,7 @@ import { APP } from '..'; import { SearchSessionsMgmtAPI } from '../lib/api'; import { AsyncSearchIntroDocumentation } from '../lib/documentation'; import { renderApp } from './render'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; export class SearchSessionsMgmtApp { constructor( diff --git a/src/plugins/data/public/search/session/sessions_mgmt/components/main.test.tsx b/src/plugins/data/public/search/session/sessions_mgmt/components/main.test.tsx index 38ccbb9646dee..97b446cae47c1 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/components/main.test.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/components/main.test.tsx @@ -20,7 +20,7 @@ import { LocaleWrapper } from '../__mocks__'; import { SearchSessionsMgmtMain } from './main'; import { SharePluginStart } from '@kbn/share-plugin/public'; import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; import { createSearchUsageCollectorMock } from '../../../collectors/mocks'; let mockCoreSetup: MockedKeys; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/components/main.tsx b/src/plugins/data/public/search/session/sessions_mgmt/components/main.tsx index 009e3ce7d9b07..af5e1c3bc9a8c 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/components/main.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/components/main.tsx @@ -14,7 +14,7 @@ import type { SearchSessionsMgmtAPI } from '../lib/api'; import type { AsyncSearchIntroDocumentation } from '../lib/documentation'; import { SearchSessionsMgmtTable } from './table'; import { SearchSessionsDeprecatedWarning } from '../../search_sessions_deprecation_message'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; import { SearchUsageCollector } from '../../../collectors'; interface Props { diff --git a/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.test.tsx b/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.test.tsx index 6394deeab843b..efe35f206dc5f 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.test.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.test.tsx @@ -20,7 +20,7 @@ import { LocaleWrapper } from '../../__mocks__'; import { SearchSessionsMgmtTable } from './table'; import { SharePluginStart } from '@kbn/share-plugin/public'; import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; -import { SearchSessionsConfigSchema } from '../../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../../server/config'; import { createSearchUsageCollectorMock } from '../../../../collectors/mocks'; let mockCoreSetup: MockedKeys; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.tsx b/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.tsx index 833ad3ec7e75e..649a3d4316a5e 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/components/table/table.tsx @@ -21,7 +21,7 @@ import { OnActionComplete } from '../actions'; import { getAppFilter } from './app_filter'; import { getStatusFilter } from './status_filter'; import { SearchUsageCollector } from '../../../../collectors'; -import { SearchSessionsConfigSchema } from '../../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../../server/config'; interface Props { core: CoreStart; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/index.ts b/src/plugins/data/public/search/session/sessions_mgmt/index.ts index 7b95305f6e568..59437bf27b6e4 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/index.ts +++ b/src/plugins/data/public/search/session/sessions_mgmt/index.ts @@ -15,7 +15,7 @@ import type { ISessionsClient, SearchUsageCollector } from '../../..'; import { SEARCH_SESSIONS_MANAGEMENT_ID } from '../constants'; import type { SearchSessionsMgmtAPI } from './lib/api'; import type { AsyncSearchIntroDocumentation } from './lib/documentation'; -import { SearchSessionsConfigSchema } from '../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../server/config'; export interface IManagementSectionsPluginsSetup { management: ManagementSetup; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/lib/api.test.ts b/src/plugins/data/public/search/session/sessions_mgmt/lib/api.test.ts index 52b72afd30ab3..a765bb8a0a2d4 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/lib/api.test.ts +++ b/src/plugins/data/public/search/session/sessions_mgmt/lib/api.test.ts @@ -16,7 +16,7 @@ import { SearchSessionStatus } from '../../../../../common'; import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; import { SharePluginStart } from '@kbn/share-plugin/public'; import { SearchSessionsMgmtAPI } from './api'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; let mockCoreSetup: MockedKeys; let mockCoreStart: MockedKeys; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/lib/api.ts b/src/plugins/data/public/search/session/sessions_mgmt/lib/api.ts index f5abbc4c77de8..5c426942034f6 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/lib/api.ts +++ b/src/plugins/data/public/search/session/sessions_mgmt/lib/api.ts @@ -22,7 +22,7 @@ import { import { ISessionsClient } from '../../sessions_client'; import { SearchUsageCollector } from '../../../collectors'; import { SearchSessionsFindResponse, SearchSessionStatus } from '../../../../../common'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; type LocatorsStart = SharePluginStart['url']['locators']; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.test.tsx b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.test.tsx index 3fd53ccfddec2..643cd6b6286f6 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.test.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.test.tsx @@ -21,7 +21,7 @@ import { SearchSessionsMgmtAPI } from './api'; import { getColumns } from './get_columns'; import { SharePluginStart } from '@kbn/share-plugin/public'; import { sharePluginMock } from '@kbn/share-plugin/public/mocks'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; import { createSearchUsageCollectorMock } from '../../../collectors/mocks'; let mockCoreSetup: MockedKeys; diff --git a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.tsx b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.tsx index 371ffac111e84..07862651f60ab 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.tsx +++ b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_columns.tsx @@ -30,7 +30,7 @@ import { SearchSessionsMgmtAPI } from './api'; import { getExpirationStatus } from './get_expiration_status'; import { UISession } from '../types'; import { SearchUsageCollector } from '../../../collectors'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; // Helper function: translate an app string to EuiIcon-friendly string const appToIcon = (app: string) => { diff --git a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_expiration_status.ts b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_expiration_status.ts index cde183f569f93..b80b315aaea8c 100644 --- a/src/plugins/data/public/search/session/sessions_mgmt/lib/get_expiration_status.ts +++ b/src/plugins/data/public/search/session/sessions_mgmt/lib/get_expiration_status.ts @@ -8,7 +8,7 @@ import { i18n } from '@kbn/i18n'; import moment from 'moment'; -import { SearchSessionsConfigSchema } from '../../../../../config'; +import type { SearchSessionsConfigSchema } from '../../../../../server/config'; export const getExpirationStatus = (config: SearchSessionsConfigSchema, expires: string | null) => { const tNow = moment.utc().valueOf(); diff --git a/src/plugins/data/config.ts b/src/plugins/data/server/config.ts similarity index 100% rename from src/plugins/data/config.ts rename to src/plugins/data/server/config.ts diff --git a/src/plugins/data/server/index.ts b/src/plugins/data/server/index.ts index 7a41c20094d6e..e9292342f917e 100644 --- a/src/plugins/data/server/index.ts +++ b/src/plugins/data/server/index.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ import { PluginConfigDescriptor, PluginInitializerContext } from '@kbn/core/server'; -import { ConfigSchema, configSchema } from '../config'; +import { ConfigSchema, configSchema } from './config'; import type { DataServerPlugin, DataPluginSetup, DataPluginStart } from './plugin'; export { getEsQueryConfig, DEFAULT_QUERY_LANGUAGE } from '../common'; diff --git a/src/plugins/data/server/plugin.ts b/src/plugins/data/server/plugin.ts index 77f90393a6164..296d59dc2f632 100644 --- a/src/plugins/data/server/plugin.ts +++ b/src/plugins/data/server/plugin.ts @@ -12,7 +12,7 @@ import { BfetchServerSetup } from '@kbn/bfetch-plugin/server'; import { PluginStart as DataViewsServerPluginStart } from '@kbn/data-views-plugin/server'; import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; import { FieldFormatsSetup, FieldFormatsStart } from '@kbn/field-formats-plugin/server'; -import { ConfigSchema } from '../config'; +import { ConfigSchema } from './config'; import type { ISearchSetup, ISearchStart } from './search'; import { DatatableUtilitiesService } from './datatable_utilities'; import { SearchService } from './search/search_service'; diff --git a/src/plugins/data/server/search/search_service.ts b/src/plugins/data/server/search/search_service.ts index 09bee9bb8bab8..b9449b7f5da11 100644 --- a/src/plugins/data/server/search/search_service.ts +++ b/src/plugins/data/server/search/search_service.ts @@ -90,7 +90,7 @@ import { SHARD_DELAY_AGG_NAME, } from '../../common/search/aggs/buckets/shard_delay'; import { aggShardDelay } from '../../common/search/aggs/buckets/shard_delay_fn'; -import { ConfigSchema } from '../../config'; +import { ConfigSchema } from '../config'; import { SearchSessionService } from './session'; import { registerBsearchRoute } from './routes/bsearch'; import { enhancedEsSearchStrategyProvider } from './strategies/ese_search'; diff --git a/src/plugins/data/server/search/session/get_session_status.test.ts b/src/plugins/data/server/search/session/get_session_status.test.ts index 1d59fd11c471e..128c19220c2f3 100644 --- a/src/plugins/data/server/search/session/get_session_status.test.ts +++ b/src/plugins/data/server/search/session/get_session_status.test.ts @@ -10,7 +10,7 @@ import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { getSessionStatus } from './get_session_status'; import { SearchSessionSavedObjectAttributes, SearchSessionStatus } from '../../../common'; import moment from 'moment'; -import { SearchSessionsConfigSchema } from '../../../config'; +import { SearchSessionsConfigSchema } from '../../config'; const mockInProgressSearchResponse = { body: { diff --git a/src/plugins/data/server/search/session/get_session_status.ts b/src/plugins/data/server/search/session/get_session_status.ts index e1cb50a6c4cd4..57bae3c38e9a7 100644 --- a/src/plugins/data/server/search/session/get_session_status.ts +++ b/src/plugins/data/server/search/session/get_session_status.ts @@ -10,7 +10,7 @@ import moment from 'moment'; import { ElasticsearchClient } from '@kbn/core/server'; import { SearchSessionSavedObjectAttributes, SearchSessionStatus } from '../../../common'; import { SearchStatus } from './types'; -import { SearchSessionsConfigSchema } from '../../../config'; +import { SearchSessionsConfigSchema } from '../../config'; import { getSearchStatus } from './get_search_status'; export async function getSessionStatus( diff --git a/src/plugins/data/server/search/session/mocks.ts b/src/plugins/data/server/search/session/mocks.ts index 339ef628356db..a5bbb06a5621a 100644 --- a/src/plugins/data/server/search/session/mocks.ts +++ b/src/plugins/data/server/search/session/mocks.ts @@ -8,7 +8,7 @@ import moment from 'moment'; import type { IScopedSearchSessionsClient } from './types'; -import { SearchSessionsConfigSchema } from '../../../config'; +import { SearchSessionsConfigSchema } from '../../config'; export function createSearchSessionsClientMock(): jest.Mocked { return { diff --git a/src/plugins/data/server/search/session/session_service.test.ts b/src/plugins/data/server/search/session/session_service.test.ts index 5a31cefe7b998..5f596da374703 100644 --- a/src/plugins/data/server/search/session/session_service.test.ts +++ b/src/plugins/data/server/search/session/session_service.test.ts @@ -17,7 +17,7 @@ import { SearchSessionService } from './session_service'; import { createRequestHash } from './utils'; import moment from 'moment'; import { coreMock } from '@kbn/core/server/mocks'; -import { ConfigSchema } from '../../../config'; +import { ConfigSchema } from '../../config'; import type { AuthenticatedUser } from '@kbn/core/server'; import { SEARCH_SESSION_TYPE, SearchSessionStatus } from '../../../common'; import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; diff --git a/src/plugins/data/server/search/session/session_service.ts b/src/plugins/data/server/search/session/session_service.ts index efd41990493b3..ef367b552983d 100644 --- a/src/plugins/data/server/search/session/session_service.ts +++ b/src/plugins/data/server/search/session/session_service.ts @@ -32,7 +32,7 @@ import { } from '../../../common'; import { ISearchSessionService, NoSearchIdInSessionError } from '../..'; import { createRequestHash } from './utils'; -import { ConfigSchema, SearchSessionsConfigSchema } from '../../../config'; +import { ConfigSchema, SearchSessionsConfigSchema } from '../../config'; import { getSessionStatus } from './get_session_status'; export interface SearchSessionDependencies { diff --git a/src/plugins/data/server/search/session/types.ts b/src/plugins/data/server/search/session/types.ts index cc3d604f50efc..02cb9cad55bb7 100644 --- a/src/plugins/data/server/search/session/types.ts +++ b/src/plugins/data/server/search/session/types.ts @@ -19,7 +19,7 @@ import { SearchSessionSavedObjectAttributes, SearchSessionStatusResponse, } from '../../../common/search'; -import { SearchSessionsConfigSchema } from '../../../config'; +import { SearchSessionsConfigSchema } from '../../config'; export { SearchStatus } from '../../../common/search'; diff --git a/src/plugins/data/server/search/strategies/common/async_utils.ts b/src/plugins/data/server/search/strategies/common/async_utils.ts index ca33a01ac8064..6c2d4b98bc5a1 100644 --- a/src/plugins/data/server/search/strategies/common/async_utils.ts +++ b/src/plugins/data/server/search/strategies/common/async_utils.ts @@ -11,7 +11,7 @@ import { AsyncSearchGetRequest, } from '@elastic/elasticsearch/lib/api/types'; import { ISearchOptions } from '@kbn/search-types'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; /** @internal diff --git a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts index 9db01189ff66d..2bf6ff4f68601 100644 --- a/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/eql_search/eql_search_strategy.ts @@ -10,7 +10,7 @@ import type { TransportResult } from '@elastic/elasticsearch'; import { tap } from 'rxjs'; import type { IScopedClusterClient, Logger } from '@kbn/core/server'; import { getKbnServerError } from '@kbn/kibana-utils-plugin/server'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; import { EqlSearchStrategyRequest, EqlSearchStrategyResponse, diff --git a/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts b/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts index 8c5a8ad204a72..d4933357e0334 100644 --- a/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts @@ -31,7 +31,7 @@ import { getTotalLoaded, shimHitsTotal, } from '../es_search'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; import { sanitizeRequestParams } from '../../sanitize_request_params'; export const enhancedEsSearchStrategyProvider = ( diff --git a/src/plugins/data/server/search/strategies/ese_search/request_utils.ts b/src/plugins/data/server/search/strategies/ese_search/request_utils.ts index ec182534e089f..d3341b192660c 100644 --- a/src/plugins/data/server/search/strategies/ese_search/request_utils.ts +++ b/src/plugins/data/server/search/strategies/ese_search/request_utils.ts @@ -12,7 +12,7 @@ import { AsyncSearchSubmitRequest } from '@elastic/elasticsearch/lib/api/types'; import { ISearchOptions } from '@kbn/search-types'; import { UI_SETTINGS } from '../../../../common'; import { getDefaultSearchParams } from '../es_search'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; import { getCommonDefaultAsyncGetParams, getCommonDefaultAsyncSubmitParams, diff --git a/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts b/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts index a204b6ca69cca..a8623d99e1632 100644 --- a/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/esql_async_search/esql_async_search_strategy.ts @@ -22,7 +22,7 @@ import { getKbnSearchError } from '../../report_search_error'; import type { ISearchStrategy, SearchStrategyDependencies } from '../../types'; import type { IAsyncSearchOptions } from '../../../../common'; import { toAsyncKibanaSearchResponse } from './response_utils'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; // `drop_null_columns` is going to change the response // now we get `all_columns` and `columns` diff --git a/src/plugins/data/server/search/strategies/sql_search/request_utils.ts b/src/plugins/data/server/search/strategies/sql_search/request_utils.ts index 1a2540796d690..a608409467824 100644 --- a/src/plugins/data/server/search/strategies/sql_search/request_utils.ts +++ b/src/plugins/data/server/search/strategies/sql_search/request_utils.ts @@ -8,7 +8,7 @@ import { SqlGetAsyncRequest, SqlQueryRequest } from '@elastic/elasticsearch/lib/api/types'; import { ISearchOptions } from '@kbn/search-types'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; import { getCommonDefaultAsyncGetParams, getCommonDefaultAsyncSubmitParams, diff --git a/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts b/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts index 55ed1bfecb995..c71e926d764db 100644 --- a/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/sql_search/sql_search_strategy.ts @@ -22,7 +22,7 @@ import type { import { pollSearch } from '../../../../common'; import { getDefaultAsyncGetParams, getDefaultAsyncSubmitParams } from './request_utils'; import { toAsyncKibanaSearchResponse } from './response_utils'; -import { SearchConfigSchema } from '../../../../config'; +import { SearchConfigSchema } from '../../../config'; export const sqlSearchStrategyProvider = ( searchConfig: SearchConfigSchema, diff --git a/src/plugins/data/tsconfig.json b/src/plugins/data/tsconfig.json index cc6ced2bef611..1b2fc9ed85e93 100644 --- a/src/plugins/data/tsconfig.json +++ b/src/plugins/data/tsconfig.json @@ -7,7 +7,6 @@ "common/**/*", "public/**/*", "server/**/*", - "config.ts", "config.mock.ts", "common/**/*.json", "public/**/*.json", From 1805b5cdd11d01c0be1f6a1449c3f92335c2fec9 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Fri, 30 Aug 2024 10:03:09 +0200 Subject: [PATCH 03/12] [CI] Use `execFile` for quick-checks (#191638) ## Summary As per the suggestion for https://github.com/elastic/kibana/security/code-scanning/448 - using a safer script execution --- src/dev/run_quick_checks.ts | 59 ++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/src/dev/run_quick_checks.ts b/src/dev/run_quick_checks.ts index cdb59bdce3cb2..2fe4b712bdbb1 100644 --- a/src/dev/run_quick_checks.ts +++ b/src/dev/run_quick_checks.ts @@ -6,10 +6,10 @@ * Side Public License, v 1. */ -import { exec } from 'child_process'; +import { execFile } from 'child_process'; import { availableParallelism } from 'os'; -import { join, isAbsolute } from 'path'; -import { readdirSync, readFileSync } from 'fs'; +import { isAbsolute, join } from 'path'; +import { existsSync, readdirSync, readFileSync } from 'fs'; import { run, RunOptions } from '@kbn/dev-cli-runner'; import { REPO_ROOT } from '@kbn/repo-info'; @@ -54,7 +54,7 @@ void run(async ({ log, flagsReader }) => { targetFile: flagsReader.string('file'), targetDir: flagsReader.string('dir'), checks: flagsReader.string('checks'), - }); + }).map((script) => (isAbsolute(script) ? script : join(REPO_ROOT, script))); logger.write( `--- Running ${scriptsToRun.length} checks, with parallelism ${MAX_PARALLELISM}...`, @@ -108,7 +108,7 @@ function collectScriptsToRun(inputOptions: { } } -async function runAllChecks(scriptsToRun: string[]) { +async function runAllChecks(scriptsToRun: string[]): Promise { const checksRunning: Array> = []; const checksFinished: CheckResult[] = []; @@ -121,10 +121,20 @@ async function runAllChecks(scriptsToRun: string[]) { const check = runCheckAsync(script); checksRunning.push(check); - check.then((result) => { - checksRunning.splice(checksRunning.indexOf(check), 1); - checksFinished.push(result); - }); + check + .then((result) => { + checksRunning.splice(checksRunning.indexOf(check), 1); + checksFinished.push(result); + }) + .catch((error) => { + checksRunning.splice(checksRunning.indexOf(check), 1); + checksFinished.push({ + success: false, + script, + output: error.message, + durationMs: 0, + }); + }); } await sleep(1000); @@ -138,9 +148,10 @@ async function runCheckAsync(script: string): Promise { const startTime = Date.now(); return new Promise((resolve) => { - const scriptProcess = exec(script); + validateScriptPath(script); + const scriptProcess = execFile('bash', [script]); let output = ''; - const appendToOutput = (data: string | Buffer) => (output += data); + const appendToOutput = (data: string | Buffer) => (output += data.toString()); scriptProcess.stdout?.on('data', appendToOutput); scriptProcess.stderr?.on('data', appendToOutput); @@ -170,9 +181,10 @@ function printResults(startTimestamp: number, results: CheckResult[]) { logger.info(`- Total time: ${total}, effective: ${effective}`); results.forEach((result) => { - logger.write( - `--- ${result.success ? '✅' : '❌'} ${result.script}: ${humanizeTime(result.durationMs)}` - ); + const resultLabel = result.success ? '✅' : '❌'; + const scriptPath = stripRoot(result.script); + const runtime = humanizeTime(result.durationMs); + logger.write(`--- ${resultLabel} ${scriptPath}: ${runtime}`); if (result.success) { logger.debug(result.output); } else { @@ -194,3 +206,22 @@ function humanizeTime(ms: number) { return `${minutes}m ${seconds}s`; } } + +function validateScriptPath(scriptPath: string) { + if (!isAbsolute(scriptPath)) { + logger.error(`Invalid script path: ${scriptPath}`); + throw new Error('Invalid script path'); + } else if (!scriptPath.endsWith('.sh')) { + logger.error(`Invalid script extension: ${scriptPath}`); + throw new Error('Invalid script extension'); + } else if (!existsSync(scriptPath)) { + logger.error(`Script not found: ${scriptPath}`); + throw new Error('Script not found'); + } else { + return; + } +} + +function stripRoot(script: string) { + return script.replace(REPO_ROOT, ''); +} From b95d9f1efc3fecce09ebe861c36a41f160091036 Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Fri, 30 Aug 2024 04:12:28 -0400 Subject: [PATCH 04/12] [Synthetics] project monitor delete api - escape quotes (#191769) ## Summary Quotes KQL values for the synthetics project monitor delete API. --------- Co-authored-by: Shahzad --- .../synthetics/server/routes/common.test.ts | 38 +++++++++++++++++++ .../synthetics/server/routes/common.ts | 21 +++++----- .../add_monitor/add_monitor_api.ts | 4 +- .../monitor_cruds/delete_monitor_project.ts | 4 +- .../get_location_monitors.ts | 4 +- .../project_monitor_formatter.ts | 7 +++- 6 files changed, 61 insertions(+), 17 deletions(-) create mode 100644 x-pack/plugins/observability_solution/synthetics/server/routes/common.test.ts diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/common.test.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/common.test.ts new file mode 100644 index 0000000000000..82520c68b5fc4 --- /dev/null +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/common.test.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getSavedObjectKqlFilter } from './common'; + +describe('getSavedObjectKqlFilter', () => { + it('returns empty string if no values are provided', () => { + expect(getSavedObjectKqlFilter({ field: 'tags' })).toBe(''); + }); + + it('returns KQL string if values are provided', () => { + expect(getSavedObjectKqlFilter({ field: 'tags', values: 'apm' })).toBe( + 'synthetics-monitor.attributes.tags:"apm"' + ); + }); + + it('searches at root when specified', () => { + expect(getSavedObjectKqlFilter({ field: 'tags', values: 'apm', searchAtRoot: true })).toBe( + 'tags:"apm"' + ); + }); + + it('handles array values', () => { + expect(getSavedObjectKqlFilter({ field: 'tags', values: ['apm', 'synthetics'] })).toBe( + 'synthetics-monitor.attributes.tags:("apm" OR "synthetics")' + ); + }); + + it('escapes quotes', () => { + expect(getSavedObjectKqlFilter({ field: 'tags', values: ['"apm', 'synthetics'] })).toBe( + 'synthetics-monitor.attributes.tags:("\\"apm" OR "synthetics")' + ); + }); +}); diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/common.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/common.ts index 491b67160677e..bc58a866bef83 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/common.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/common.ts @@ -7,6 +7,7 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { SavedObjectsFindResponse } from '@kbn/core/server'; +import { escapeQuotes } from '@kbn/es-query'; import { RouteContext } from './types'; import { MonitorSortFieldSchema } from '../../common/runtime_types/monitor_management/sort_field'; import { getAllLocations } from '../synthetics_service/get_all_locations'; @@ -132,19 +133,19 @@ export const getMonitorFilters = async ({ const filtersStr = [ filter, - getKqlFilter({ field: 'tags', values: tags }), - getKqlFilter({ field: 'project_id', values: projects }), - getKqlFilter({ field: 'type', values: monitorTypes }), - getKqlFilter({ field: 'locations.id', values: locationFilter }), - getKqlFilter({ field: 'schedule.number', values: schedules }), - getKqlFilter({ field: 'id', values: monitorQueryIds }), + getSavedObjectKqlFilter({ field: 'tags', values: tags }), + getSavedObjectKqlFilter({ field: 'project_id', values: projects }), + getSavedObjectKqlFilter({ field: 'type', values: monitorTypes }), + getSavedObjectKqlFilter({ field: 'locations.id', values: locationFilter }), + getSavedObjectKqlFilter({ field: 'schedule.number', values: schedules }), + getSavedObjectKqlFilter({ field: 'id', values: monitorQueryIds }), ] .filter((f) => !!f) .join(' AND '); return { filtersStr, locationFilter }; }; -export const getKqlFilter = ({ +export const getSavedObjectKqlFilter = ({ field, values, operator = 'OR', @@ -166,10 +167,12 @@ export const getKqlFilter = ({ } if (Array.isArray(values)) { - return ` (${fieldKey}:"${values.join(`" ${operator} ${fieldKey}:"`)}" )`; + return `${fieldKey}:(${values + .map((value) => `"${escapeQuotes(value)}"`) + .join(` ${operator} `)})`; } - return `${fieldKey}:"${values}"`; + return `${fieldKey}:"${escapeQuotes(values)}"`; }; const parseLocationFilter = async (context: RouteContext, locations?: string | string[]) => { diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor/add_monitor_api.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor/add_monitor_api.ts index 1c38e093237e1..359f3373cfd35 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor/add_monitor_api.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/add_monitor/add_monitor_api.ts @@ -12,7 +12,7 @@ import { isValidNamespace } from '@kbn/fleet-plugin/common'; import { i18n } from '@kbn/i18n'; import { parseMonitorLocations } from './utils'; import { MonitorValidationError } from '../monitor_validation'; -import { getKqlFilter } from '../../common'; +import { getSavedObjectKqlFilter } from '../../common'; import { deleteMonitor } from '../delete_monitor'; import { monitorAttributes, syntheticsMonitorType } from '../../../../common/types/saved_objects'; import { PrivateLocationAttributes } from '../../../runtime_types/private_locations'; @@ -238,7 +238,7 @@ export class AddEditMonitorAPI { async validateUniqueMonitorName(name: string, id?: string) { const { savedObjectsClient } = this.routeContext; - const kqlFilter = getKqlFilter({ field: 'name.keyword', values: name }); + const kqlFilter = getSavedObjectKqlFilter({ field: 'name.keyword', values: name }); const { total } = await savedObjectsClient.find({ perPage: 0, type: syntheticsMonitorType, diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts index b5c8dc2b4f0b1..2136634be7ef7 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/monitor_cruds/delete_monitor_project.ts @@ -10,7 +10,7 @@ import { SyntheticsRestApiRouteFactory } from '../types'; import { syntheticsMonitorType } from '../../../common/types/saved_objects'; import { ConfigKey } from '../../../common/runtime_types'; import { SYNTHETICS_API_URLS } from '../../../common/constants'; -import { getMonitors, getKqlFilter } from '../common'; +import { getMonitors, getSavedObjectKqlFilter } from '../common'; import { deleteMonitorBulk } from './bulk_cruds/delete_monitor_bulk'; export const deleteSyntheticsMonitorProjectRoute: SyntheticsRestApiRouteFactory = () => ({ @@ -39,7 +39,7 @@ export const deleteSyntheticsMonitorProjectRoute: SyntheticsRestApiRouteFactory const deleteFilter = `${syntheticsMonitorType}.attributes.${ ConfigKey.PROJECT_ID - }: "${decodedProjectName}" AND ${getKqlFilter({ + }: "${decodedProjectName}" AND ${getSavedObjectKqlFilter({ field: 'journey_id', values: monitorsToDelete.map((id: string) => `${id}`), })}`; diff --git a/x-pack/plugins/observability_solution/synthetics/server/routes/settings/private_locations/get_location_monitors.ts b/x-pack/plugins/observability_solution/synthetics/server/routes/settings/private_locations/get_location_monitors.ts index de96ba8eb9438..6701946c8a6d6 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/routes/settings/private_locations/get_location_monitors.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/routes/settings/private_locations/get_location_monitors.ts @@ -6,7 +6,7 @@ */ import { ALL_SPACES_ID } from '@kbn/spaces-plugin/common/constants'; -import { getKqlFilter } from '../../common'; +import { getSavedObjectKqlFilter } from '../../common'; import { SyntheticsRestApiRouteFactory } from '../../types'; import { SYNTHETICS_API_URLS } from '../../../../common/constants'; import { monitorAttributes, syntheticsMonitorType } from '../../../../common/types/saved_objects'; @@ -47,7 +47,7 @@ export const getLocationMonitors: SyntheticsRestApiRouteFactory = () => export const getMonitorsByLocation = async (server: SyntheticsServerSetup, locationId?: string) => { const soClient = server.coreStart.savedObjects.createInternalRepository(); - const locationFilter = getKqlFilter({ field: 'locations.id', values: locationId }); + const locationFilter = getSavedObjectKqlFilter({ field: 'locations.id', values: locationId }); const locationMonitors = await soClient.find({ type: syntheticsMonitorType, diff --git a/x-pack/plugins/observability_solution/synthetics/server/synthetics_service/project_monitor/project_monitor_formatter.ts b/x-pack/plugins/observability_solution/synthetics/server/synthetics_service/project_monitor/project_monitor_formatter.ts index fd59e7a0ec40a..6069076e375a2 100644 --- a/x-pack/plugins/observability_solution/synthetics/server/synthetics_service/project_monitor/project_monitor_formatter.ts +++ b/x-pack/plugins/observability_solution/synthetics/server/synthetics_service/project_monitor/project_monitor_formatter.ts @@ -11,7 +11,7 @@ import { } from '@kbn/core/server'; import { i18n } from '@kbn/i18n'; import { EncryptedSavedObjectsClient } from '@kbn/encrypted-saved-objects-plugin/server'; -import { getKqlFilter } from '../../routes/common'; +import { getSavedObjectKqlFilter } from '../../routes/common'; import { InvalidLocationError } from './normalizers/common_fields'; import { SyntheticsServerSetup } from '../../types'; import { RouteContext } from '../../routes/types'; @@ -337,7 +337,10 @@ export class ProjectMonitorFormatter { monitors: Array> ) => { const configIds = monitors.map((monitor) => monitor.attributes[ConfigKey.CONFIG_ID]); - const monitorFilter = getKqlFilter({ field: ConfigKey.CONFIG_ID, values: configIds }); + const monitorFilter = getSavedObjectKqlFilter({ + field: ConfigKey.CONFIG_ID, + values: configIds, + }); const finder = await this.encryptedSavedObjectsClient.createPointInTimeFinderDecryptedAsInternalUser( { From 36bf391e62482e35f8eee81818b4bab19c4231d7 Mon Sep 17 00:00:00 2001 From: Tre Date: Fri, 30 Aug 2024 09:30:16 +0100 Subject: [PATCH 05/12] [FTR][ML] Drop skip firefox tags from ml area ui tests (#191705) ## Summary Previously, the ML area was included in the Firefox config. Over time, we have reduced the set of tests that we run on Firefox due to stability issues with the geckodriver. Today, ML tests are not included in the [Firefox config](https://github.com/elastic/kibana/blob/main/x-pack/test/functional/config.firefox.js) at all, so the skipFirefox tags have no effect. --- x-pack/test/functional/apps/ml/data_frame_analytics/index.ts | 2 +- x-pack/test/functional/apps/ml/permissions/index.ts | 2 +- .../functional/apps/ml/short_tests/model_management/index.ts | 2 +- .../test/functional/apps/ml/short_tests/notifications/index.ts | 2 +- x-pack/test/functional/apps/ml/short_tests/settings/index.ts | 2 +- x-pack/test/functional/apps/ml/stack_management_jobs/index.ts | 2 +- .../functional_with_es_ssl/apps/discover_ml_uptime/ml/index.ts | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/index.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/index.ts index 8a2bc1449cd7e..53b21e013c06a 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/index.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/index.ts @@ -12,7 +12,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { const ml = getService('ml'); describe('machine learning - data frame analytics', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); before(async () => { await ml.securityCommon.createMlRoles(); diff --git a/x-pack/test/functional/apps/ml/permissions/index.ts b/x-pack/test/functional/apps/ml/permissions/index.ts index 8b28c9e6ccda4..224544a015d8a 100644 --- a/x-pack/test/functional/apps/ml/permissions/index.ts +++ b/x-pack/test/functional/apps/ml/permissions/index.ts @@ -12,7 +12,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { const ml = getService('ml'); describe('machine learning - permissions', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); before(async () => { await ml.securityCommon.createMlRoles(); diff --git a/x-pack/test/functional/apps/ml/short_tests/model_management/index.ts b/x-pack/test/functional/apps/ml/short_tests/model_management/index.ts index c20957beb1ea5..55cb854c69efd 100644 --- a/x-pack/test/functional/apps/ml/short_tests/model_management/index.ts +++ b/x-pack/test/functional/apps/ml/short_tests/model_management/index.ts @@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('model management', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); loadTestFile(require.resolve('./model_list')); }); diff --git a/x-pack/test/functional/apps/ml/short_tests/notifications/index.ts b/x-pack/test/functional/apps/ml/short_tests/notifications/index.ts index e026d44a67af2..d7756a75a66de 100644 --- a/x-pack/test/functional/apps/ml/short_tests/notifications/index.ts +++ b/x-pack/test/functional/apps/ml/short_tests/notifications/index.ts @@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('Notifcations', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); loadTestFile(require.resolve('./notification_list')); }); diff --git a/x-pack/test/functional/apps/ml/short_tests/settings/index.ts b/x-pack/test/functional/apps/ml/short_tests/settings/index.ts index d3f7000918a8e..2f46e75038ff9 100644 --- a/x-pack/test/functional/apps/ml/short_tests/settings/index.ts +++ b/x-pack/test/functional/apps/ml/short_tests/settings/index.ts @@ -9,7 +9,7 @@ import { FtrProviderContext } from '../../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('settings', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); loadTestFile(require.resolve('./calendar_creation')); loadTestFile(require.resolve('./calendar_edit')); diff --git a/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts b/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts index 37f238dbeecc9..53f4b7cbf943e 100644 --- a/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts +++ b/x-pack/test/functional/apps/ml/stack_management_jobs/index.ts @@ -12,7 +12,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { const ml = getService('ml'); describe('machine learning - stack management jobs', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); before(async () => { await ml.securityCommon.createMlRoles(); await ml.securityCommon.createMlUsers(); diff --git a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/ml/index.ts b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/ml/index.ts index 7a898b63c3f22..ea21b37e86a66 100644 --- a/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/ml/index.ts +++ b/x-pack/test/functional_with_es_ssl/apps/discover_ml_uptime/ml/index.ts @@ -12,7 +12,7 @@ export default ({ loadTestFile, getService }: FtrProviderContext) => { const esArchiver = getService('esArchiver'); describe('ML app', function () { - this.tags(['ml', 'skipFirefox']); + this.tags(['ml']); before(async () => { await ml.securityCommon.createMlRoles(); From ac1d5add478f0c04dd450b1ee45963bc469ea1ee Mon Sep 17 00:00:00 2001 From: natasha-moore-elastic <137783811+natasha-moore-elastic@users.noreply.github.com> Date: Fri, 30 Aug 2024 09:42:56 +0100 Subject: [PATCH 06/12] Improves Osquery API docs content (#191753) ## Summary Resolves https://github.com/elastic/security-docs-internal/issues/38 by improving the Osquery API docs content. Adds missing and improves existing operation summaries and operation descriptions to adhere to our [OAS standards](https://elasticco.atlassian.net/wiki/spaces/DOC/pages/450494532/API+reference+docs). --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../api/live_query/live_queries.schema.yaml | 6 ++- .../common/api/packs/packs.schema.yaml | 18 +++++-- .../api/saved_query/saved_query.schema.yaml | 14 ++++-- ...osquery_api_2023_10_31.bundled.schema.yaml | 38 +++++++++++---- ...osquery_api_2023_10_31.bundled.schema.yaml | 38 +++++++++++---- .../security_solution_osquery_api.gen.ts | 48 +++++++++++++++++++ 6 files changed, 135 insertions(+), 27 deletions(-) diff --git a/x-pack/plugins/osquery/common/api/live_query/live_queries.schema.yaml b/x-pack/plugins/osquery/common/api/live_query/live_queries.schema.yaml index 964d0938676a9..fa9fc90742bbb 100644 --- a/x-pack/plugins/osquery/common/api/live_query/live_queries.schema.yaml +++ b/x-pack/plugins/osquery/common/api/live_query/live_queries.schema.yaml @@ -5,7 +5,8 @@ info: paths: /api/osquery/live_queries: get: - summary: Find live queries + summary: Get live queries + description: Get a list of all live queries. operationId: OsqueryFindLiveQueries x-codegen-enabled: true x-labels: [serverless, ess] @@ -25,6 +26,7 @@ paths: post: summary: Create a live query + description: Create and run a live query. operationId: OsqueryCreateLiveQuery x-codegen-enabled: true x-labels: [serverless, ess] @@ -45,6 +47,7 @@ paths: /api/osquery/live_queries/{id}: get: summary: Get live query details + description: Get the details of a live query using the query ID. operationId: OsqueryGetLiveQueryDetails x-codegen-enabled: true x-labels: [serverless, ess] @@ -70,6 +73,7 @@ paths: /api/osquery/live_queries/{id}/results/{actionId}: get: summary: Get live query results + description: Get the results of a live query using the query action ID. operationId: OsqueryGetLiveQueryResults x-codegen-enabled: true x-labels: [serverless, ess] diff --git a/x-pack/plugins/osquery/common/api/packs/packs.schema.yaml b/x-pack/plugins/osquery/common/api/packs/packs.schema.yaml index bc9afa2ff0964..7bdb90151b73d 100644 --- a/x-pack/plugins/osquery/common/api/packs/packs.schema.yaml +++ b/x-pack/plugins/osquery/common/api/packs/packs.schema.yaml @@ -5,7 +5,8 @@ info: paths: /api/osquery/packs: get: - summary: Find packs + summary: Get packs + description: Get a list of all query packs. operationId: OsqueryFindPacks x-codegen-enabled: true x-labels: [serverless, ess] @@ -23,7 +24,8 @@ paths: schema: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' post: - summary: Create a packs + summary: Create a pack + description: Create a query pack. operationId: OsqueryCreatePacks x-codegen-enabled: true x-labels: [serverless, ess] @@ -42,7 +44,8 @@ paths: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' /api/osquery/packs/{id}: get: - summary: Get packs details + summary: Get pack details + description: Get the details of a query pack using the pack ID. operationId: OsqueryGetPacksDetails x-codegen-enabled: true x-labels: [serverless, ess] @@ -60,7 +63,8 @@ paths: schema: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' delete: - summary: Delete packs + summary: Delete a pack + description: Delete a query pack using the pack ID. operationId: OsqueryDeletePacks x-codegen-enabled: true x-labels: [serverless, ess] @@ -78,7 +82,11 @@ paths: schema: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' put: - summary: Update packs + summary: Update a pack + description: | + Update a query pack using the pack ID. + > info + > You cannot update a prebuilt pack. operationId: OsqueryUpdatePacks x-codegen-enabled: true x-labels: [serverless, ess] diff --git a/x-pack/plugins/osquery/common/api/saved_query/saved_query.schema.yaml b/x-pack/plugins/osquery/common/api/saved_query/saved_query.schema.yaml index 181dbb6350b56..359770016fb82 100644 --- a/x-pack/plugins/osquery/common/api/saved_query/saved_query.schema.yaml +++ b/x-pack/plugins/osquery/common/api/saved_query/saved_query.schema.yaml @@ -5,7 +5,8 @@ info: paths: /api/osquery/saved_queries: get: - summary: Find saved queries + summary: Get saved queries + description: Get a list of all saved queries. operationId: OsqueryFindSavedQueries x-codegen-enabled: true x-labels: [serverless, ess] @@ -24,6 +25,7 @@ paths: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' post: summary: Create a saved query + description: Create and run a saved query. operationId: OsqueryCreateSavedQuery x-codegen-enabled: true x-labels: [serverless, ess] @@ -43,6 +45,7 @@ paths: /api/osquery/saved_queries/{id}: get: summary: Get saved query details + description: Get the details of a saved query using the query ID. operationId: OsqueryGetSavedQueryDetails x-codegen-enabled: true x-labels: [serverless, ess] @@ -60,7 +63,8 @@ paths: schema: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' delete: - summary: Delete saved query + summary: Delete a saved query + description: Delete a saved query using the query ID. operationId: OsqueryDeleteSavedQuery x-codegen-enabled: true x-labels: [serverless, ess] @@ -78,7 +82,11 @@ paths: schema: $ref: '../model/schema/common_attributes.schema.yaml#/components/schemas/DefaultSuccessResponse' put: - summary: Update saved query + summary: Update a saved query + description: | + Update a saved query using the query ID. + > info + > You cannot update a prebuilt saved query. operationId: OsqueryUpdateSavedQuery x-codegen-enabled: true x-labels: [serverless, ess] diff --git a/x-pack/plugins/osquery/docs/openapi/ess/osquery_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/osquery/docs/openapi/ess/osquery_api_2023_10_31.bundled.schema.yaml index e8635fbc478e4..4f6933aef5f2f 100644 --- a/x-pack/plugins/osquery/docs/openapi/ess/osquery_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/osquery/docs/openapi/ess/osquery_api_2023_10_31.bundled.schema.yaml @@ -13,6 +13,7 @@ servers: paths: /api/osquery/live_queries: get: + description: Get a list of all live queries. operationId: OsqueryFindLiveQueries parameters: - in: query @@ -27,10 +28,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find live queries + summary: Get live queries tags: - Security Solution Osquery API post: + description: Create and run a live query. operationId: OsqueryCreateLiveQuery requestBody: content: @@ -50,6 +52,7 @@ paths: - Security Solution Osquery API '/api/osquery/live_queries/{id}': get: + description: Get the details of a live query using the query ID. operationId: OsqueryGetLiveQueryDetails parameters: - in: path @@ -74,6 +77,7 @@ paths: - Security Solution Osquery API '/api/osquery/live_queries/{id}/results/{actionId}': get: + description: Get the results of a live query using the query action ID. operationId: OsqueryGetLiveQueryResults parameters: - in: path @@ -103,6 +107,7 @@ paths: - Security Solution Osquery API /api/osquery/packs: get: + description: Get a list of all query packs. operationId: OsqueryFindPacks parameters: - in: query @@ -117,10 +122,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find packs + summary: Get packs tags: - Security Solution Osquery API post: + description: Create a query pack. operationId: OsqueryCreatePacks requestBody: content: @@ -135,11 +141,12 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Create a packs + summary: Create a pack tags: - Security Solution Osquery API '/api/osquery/packs/{id}': delete: + description: Delete a query pack using the pack ID. operationId: OsqueryDeletePacks parameters: - in: path @@ -154,10 +161,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Delete packs + summary: Delete a pack tags: - Security Solution Osquery API get: + description: Get the details of a query pack using the pack ID. operationId: OsqueryGetPacksDetails parameters: - in: path @@ -172,10 +180,14 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Get packs details + summary: Get pack details tags: - Security Solution Osquery API put: + description: | + Update a query pack using the pack ID. + > info + > You cannot update a prebuilt pack. operationId: OsqueryUpdatePacks parameters: - in: path @@ -196,11 +208,12 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Update packs + summary: Update a pack tags: - Security Solution Osquery API /api/osquery/saved_queries: get: + description: Get a list of all saved queries. operationId: OsqueryFindSavedQueries parameters: - in: query @@ -215,10 +228,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find saved queries + summary: Get saved queries tags: - Security Solution Osquery API post: + description: Create and run a saved query. operationId: OsqueryCreateSavedQuery requestBody: content: @@ -238,6 +252,7 @@ paths: - Security Solution Osquery API '/api/osquery/saved_queries/{id}': delete: + description: Delete a saved query using the query ID. operationId: OsqueryDeleteSavedQuery parameters: - in: path @@ -252,10 +267,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Delete saved query + summary: Delete a saved query tags: - Security Solution Osquery API get: + description: Get the details of a saved query using the query ID. operationId: OsqueryGetSavedQueryDetails parameters: - in: path @@ -274,6 +290,10 @@ paths: tags: - Security Solution Osquery API put: + description: | + Update a saved query using the query ID. + > info + > You cannot update a prebuilt saved query. operationId: OsqueryUpdateSavedQuery parameters: - in: path @@ -294,7 +314,7 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Update saved query + summary: Update a saved query tags: - Security Solution Osquery API components: diff --git a/x-pack/plugins/osquery/docs/openapi/serverless/osquery_api_2023_10_31.bundled.schema.yaml b/x-pack/plugins/osquery/docs/openapi/serverless/osquery_api_2023_10_31.bundled.schema.yaml index 5ee7cc382c480..836298b2e7cba 100644 --- a/x-pack/plugins/osquery/docs/openapi/serverless/osquery_api_2023_10_31.bundled.schema.yaml +++ b/x-pack/plugins/osquery/docs/openapi/serverless/osquery_api_2023_10_31.bundled.schema.yaml @@ -13,6 +13,7 @@ servers: paths: /api/osquery/live_queries: get: + description: Get a list of all live queries. operationId: OsqueryFindLiveQueries parameters: - in: query @@ -27,10 +28,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find live queries + summary: Get live queries tags: - Security Solution Osquery API post: + description: Create and run a live query. operationId: OsqueryCreateLiveQuery requestBody: content: @@ -50,6 +52,7 @@ paths: - Security Solution Osquery API '/api/osquery/live_queries/{id}': get: + description: Get the details of a live query using the query ID. operationId: OsqueryGetLiveQueryDetails parameters: - in: path @@ -74,6 +77,7 @@ paths: - Security Solution Osquery API '/api/osquery/live_queries/{id}/results/{actionId}': get: + description: Get the results of a live query using the query action ID. operationId: OsqueryGetLiveQueryResults parameters: - in: path @@ -103,6 +107,7 @@ paths: - Security Solution Osquery API /api/osquery/packs: get: + description: Get a list of all query packs. operationId: OsqueryFindPacks parameters: - in: query @@ -117,10 +122,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find packs + summary: Get packs tags: - Security Solution Osquery API post: + description: Create a query pack. operationId: OsqueryCreatePacks requestBody: content: @@ -135,11 +141,12 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Create a packs + summary: Create a pack tags: - Security Solution Osquery API '/api/osquery/packs/{id}': delete: + description: Delete a query pack using the pack ID. operationId: OsqueryDeletePacks parameters: - in: path @@ -154,10 +161,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Delete packs + summary: Delete a pack tags: - Security Solution Osquery API get: + description: Get the details of a query pack using the pack ID. operationId: OsqueryGetPacksDetails parameters: - in: path @@ -172,10 +180,14 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Get packs details + summary: Get pack details tags: - Security Solution Osquery API put: + description: | + Update a query pack using the pack ID. + > info + > You cannot update a prebuilt pack. operationId: OsqueryUpdatePacks parameters: - in: path @@ -196,11 +208,12 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Update packs + summary: Update a pack tags: - Security Solution Osquery API /api/osquery/saved_queries: get: + description: Get a list of all saved queries. operationId: OsqueryFindSavedQueries parameters: - in: query @@ -215,10 +228,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Find saved queries + summary: Get saved queries tags: - Security Solution Osquery API post: + description: Create and run a saved query. operationId: OsqueryCreateSavedQuery requestBody: content: @@ -238,6 +252,7 @@ paths: - Security Solution Osquery API '/api/osquery/saved_queries/{id}': delete: + description: Delete a saved query using the query ID. operationId: OsqueryDeleteSavedQuery parameters: - in: path @@ -252,10 +267,11 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Delete saved query + summary: Delete a saved query tags: - Security Solution Osquery API get: + description: Get the details of a saved query using the query ID. operationId: OsqueryGetSavedQueryDetails parameters: - in: path @@ -274,6 +290,10 @@ paths: tags: - Security Solution Osquery API put: + description: | + Update a saved query using the query ID. + > info + > You cannot update a prebuilt saved query. operationId: OsqueryUpdateSavedQuery parameters: - in: path @@ -294,7 +314,7 @@ paths: schema: $ref: '#/components/schemas/DefaultSuccessResponse' description: OK - summary: Update saved query + summary: Update a saved query tags: - Security Solution Osquery API components: diff --git a/x-pack/test/api_integration/services/security_solution_osquery_api.gen.ts b/x-pack/test/api_integration/services/security_solution_osquery_api.gen.ts index 1093c30a5d357..c133ebce15c90 100644 --- a/x-pack/test/api_integration/services/security_solution_osquery_api.gen.ts +++ b/x-pack/test/api_integration/services/security_solution_osquery_api.gen.ts @@ -93,6 +93,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Create and run a live query. + */ osqueryCreateLiveQuery(props: OsqueryCreateLiveQueryProps) { return supertest .post('/api/osquery/live_queries') @@ -101,6 +104,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Create a query pack. + */ osqueryCreatePacks(props: OsqueryCreatePacksProps) { return supertest .post('/api/osquery/packs') @@ -109,6 +115,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Create and run a saved query. + */ osqueryCreateSavedQuery(props: OsqueryCreateSavedQueryProps) { return supertest .post('/api/osquery/saved_queries') @@ -117,6 +126,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Delete a query pack using the pack ID. + */ osqueryDeletePacks(props: OsqueryDeletePacksProps) { return supertest .delete(replaceParams('/api/osquery/packs/{id}', props.params)) @@ -124,6 +136,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Delete a saved query using the query ID. + */ osqueryDeleteSavedQuery(props: OsqueryDeleteSavedQueryProps) { return supertest .delete(replaceParams('/api/osquery/saved_queries/{id}', props.params)) @@ -131,6 +146,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Get a list of all live queries. + */ osqueryFindLiveQueries(props: OsqueryFindLiveQueriesProps) { return supertest .get('/api/osquery/live_queries') @@ -139,6 +157,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get a list of all query packs. + */ osqueryFindPacks(props: OsqueryFindPacksProps) { return supertest .get('/api/osquery/packs') @@ -147,6 +168,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get a list of all saved queries. + */ osqueryFindSavedQueries(props: OsqueryFindSavedQueriesProps) { return supertest .get('/api/osquery/saved_queries') @@ -155,6 +179,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get the details of a live query using the query ID. + */ osqueryGetLiveQueryDetails(props: OsqueryGetLiveQueryDetailsProps) { return supertest .get(replaceParams('/api/osquery/live_queries/{id}', props.params)) @@ -163,6 +190,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get the results of a live query using the query action ID. + */ osqueryGetLiveQueryResults(props: OsqueryGetLiveQueryResultsProps) { return supertest .get(replaceParams('/api/osquery/live_queries/{id}/results/{actionId}', props.params)) @@ -171,6 +201,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .query(props.query); }, + /** + * Get the details of a query pack using the pack ID. + */ osqueryGetPacksDetails(props: OsqueryGetPacksDetailsProps) { return supertest .get(replaceParams('/api/osquery/packs/{id}', props.params)) @@ -178,6 +211,9 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Get the details of a saved query using the query ID. + */ osqueryGetSavedQueryDetails(props: OsqueryGetSavedQueryDetailsProps) { return supertest .get(replaceParams('/api/osquery/saved_queries/{id}', props.params)) @@ -185,6 +221,12 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(ELASTIC_HTTP_VERSION_HEADER, '2023-10-31') .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana'); }, + /** + * Update a query pack using the pack ID. +> info +> You cannot update a prebuilt pack. + + */ osqueryUpdatePacks(props: OsqueryUpdatePacksProps) { return supertest .put(replaceParams('/api/osquery/packs/{id}', props.params)) @@ -193,6 +235,12 @@ export function SecuritySolutionApiProvider({ getService }: FtrProviderContext) .set(X_ELASTIC_INTERNAL_ORIGIN_REQUEST, 'kibana') .send(props.body as object); }, + /** + * Update a saved query using the query ID. +> info +> You cannot update a prebuilt saved query. + + */ osqueryUpdateSavedQuery(props: OsqueryUpdateSavedQueryProps) { return supertest .put(replaceParams('/api/osquery/saved_queries/{id}', props.params)) From f26463267ae680e80817d05613e93a71eedde87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Fri, 30 Aug 2024 11:08:46 +0200 Subject: [PATCH 07/12] [Move `@kbn/config-schema` to server] sharedux plugins (#191782) --- src/plugins/no_data_page/public/plugin.ts | 2 +- src/plugins/no_data_page/{ => server}/config.ts | 0 src/plugins/no_data_page/server/index.ts | 2 +- src/plugins/no_data_page/tsconfig.json | 2 +- src/plugins/share/public/index.ts | 2 +- src/plugins/share/{common => server}/config.ts | 0 src/plugins/share/server/index.ts | 2 +- 7 files changed, 5 insertions(+), 5 deletions(-) rename src/plugins/no_data_page/{ => server}/config.ts (100%) rename src/plugins/share/{common => server}/config.ts (100%) diff --git a/src/plugins/no_data_page/public/plugin.ts b/src/plugins/no_data_page/public/plugin.ts index 910208f0f94be..dc7dff0e0781e 100644 --- a/src/plugins/no_data_page/public/plugin.ts +++ b/src/plugins/no_data_page/public/plugin.ts @@ -8,7 +8,7 @@ import type { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public'; import type { NoDataPagePublicSetup, NoDataPagePublicStart } from './types'; -import type { NoDataPageConfig } from '../config'; +import type { NoDataPageConfig } from '../server/config'; export class NoDataPagePlugin implements Plugin { constructor(private initializerContext: PluginInitializerContext) {} diff --git a/src/plugins/no_data_page/config.ts b/src/plugins/no_data_page/server/config.ts similarity index 100% rename from src/plugins/no_data_page/config.ts rename to src/plugins/no_data_page/server/config.ts diff --git a/src/plugins/no_data_page/server/index.ts b/src/plugins/no_data_page/server/index.ts index ba02a016a9676..cabe47bf65a33 100644 --- a/src/plugins/no_data_page/server/index.ts +++ b/src/plugins/no_data_page/server/index.ts @@ -8,7 +8,7 @@ import { PluginConfigDescriptor } from '@kbn/core-plugins-server'; -import { configSchema, NoDataPageConfig } from '../config'; +import { configSchema, NoDataPageConfig } from './config'; export const config: PluginConfigDescriptor = { exposeToBrowser: { diff --git a/src/plugins/no_data_page/tsconfig.json b/src/plugins/no_data_page/tsconfig.json index bab1c8c23edfb..e92a0c1560380 100644 --- a/src/plugins/no_data_page/tsconfig.json +++ b/src/plugins/no_data_page/tsconfig.json @@ -3,7 +3,7 @@ "compilerOptions": { "outDir": "target/types" }, - "include": ["common/**/*", "public/**/*", "server/**/*", "config.ts"], + "include": ["common/**/*", "public/**/*", "server/**/*"], "kbn_references": [ "@kbn/core", "@kbn/core-plugins-browser", diff --git a/src/plugins/share/public/index.ts b/src/plugins/share/public/index.ts index f212081133b2d..842f31bf11dc9 100644 --- a/src/plugins/share/public/index.ts +++ b/src/plugins/share/public/index.ts @@ -8,7 +8,7 @@ import type { PluginInitializerContext } from '@kbn/core/public'; -export type { ConfigSchema } from '../common/config'; +export type { ConfigSchema } from '../server/config'; export { CSV_QUOTE_VALUES_SETTING, CSV_SEPARATOR_SETTING } from '../common/constants'; diff --git a/src/plugins/share/common/config.ts b/src/plugins/share/server/config.ts similarity index 100% rename from src/plugins/share/common/config.ts rename to src/plugins/share/server/config.ts diff --git a/src/plugins/share/server/index.ts b/src/plugins/share/server/index.ts index 5f61ba8693814..c3bac70cc6e25 100644 --- a/src/plugins/share/server/index.ts +++ b/src/plugins/share/server/index.ts @@ -7,7 +7,7 @@ */ import { PluginConfigDescriptor, PluginInitializerContext } from '@kbn/core/server'; -import { ConfigSchema, configSchema } from '../common/config'; +import { ConfigSchema, configSchema } from './config'; export type { SharePublicSetup as SharePluginSetup, From 0a99955c591c2df9918bd19ed5a6df24898c8d90 Mon Sep 17 00:00:00 2001 From: Kevin Lacabane Date: Fri, 30 Aug 2024 11:24:07 +0200 Subject: [PATCH 08/12] [eem] add server entity client (#191645) --- .../server/lib/entity_client.ts | 81 +++++++++++++++++++ .../entity_manager/server/plugin.ts | 8 ++ .../server/routes/entities/create.ts | 25 ++---- .../server/routes/entities/delete.ts | 20 ++--- .../server/routes/entities/get.ts | 19 ++--- .../entity_manager/server/routes/types.ts | 4 +- 6 files changed, 113 insertions(+), 44 deletions(-) create mode 100644 x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts diff --git a/x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts new file mode 100644 index 0000000000000..7fd0bb3c5ee18 --- /dev/null +++ b/x-pack/plugins/observability_solution/entity_manager/server/lib/entity_client.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EntityDefinition } from '@kbn/entities-schema'; +import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { Logger } from '@kbn/logging'; +import { installEntityDefinition } from './entities/install_entity_definition'; +import { startTransform } from './entities/start_transform'; +import { findEntityDefinitions } from './entities/find_entity_definition'; +import { uninstallEntityDefinition } from './entities/uninstall_entity_definition'; +import { EntityDefinitionNotFound } from './entities/errors/entity_not_found'; + +export class EntityClient { + constructor( + private options: { + esClient: ElasticsearchClient; + soClient: SavedObjectsClientContract; + logger: Logger; + } + ) {} + + async createEntityDefinition({ + definition, + installOnly = false, + }: { + definition: EntityDefinition; + installOnly?: boolean; + }) { + const installedDefinition = await installEntityDefinition({ + definition, + soClient: this.options.soClient, + esClient: this.options.esClient, + logger: this.options.logger, + }); + + if (!installOnly) { + await startTransform(this.options.esClient, definition, this.options.logger); + } + + return installedDefinition; + } + + async deleteEntityDefinition({ id, deleteData = false }: { id: string; deleteData?: boolean }) { + const [definition] = await findEntityDefinitions({ + id, + perPage: 1, + soClient: this.options.soClient, + esClient: this.options.esClient, + }); + + if (!definition) { + const message = `Unable to find entity definition with [${id}]`; + this.options.logger.error(message); + throw new EntityDefinitionNotFound(message); + } + + await uninstallEntityDefinition({ + definition, + deleteData, + soClient: this.options.soClient, + esClient: this.options.esClient, + logger: this.options.logger, + }); + } + + async getEntityDefinitions({ page = 1, perPage = 10 }: { page?: number; perPage?: number }) { + const definitions = await findEntityDefinitions({ + esClient: this.options.esClient, + soClient: this.options.soClient, + page, + perPage, + }); + + return { definitions }; + } +} diff --git a/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts b/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts index de9e8bec2826f..d65a2a9e186f3 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/plugin.ts @@ -14,6 +14,7 @@ import { PluginInitializerContext, PluginConfigDescriptor, Logger, + KibanaRequest, } from '@kbn/core/server'; import { installEntityManagerTemplates } from './lib/manage_index_templates'; import { setupRoutes } from './routes'; @@ -26,6 +27,7 @@ import { EntityManagerConfig, configSchema, exposeToBrowserConfig } from '../com import { entityDefinition, EntityDiscoveryApiKeyType } from './saved_objects'; import { upgradeBuiltInEntityDefinitions } from './lib/entities/upgrade_entity_definition'; import { builtInDefinitions } from './lib/entities/built_in'; +import { EntityClient } from './lib/entity_client'; export type EntityManagerServerPluginSetup = ReturnType; export type EntityManagerServerPluginStart = ReturnType; @@ -73,6 +75,12 @@ export class EntityManagerServerPlugin router, logger: this.logger, server: this.server, + getScopedClient: async ({ request }: { request: KibanaRequest }) => { + const [coreStart] = await core.getStartServices(); + const esClient = coreStart.elasticsearch.client.asScoped(request).asCurrentUser; + const soClient = coreStart.savedObjects.getScopedClient(request); + return new EntityClient({ esClient, soClient, logger: this.logger }); + }, }); return {}; diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/create.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/create.ts index 973ba3507c455..62a2b88cd99f8 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/create.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/create.ts @@ -16,8 +16,6 @@ import { SetupRouteOptions } from '../types'; import { EntityIdConflict } from '../../lib/entities/errors/entity_id_conflict_error'; import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception'; import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; -import { startTransform } from '../../lib/entities/start_transform'; -import { installEntityDefinition } from '../../lib/entities/install_entity_definition'; import { EntityDefinitionIdInvalid } from '../../lib/entities/errors/entity_definition_id_invalid'; /** @@ -56,7 +54,8 @@ import { EntityDefinitionIdInvalid } from '../../lib/entities/errors/entity_defi */ export function createEntityDefinitionRoute({ router, - server, + getScopedClient, + logger, }: SetupRouteOptions) { router.post( { @@ -66,24 +65,14 @@ export function createEntityDefinitionRoute({ query: createEntityDefinitionQuerySchema, }, }, - async (context, req, res) => { - const { logger } = server; - const core = await context.core; - const soClient = core.savedObjects.client; - const esClient = core.elasticsearch.client.asCurrentUser; - + async (context, request, res) => { try { - const definition = await installEntityDefinition({ - soClient, - esClient, - logger, - definition: req.body, + const client = await getScopedClient({ request }); + const definition = await client.createEntityDefinition({ + definition: request.body, + installOnly: request.query.installOnly, }); - if (!req.query.installOnly) { - await startTransform(esClient, definition, logger); - } - return res.ok({ body: definition }); } catch (e) { logger.error(e); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/delete.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/delete.ts index d0748e5b52e67..c2798aef9eb14 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/delete.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/delete.ts @@ -13,9 +13,7 @@ import { import { SetupRouteOptions } from '../types'; import { EntitySecurityException } from '../../lib/entities/errors/entity_security_exception'; import { InvalidTransformError } from '../../lib/entities/errors/invalid_transform_error'; -import { readEntityDefinition } from '../../lib/entities/read_entity_definition'; import { EntityDefinitionNotFound } from '../../lib/entities/errors/entity_not_found'; -import { uninstallEntityDefinition } from '../../lib/entities/uninstall_entity_definition'; /** * @openapi @@ -53,7 +51,7 @@ import { uninstallEntityDefinition } from '../../lib/entities/uninstall_entity_d */ export function deleteEntityDefinitionRoute({ router, - server, + getScopedClient, logger, }: SetupRouteOptions) { router.delete<{ id: string }, { deleteData?: boolean }, unknown>( @@ -64,18 +62,12 @@ export function deleteEntityDefinitionRoute({ query: deleteEntityDefinitionQuerySchema.strict(), }, }, - async (context, req, res) => { + async (context, request, res) => { try { - const soClient = (await context.core).savedObjects.client; - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - - const definition = await readEntityDefinition(soClient, req.params.id, logger); - await uninstallEntityDefinition({ - definition, - soClient, - esClient, - logger, - deleteData: req.query.deleteData, + const client = await getScopedClient({ request }); + await client.deleteEntityDefinition({ + id: request.params.id, + deleteData: request.query.deleteData, }); return res.ok({ body: { acknowledged: true } }); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts index 8039ee176a9b1..454679779c6a9 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/entities/get.ts @@ -9,7 +9,6 @@ import { z } from '@kbn/zod'; import { RequestHandlerContext } from '@kbn/core/server'; import { getEntityDefinitionQuerySchema } from '@kbn/entities-schema'; import { SetupRouteOptions } from '../types'; -import { findEntityDefinitions } from '../../lib/entities/find_entity_definition'; /** * @openapi @@ -52,6 +51,7 @@ import { findEntityDefinitions } from '../../lib/entities/find_entity_definition */ export function getEntityDefinitionRoute({ router, + getScopedClient, logger, }: SetupRouteOptions) { router.get<{ id?: string }, { page?: number; perPage?: number }, unknown>( @@ -62,18 +62,15 @@ export function getEntityDefinitionRoute({ params: z.object({ id: z.optional(z.string()) }), }, }, - async (context, req, res) => { + async (context, request, res) => { try { - const esClient = (await context.core).elasticsearch.client.asCurrentUser; - const soClient = (await context.core).savedObjects.client; - const definitions = await findEntityDefinitions({ - esClient, - soClient, - page: req.query.page ?? 1, - perPage: req.query.perPage ?? 10, - id: req.params.id, + const client = await getScopedClient({ request }); + const result = await client.getEntityDefinitions({ + page: request.query.page, + perPage: request.query.perPage, }); - return res.ok({ body: { definitions } }); + + return res.ok({ body: result }); } catch (e) { logger.error(e); return res.customError({ body: e, statusCode: 500 }); diff --git a/x-pack/plugins/observability_solution/entity_manager/server/routes/types.ts b/x-pack/plugins/observability_solution/entity_manager/server/routes/types.ts index 8e3dc7111298d..d4d8cfba815ae 100644 --- a/x-pack/plugins/observability_solution/entity_manager/server/routes/types.ts +++ b/x-pack/plugins/observability_solution/entity_manager/server/routes/types.ts @@ -5,12 +5,14 @@ * 2.0. */ -import { IRouter, RequestHandlerContextBase } from '@kbn/core-http-server'; +import { IRouter, KibanaRequest, RequestHandlerContextBase } from '@kbn/core-http-server'; import { Logger } from '@kbn/core/server'; import { EntityManagerServerSetup } from '../types'; +import { EntityClient } from '../lib/entity_client'; export interface SetupRouteOptions { router: IRouter; server: EntityManagerServerSetup; logger: Logger; + getScopedClient: ({ request }: { request: KibanaRequest }) => Promise; } From fbbed11a0fdbf84b346e3f9fd3f8d4b22bf26de6 Mon Sep 17 00:00:00 2001 From: jennypavlova Date: Fri, 30 Aug 2024 11:26:18 +0200 Subject: [PATCH 09/12] [Infra][Flaky Test Fix] Failing test: Single Host Flyout Tabs Metadata Tab should render metadata tab, add and remove filter (#191745) --- .../tabs/metadata/add_metadata_filter_button.tsx | 2 +- x-pack/test/functional/page_objects/asset_details.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx b/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx index f1e0b9073c036..49dfd3d42d5c4 100644 --- a/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx +++ b/x-pack/plugins/observability_solution/infra/public/components/asset_details/tabs/metadata/add_metadata_filter_button.tsx @@ -90,7 +90,7 @@ export const AddMetadataFilterButton = ({ item }: AddMetadataFilterButtonProps) } return ( - + Date: Fri, 30 Aug 2024 20:35:47 +1000 Subject: [PATCH 10/12] skip failing test suite (#191806) --- x-pack/test/functional/apps/infra/hosts_view.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/infra/hosts_view.ts b/x-pack/test/functional/apps/infra/hosts_view.ts index ccea5affc9202..5f2dd01bb980e 100644 --- a/x-pack/test/functional/apps/infra/hosts_view.ts +++ b/x-pack/test/functional/apps/infra/hosts_view.ts @@ -298,7 +298,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { (await pageObjects.infraHostsView.isKPIChartsLoaded()) ); - describe('Hosts View', function () { + // Failing: See https://github.com/elastic/kibana/issues/191806 + describe.skip('Hosts View', function () { let synthEsInfraClient: InfraSynthtraceEsClient; let syntEsLogsClient: LogsSynthtraceEsClient; From 3e0a431144d6afbc35c5bab5cf09735d73cb879f Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Fri, 30 Aug 2024 12:47:06 +0200 Subject: [PATCH 11/12] Inference plugin: Add Bedrock model adapter (#191434) ## Summary Add the `bedrock` (well, bedrock-claude) model adapter for the inference plugin. Also had to perform minor changes on the associated connector to add support for new capabilities. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../connector_types.test.ts.snap | 373 +++++++++++++++++- .../bedrock/bedrock_claude_adapter.test.ts | 310 +++++++++++++++ .../bedrock/bedrock_claude_adapter.ts | 140 +++++++ .../chat_complete/adapters/bedrock/index.ts | 8 + .../bedrock/process_completion_chunks.test.ts | 336 ++++++++++++++++ .../bedrock/process_completion_chunks.ts | 113 ++++++ .../chat_complete/adapters/bedrock/prompts.ts | 15 + .../serde_eventstream_into_observable.test.ts | 87 ++++ .../serde_eventstream_into_observable.ts | 76 ++++ .../adapters/bedrock/serde_utils.test.ts | 20 + .../adapters/bedrock/serde_utils.ts | 33 ++ .../chat_complete/adapters/bedrock/types.ts | 98 +++++ .../adapters/get_inference_adapter.test.ts | 5 +- .../adapters/get_inference_adapter.ts | 4 +- x-pack/plugins/inference/tsconfig.json | 3 +- .../stack_connectors/common/bedrock/schema.ts | 38 +- .../stack_connectors/common/bedrock/types.ts | 4 + .../server/connector_types/bedrock/bedrock.ts | 60 ++- 18 files changed, 1691 insertions(+), 32 deletions(-) create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.test.ts create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.ts create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/bedrock/index.ts create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.test.ts create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.ts create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/bedrock/prompts.ts create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.test.ts create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.ts create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.test.ts create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.ts create mode 100644 x-pack/plugins/inference/server/chat_complete/adapters/bedrock/types.ts diff --git a/x-pack/plugins/actions/server/integration_tests/__snapshots__/connector_types.test.ts.snap b/x-pack/plugins/actions/server/integration_tests/__snapshots__/connector_types.test.ts.snap index ae1289b2a9e2e..6e4392e3f737e 100644 --- a/x-pack/plugins/actions/server/integration_tests/__snapshots__/connector_types.test.ts.snap +++ b/x-pack/plugins/actions/server/integration_tests/__snapshots__/connector_types.test.ts.snap @@ -273,8 +273,15 @@ Object { "keys": Object { "content": Object { "flags": Object { + "default": [Function], "error": [Function], + "presence": "optional", }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], "rules": Array [ Object { "args": Object { @@ -285,6 +292,33 @@ Object { ], "type": "string", }, + "rawContent": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "items": Array [ + Object { + "flags": Object { + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-any-type": true, + }, + ], + "type": "any", + }, + ], + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "array", + }, "role": Object { "flags": Object { "error": [Function], @@ -300,6 +334,14 @@ Object { "type": "string", }, }, + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], "type": "object", }, ], @@ -419,6 +461,86 @@ Object { ], "type": "number", }, + "toolChoice": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "keys": Object { + "name": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], + "type": "string", + }, + "type": Object { + "flags": Object { + "error": [Function], + }, + "matches": Array [ + Object { + "schema": Object { + "allow": Array [ + "auto", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "any", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "tool", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + ], + "type": "alternatives", + }, + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "object", + }, "tools": Object { "flags": Object { "default": [Function], @@ -556,8 +678,15 @@ Object { "keys": Object { "content": Object { "flags": Object { + "default": [Function], "error": [Function], + "presence": "optional", }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], "rules": Array [ Object { "args": Object { @@ -568,6 +697,33 @@ Object { ], "type": "string", }, + "rawContent": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "items": Array [ + Object { + "flags": Object { + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-any-type": true, + }, + ], + "type": "any", + }, + ], + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "array", + }, "role": Object { "flags": Object { "error": [Function], @@ -583,6 +739,14 @@ Object { "type": "string", }, }, + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], "type": "object", }, ], @@ -702,6 +866,86 @@ Object { ], "type": "number", }, + "toolChoice": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "keys": Object { + "name": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], + "type": "string", + }, + "type": Object { + "flags": Object { + "error": [Function], + }, + "matches": Array [ + Object { + "schema": Object { + "allow": Array [ + "auto", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "any", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "tool", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + ], + "type": "alternatives", + }, + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "object", + }, "tools": Object { "flags": Object { "default": [Function], @@ -839,14 +1083,51 @@ Object { "keys": Object { "content": Object { "flags": Object { + "default": [Function], "error": [Function], + "presence": "optional", }, "metas": Array [ Object { - "x-oas-any-type": true, + "x-oas-optional": true, }, ], - "type": "any", + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], + "type": "string", + }, + "rawContent": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "items": Array [ + Object { + "flags": Object { + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-any-type": true, + }, + ], + "type": "any", + }, + ], + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "array", }, "role": Object { "flags": Object { @@ -863,6 +1144,14 @@ Object { "type": "string", }, }, + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], "type": "object", }, ], @@ -982,6 +1271,86 @@ Object { ], "type": "number", }, + "toolChoice": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "keys": Object { + "name": Object { + "flags": Object { + "default": [Function], + "error": [Function], + "presence": "optional", + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "rules": Array [ + Object { + "args": Object { + "method": [Function], + }, + "name": "custom", + }, + ], + "type": "string", + }, + "type": Object { + "flags": Object { + "error": [Function], + }, + "matches": Array [ + Object { + "schema": Object { + "allow": Array [ + "auto", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "any", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + Object { + "schema": Object { + "allow": Array [ + "tool", + ], + "flags": Object { + "error": [Function], + "only": true, + }, + "type": "any", + }, + }, + ], + "type": "alternatives", + }, + }, + "metas": Array [ + Object { + "x-oas-optional": true, + }, + ], + "type": "object", + }, "tools": Object { "flags": Object { "default": [Function], diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.test.ts new file mode 100644 index 0000000000000..1d25f09dce3bc --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.test.ts @@ -0,0 +1,310 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PassThrough } from 'stream'; +import type { InferenceExecutor } from '../../utils/inference_executor'; +import { MessageRole } from '../../../../common/chat_complete'; +import { ToolChoiceType } from '../../../../common/chat_complete/tools'; +import { bedrockClaudeAdapter } from './bedrock_claude_adapter'; +import { addNoToolUsageDirective } from './prompts'; + +describe('bedrockClaudeAdapter', () => { + const executorMock = { + invoke: jest.fn(), + } as InferenceExecutor & { invoke: jest.MockedFn }; + + beforeEach(() => { + executorMock.invoke.mockReset(); + executorMock.invoke.mockImplementation(async () => { + return { + actionId: '', + status: 'ok', + data: new PassThrough(), + }; + }); + }); + + function getCallParams() { + const params = executorMock.invoke.mock.calls[0][0].subActionParams as Record; + return { + system: params.system, + messages: params.messages, + tools: params.tools, + toolChoice: params.toolChoice, + }; + } + + describe('#chatComplete()', () => { + it('calls `executor.invoke` with the right fixed parameters', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + expect(executorMock.invoke).toHaveBeenCalledWith({ + subAction: 'invokeStream', + subActionParams: { + messages: [ + { + role: 'user', + rawContent: [{ type: 'text', text: 'question' }], + }, + ], + temperature: 0, + stopSequences: ['\n\nHuman:'], + }, + }); + }); + + it('correctly format tools', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + tools: { + myFunction: { + description: 'myFunction', + }, + myFunctionWithArgs: { + description: 'myFunctionWithArgs', + schema: { + type: 'object', + properties: { + foo: { + type: 'string', + description: 'foo', + }, + }, + required: ['foo'], + }, + }, + }, + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { tools } = getCallParams(); + expect(tools).toEqual([ + { + name: 'myFunction', + description: 'myFunction', + input_schema: { + properties: {}, + type: 'object', + }, + }, + { + name: 'myFunctionWithArgs', + description: 'myFunctionWithArgs', + input_schema: { + properties: { + foo: { + description: 'foo', + type: 'string', + }, + }, + required: ['foo'], + type: 'object', + }, + }, + ]); + }); + + it('correctly format messages', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + { + role: MessageRole.Assistant, + content: 'answer', + }, + { + role: MessageRole.User, + content: 'another question', + }, + { + role: MessageRole.Assistant, + content: null, + toolCalls: [ + { + function: { + name: 'my_function', + arguments: { + foo: 'bar', + }, + }, + toolCallId: '0', + }, + ], + }, + { + role: MessageRole.Tool, + toolCallId: '0', + response: { + bar: 'foo', + }, + }, + ], + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { messages } = getCallParams(); + expect(messages).toEqual([ + { + rawContent: [ + { + text: 'question', + type: 'text', + }, + ], + role: 'user', + }, + { + rawContent: [ + { + text: 'answer', + type: 'text', + }, + ], + role: 'assistant', + }, + { + rawContent: [ + { + text: 'another question', + type: 'text', + }, + ], + role: 'user', + }, + { + rawContent: [ + { + id: '0', + input: { + foo: 'bar', + }, + name: 'my_function', + type: 'tool_use', + }, + ], + role: 'assistant', + }, + { + rawContent: [ + { + content: '{"bar":"foo"}', + tool_use_id: '0', + type: 'tool_result', + }, + ], + role: 'user', + }, + ]); + }); + + it('correctly format system message', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + system: 'Some system message', + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { system } = getCallParams(); + expect(system).toEqual('Some system message'); + }); + + it('correctly format tool choice', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + toolChoice: ToolChoiceType.required, + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { toolChoice } = getCallParams(); + expect(toolChoice).toEqual({ + type: 'any', + }); + }); + + it('correctly format tool choice for named function', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + toolChoice: { function: 'foobar' }, + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { toolChoice } = getCallParams(); + expect(toolChoice).toEqual({ + type: 'tool', + name: 'foobar', + }); + }); + + it('correctly adapt the request for ToolChoiceType.None', () => { + bedrockClaudeAdapter.chatComplete({ + executor: executorMock, + system: 'some system instruction', + messages: [ + { + role: MessageRole.User, + content: 'question', + }, + ], + tools: { + myFunction: { + description: 'myFunction', + }, + }, + toolChoice: ToolChoiceType.none, + }); + + expect(executorMock.invoke).toHaveBeenCalledTimes(1); + + const { toolChoice, tools, system } = getCallParams(); + expect(toolChoice).toBeUndefined(); + expect(tools).toEqual([]); + expect(system).toEqual(addNoToolUsageDirective('some system instruction')); + }); + }); +}); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.ts new file mode 100644 index 0000000000000..5a03dc04347b1 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/bedrock_claude_adapter.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { filter, from, map, switchMap, tap } from 'rxjs'; +import { Readable } from 'stream'; +import type { InvokeAIActionParams } from '@kbn/stack-connectors-plugin/common/bedrock/types'; +import { parseSerdeChunkMessage } from './serde_utils'; +import { Message, MessageRole } from '../../../../common/chat_complete'; +import { createInferenceInternalError } from '../../../../common/errors'; +import { ToolChoiceType, type ToolOptions } from '../../../../common/chat_complete/tools'; +import { InferenceConnectorAdapter } from '../../types'; +import type { BedRockMessage, BedrockToolChoice } from './types'; +import { + BedrockChunkMember, + serdeEventstreamIntoObservable, +} from './serde_eventstream_into_observable'; +import { processCompletionChunks } from './process_completion_chunks'; +import { addNoToolUsageDirective } from './prompts'; + +export const bedrockClaudeAdapter: InferenceConnectorAdapter = { + chatComplete: ({ executor, system, messages, toolChoice, tools }) => { + const noToolUsage = toolChoice === ToolChoiceType.none; + + const connectorInvokeRequest: InvokeAIActionParams = { + system: noToolUsage ? addNoToolUsageDirective(system) : system, + messages: messagesToBedrock(messages), + tools: noToolUsage ? [] : toolsToBedrock(tools), + toolChoice: toolChoiceToBedrock(toolChoice), + temperature: 0, + stopSequences: ['\n\nHuman:'], + }; + + return from( + executor.invoke({ + subAction: 'invokeStream', + subActionParams: connectorInvokeRequest, + }) + ).pipe( + switchMap((response) => { + const readable = response.data as Readable; + return serdeEventstreamIntoObservable(readable); + }), + tap((eventData) => { + if ('modelStreamErrorException' in eventData) { + throw createInferenceInternalError(eventData.modelStreamErrorException.originalMessage); + } + }), + filter((value): value is BedrockChunkMember => { + return 'chunk' in value && value.chunk?.headers?.[':event-type']?.value === 'chunk'; + }), + map((message) => { + return parseSerdeChunkMessage(message.chunk); + }), + processCompletionChunks() + ); + }, +}; + +const toolChoiceToBedrock = ( + toolChoice: ToolOptions['toolChoice'] +): BedrockToolChoice | undefined => { + if (toolChoice === ToolChoiceType.required) { + return { + type: 'any', + }; + } else if (toolChoice === ToolChoiceType.auto) { + return { + type: 'auto', + }; + } else if (typeof toolChoice === 'object') { + return { + type: 'tool', + name: toolChoice.function, + }; + } + // ToolChoiceType.none is not supported by claude + // we are adding a directive to the system instructions instead in that case. + return undefined; +}; + +const toolsToBedrock = (tools: ToolOptions['tools']) => { + return tools + ? Object.entries(tools).map(([toolName, toolDef]) => { + return { + name: toolName, + description: toolDef.description, + input_schema: toolDef.schema ?? { + type: 'object' as const, + properties: {}, + }, + }; + }) + : undefined; +}; + +const messagesToBedrock = (messages: Message[]): BedRockMessage[] => { + return messages.map((message) => { + switch (message.role) { + case MessageRole.User: + return { + role: 'user' as const, + rawContent: [{ type: 'text' as const, text: message.content }], + }; + case MessageRole.Assistant: + return { + role: 'assistant' as const, + rawContent: [ + ...(message.content ? [{ type: 'text' as const, text: message.content }] : []), + ...(message.toolCalls + ? message.toolCalls.map((toolCall) => { + return { + type: 'tool_use' as const, + id: toolCall.toolCallId, + name: toolCall.function.name, + input: ('arguments' in toolCall.function + ? toolCall.function.arguments + : {}) as Record, + }; + }) + : []), + ], + }; + case MessageRole.Tool: + return { + role: 'user' as const, + rawContent: [ + { + type: 'tool_result' as const, + tool_use_id: message.toolCallId, + content: JSON.stringify(message.response), + }, + ], + }; + } + }); +}; diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/index.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/index.ts new file mode 100644 index 0000000000000..01d849e1ea9af --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { bedrockClaudeAdapter } from './bedrock_claude_adapter'; diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.test.ts new file mode 100644 index 0000000000000..6307aecaeefc4 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.test.ts @@ -0,0 +1,336 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { lastValueFrom, of, toArray } from 'rxjs'; +import { processCompletionChunks } from './process_completion_chunks'; +import type { CompletionChunk } from './types'; + +describe('processCompletionChunks', () => { + it('does not emit for a message_start event', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'message_start', + message: 'foo', + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual( + [] + ); + }); + + it('emits the correct value for a content_block_start event with text content ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: 'foo' }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionChunk', + content: 'foo', + tool_calls: [], + }, + ]); + }); + + it('emits the correct value for a content_block_start event with tool_use content ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_start', + index: 0, + content_block: { type: 'tool_use', id: 'id', name: 'name', input: '{}' }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionChunk', + content: '', + tool_calls: [ + { + toolCallId: 'id', + index: 0, + function: { + arguments: '', + name: 'name', + }, + }, + ], + }, + ]); + }); + + it('emits the correct value for a content_block_delta event with text content ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'delta' }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionChunk', + content: 'delta', + tool_calls: [], + }, + ]); + }); + + it('emits the correct value for a content_block_delta event with tool_use content ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_delta', + index: 0, + delta: { type: 'input_json_delta', partial_json: '{ "param' }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionChunk', + content: '', + tool_calls: [ + { + index: 0, + toolCallId: '', + function: { + arguments: '{ "param', + name: '', + }, + }, + ], + }, + ]); + }); + + it('does not emit for a content_block_stop event', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_stop', + index: 0, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual( + [] + ); + }); + + it('emits the correct value for a message_delta event with tool_use content ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'message_delta', + delta: { stop_reason: 'end_turn', stop_sequence: 'stop_seq', usage: { output_tokens: 42 } }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionChunk', + content: 'stop_seq', + tool_calls: [], + }, + ]); + }); + + it('emits a token count for a message_stop event ', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'message_stop', + 'amazon-bedrock-invocationMetrics': { + inputTokenCount: 1, + outputTokenCount: 2, + invocationLatency: 3, + firstByteLatency: 4, + }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + type: 'chatCompletionTokenCount', + tokens: { + completion: 2, + prompt: 1, + total: 3, + }, + }, + ]); + }); + + it('emits the correct values for a text response scenario', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'message_start', + message: 'foo', + }, + { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: 'foo' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'delta1' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'delta2' }, + }, + { + type: 'content_block_stop', + index: 0, + }, + { + type: 'message_delta', + delta: { stop_reason: 'end_turn', stop_sequence: 'stop_seq', usage: { output_tokens: 42 } }, + }, + { + type: 'message_stop', + 'amazon-bedrock-invocationMetrics': { + inputTokenCount: 1, + outputTokenCount: 2, + invocationLatency: 3, + firstByteLatency: 4, + }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + content: 'foo', + tool_calls: [], + type: 'chatCompletionChunk', + }, + { + content: 'delta1', + tool_calls: [], + type: 'chatCompletionChunk', + }, + { + content: 'delta2', + tool_calls: [], + type: 'chatCompletionChunk', + }, + { + content: 'stop_seq', + tool_calls: [], + type: 'chatCompletionChunk', + }, + { + tokens: { + completion: 2, + prompt: 1, + total: 3, + }, + type: 'chatCompletionTokenCount', + }, + ]); + }); + + it('emits the correct values for a tool_use response scenario', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'message_start', + message: 'foo', + }, + { + type: 'content_block_start', + index: 0, + content_block: { type: 'tool_use', id: 'id', name: 'name', input: '{}' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'input_json_delta', partial_json: '{ "param' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'input_json_delta', partial_json: '": 12 }' }, + }, + { + type: 'content_block_stop', + index: 0, + }, + { + type: 'message_delta', + delta: { stop_reason: 'tool_use', stop_sequence: null, usage: { output_tokens: 42 } }, + }, + { + type: 'message_stop', + 'amazon-bedrock-invocationMetrics': { + inputTokenCount: 1, + outputTokenCount: 2, + invocationLatency: 3, + firstByteLatency: 4, + }, + }, + ]; + + expect(await lastValueFrom(of(...chunks).pipe(processCompletionChunks(), toArray()))).toEqual([ + { + content: '', + tool_calls: [ + { + function: { + arguments: '', + name: 'name', + }, + index: 0, + toolCallId: 'id', + }, + ], + type: 'chatCompletionChunk', + }, + { + content: '', + tool_calls: [ + { + function: { + arguments: '{ "param', + name: '', + }, + index: 0, + toolCallId: '', + }, + ], + type: 'chatCompletionChunk', + }, + { + content: '', + tool_calls: [ + { + function: { + arguments: '": 12 }', + name: '', + }, + index: 0, + toolCallId: '', + }, + ], + type: 'chatCompletionChunk', + }, + { + tokens: { + completion: 2, + prompt: 1, + total: 3, + }, + type: 'chatCompletionTokenCount', + }, + ]); + }); +}); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.ts new file mode 100644 index 0000000000000..5513cc9028ac9 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/process_completion_chunks.ts @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Observable, Subscriber } from 'rxjs'; +import { + ChatCompletionChunkEvent, + ChatCompletionTokenCountEvent, + ChatCompletionChunkToolCall, + ChatCompletionEventType, +} from '../../../../common/chat_complete'; +import type { CompletionChunk, MessageStopChunk } from './types'; + +export function processCompletionChunks() { + return (source: Observable) => + new Observable((subscriber) => { + function handleNext(chunkBody: CompletionChunk) { + if (isTokenCountCompletionChunk(chunkBody)) { + return emitTokenCountEvent(subscriber, chunkBody); + } + + let completionChunk = ''; + let toolCallChunk: ChatCompletionChunkToolCall | undefined; + + switch (chunkBody.type) { + case 'content_block_start': + if (chunkBody.content_block.type === 'text') { + completionChunk = chunkBody.content_block.text || ''; + } else if (chunkBody.content_block.type === 'tool_use') { + toolCallChunk = { + index: chunkBody.index, + toolCallId: chunkBody.content_block.id, + function: { + name: chunkBody.content_block.name, + // the API returns '{}' here, which can't be merged with the deltas... + arguments: '', + }, + }; + } + break; + + case 'content_block_delta': + if (chunkBody.delta.type === 'text_delta') { + completionChunk = chunkBody.delta.text || ''; + } else if (chunkBody.delta.type === 'input_json_delta') { + toolCallChunk = { + index: chunkBody.index, + toolCallId: '', + function: { + name: '', + arguments: chunkBody.delta.partial_json, + }, + }; + } + break; + + case 'message_delta': + completionChunk = chunkBody.delta.stop_sequence || ''; + break; + + default: + break; + } + + if (completionChunk || toolCallChunk) { + subscriber.next({ + type: ChatCompletionEventType.ChatCompletionChunk, + content: completionChunk, + tool_calls: toolCallChunk ? [toolCallChunk] : [], + }); + } + } + + source.subscribe({ + next: (value) => { + try { + handleNext(value); + } catch (error) { + subscriber.error(error); + } + }, + error: (err) => { + subscriber.error(err); + }, + complete: () => { + subscriber.complete(); + }, + }); + }); +} + +function isTokenCountCompletionChunk(value: CompletionChunk): value is MessageStopChunk { + return value.type === 'message_stop' && 'amazon-bedrock-invocationMetrics' in value; +} + +function emitTokenCountEvent( + subscriber: Subscriber, + chunk: MessageStopChunk +) { + const { inputTokenCount, outputTokenCount } = chunk['amazon-bedrock-invocationMetrics']; + + subscriber.next({ + type: ChatCompletionEventType.ChatCompletionTokenCount, + tokens: { + completion: outputTokenCount, + prompt: inputTokenCount, + total: inputTokenCount + outputTokenCount, + }, + }); +} diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/prompts.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/prompts.ts new file mode 100644 index 0000000000000..ed8387bf75252 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/prompts.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const noToolUsageDirective = ` +Please answer with text. You should NOT call or use a tool, even if tools might be available and even if +the user explicitly asks for it. DO NOT UNDER ANY CIRCUMSTANCES call a tool. Instead, ALWAYS reply with text. +`; + +export const addNoToolUsageDirective = (systemMessage: string | undefined): string => { + return systemMessage ? `${systemMessage}\n\n${noToolUsageDirective}` : noToolUsageDirective; +}; diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.test.ts new file mode 100644 index 0000000000000..bed6458a94dc7 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.test.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Readable } from 'stream'; +import { Observable, toArray, firstValueFrom, map, filter } from 'rxjs'; +import { + BedrockChunkMember, + BedrockStreamMember, + serdeEventstreamIntoObservable, +} from './serde_eventstream_into_observable'; +import { EventStreamMarshaller } from '@smithy/eventstream-serde-node'; +import { fromUtf8, toUtf8 } from '@smithy/util-utf8'; +import type { CompletionChunk } from './types'; +import { parseSerdeChunkMessage, serializeSerdeChunkMessage } from './serde_utils'; + +describe('serdeEventstreamIntoObservable', () => { + const marshaller = new EventStreamMarshaller({ + utf8Encoder: toUtf8, + utf8Decoder: fromUtf8, + }); + + const getSerdeEventStream = (chunks: CompletionChunk[]) => { + const input = Readable.from(chunks); + return marshaller.serialize(input, serializeSerdeChunkMessage); + }; + + const getChunks = async (serde$: Observable) => { + return await firstValueFrom( + serde$.pipe( + filter((value): value is BedrockChunkMember => { + return 'chunk' in value && value.chunk?.headers?.[':event-type']?.value === 'chunk'; + }), + map((message) => { + return parseSerdeChunkMessage(message.chunk); + }), + toArray() + ) + ); + }; + + it('converts a single chunk', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'Hello' }, + }, + ]; + + const inputStream = getSerdeEventStream(chunks); + const serde$ = serdeEventstreamIntoObservable(inputStream); + + const result = await getChunks(serde$); + + expect(result).toEqual(chunks); + }); + + it('converts multiple chunks', async () => { + const chunks: CompletionChunk[] = [ + { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: 'start' }, + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'Hello' }, + }, + { + type: 'content_block_stop', + index: 0, + }, + ]; + + const inputStream = getSerdeEventStream(chunks); + const serde$ = serdeEventstreamIntoObservable(inputStream); + + const result = await getChunks(serde$); + + expect(result).toEqual(chunks); + }); +}); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.ts new file mode 100644 index 0000000000000..24a245ab2efcc --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_eventstream_into_observable.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EventStreamMarshaller } from '@smithy/eventstream-serde-node'; +import { fromUtf8, toUtf8 } from '@smithy/util-utf8'; +import { identity } from 'lodash'; +import { Observable } from 'rxjs'; +import { Readable } from 'stream'; +import { Message } from '@smithy/types'; +import { createInferenceInternalError } from '../../../../common/errors'; + +interface ModelStreamErrorException { + name: 'ModelStreamErrorException'; + originalStatusCode?: number; + originalMessage?: string; +} + +export interface BedrockChunkMember { + chunk: Message; +} + +export interface ModelStreamErrorExceptionMember { + modelStreamErrorException: ModelStreamErrorException; +} + +export type BedrockStreamMember = BedrockChunkMember | ModelStreamErrorExceptionMember; + +// AWS uses SerDe to send over serialized data, so we use their +// @smithy library to parse the stream data + +export function serdeEventstreamIntoObservable( + readable: Readable +): Observable { + return new Observable((subscriber) => { + const marshaller = new EventStreamMarshaller({ + utf8Encoder: toUtf8, + utf8Decoder: fromUtf8, + }); + + async function processStream() { + for await (const chunk of marshaller.deserialize(readable, identity)) { + if (chunk) { + subscriber.next(chunk); + } + } + } + + processStream().then( + () => { + subscriber.complete(); + }, + (error) => { + if (!(error instanceof Error)) { + try { + const exceptionType = error.headers[':exception-type'].value; + const body = toUtf8(error.body); + let message = `Encountered error in Bedrock stream of type ${exceptionType}`; + try { + message += '\n' + JSON.parse(body).message; + } catch (parseError) { + // trap + } + error = createInferenceInternalError(message); + } catch (decodeError) { + error = createInferenceInternalError(decodeError.message); + } + } + subscriber.error(error); + } + ); + }); +} diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.test.ts new file mode 100644 index 0000000000000..c763fd8c9daf3 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.test.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CompletionChunk } from './types'; +import { serializeSerdeChunkMessage, parseSerdeChunkMessage } from './serde_utils'; + +describe('parseSerdeChunkMessage', () => { + it('parses a serde chunk message', () => { + const chunk: CompletionChunk = { + type: 'content_block_stop', + index: 0, + }; + + expect(parseSerdeChunkMessage(serializeSerdeChunkMessage(chunk))).toEqual(chunk); + }); +}); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.ts new file mode 100644 index 0000000000000..d7050b7744940 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/serde_utils.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { toUtf8, fromUtf8 } from '@smithy/util-utf8'; +import type { Message } from '@smithy/types'; +import type { CompletionChunk } from './types'; + +/** + * Extract the completion chunk from a chunk message + */ +export function parseSerdeChunkMessage(chunk: Message): CompletionChunk { + return JSON.parse(Buffer.from(JSON.parse(toUtf8(chunk.body)).bytes, 'base64').toString('utf-8')); +} + +/** + * Reverse `parseSerdeChunkMessage` + */ +export const serializeSerdeChunkMessage = (input: CompletionChunk): Message => { + const b64 = Buffer.from(JSON.stringify(input), 'utf-8').toString('base64'); + const body = fromUtf8(JSON.stringify({ bytes: b64 })); + return { + headers: { + ':event-type': { type: 'string', value: 'chunk' }, + ':content-type': { type: 'string', value: 'application/json' }, + ':message-type': { type: 'string', value: 'event' }, + }, + body, + }; +}; diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/types.ts b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/types.ts new file mode 100644 index 0000000000000..f0937a8d8ec18 --- /dev/null +++ b/x-pack/plugins/inference/server/chat_complete/adapters/bedrock/types.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * BedRock message as expected by the bedrock connector + */ +export interface BedRockMessage { + role: 'user' | 'assistant'; + content?: string; + rawContent?: BedRockMessagePart[]; +} + +/** + * Bedrock message parts + */ +export type BedRockMessagePart = + | { type: 'text'; text: string } + | { + type: 'tool_use'; + id: string; + name: string; + input: Record; + } + | { type: 'tool_result'; tool_use_id: string; content: string }; + +export type BedrockToolChoice = { type: 'auto' } | { type: 'any' } | { type: 'tool'; name: string }; + +interface CompletionChunkBase { + type: string; +} + +export interface MessageStartChunk extends CompletionChunkBase { + type: 'message_start'; + message: unknown; +} + +export interface ContentBlockStartChunk extends CompletionChunkBase { + type: 'content_block_start'; + index: number; + content_block: + | { + type: 'text'; + text: string; + } + | { type: 'tool_use'; id: string; name: string; input: string }; +} + +export interface ContentBlockDeltaChunk extends CompletionChunkBase { + type: 'content_block_delta'; + index: number; + delta: + | { + type: 'text_delta'; + text: string; + } + | { + type: 'input_json_delta'; + partial_json: string; + }; +} + +export interface ContentBlockStopChunk extends CompletionChunkBase { + type: 'content_block_stop'; + index: number; +} + +export interface MessageDeltaChunk extends CompletionChunkBase { + type: 'message_delta'; + delta: { + stop_reason: string; + stop_sequence: null | string; + usage: { + output_tokens: number; + }; + }; +} + +export interface MessageStopChunk extends CompletionChunkBase { + type: 'message_stop'; + 'amazon-bedrock-invocationMetrics': { + inputTokenCount: number; + outputTokenCount: number; + invocationLatency: number; + firstByteLatency: number; + }; +} + +export type CompletionChunk = + | MessageStartChunk + | ContentBlockStartChunk + | ContentBlockDeltaChunk + | ContentBlockStopChunk + | MessageDeltaChunk + | MessageStopChunk; diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.test.ts b/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.test.ts index 9e0b0da6d5894..558e0cd06ef91 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.test.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.test.ts @@ -9,6 +9,7 @@ import { InferenceConnectorType } from '../../../common/connectors'; import { getInferenceAdapter } from './get_inference_adapter'; import { openAIAdapter } from './openai'; import { geminiAdapter } from './gemini'; +import { bedrockClaudeAdapter } from './bedrock'; describe('getInferenceAdapter', () => { it('returns the openAI adapter for OpenAI type', () => { @@ -19,7 +20,7 @@ describe('getInferenceAdapter', () => { expect(getInferenceAdapter(InferenceConnectorType.Gemini)).toBe(geminiAdapter); }); - it('returns undefined for Bedrock type', () => { - expect(getInferenceAdapter(InferenceConnectorType.Bedrock)).toBe(undefined); + it('returns the bedrock adapter for Bedrock type', () => { + expect(getInferenceAdapter(InferenceConnectorType.Bedrock)).toBe(bedrockClaudeAdapter); }); }); diff --git a/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.ts b/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.ts index 0538d828a473a..f34b0c27a339f 100644 --- a/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.ts +++ b/x-pack/plugins/inference/server/chat_complete/adapters/get_inference_adapter.ts @@ -9,6 +9,7 @@ import { InferenceConnectorType } from '../../../common/connectors'; import type { InferenceConnectorAdapter } from '../types'; import { openAIAdapter } from './openai'; import { geminiAdapter } from './gemini'; +import { bedrockClaudeAdapter } from './bedrock'; export const getInferenceAdapter = ( connectorType: InferenceConnectorType @@ -21,8 +22,7 @@ export const getInferenceAdapter = ( return geminiAdapter; case InferenceConnectorType.Bedrock: - // not implemented yet - break; + return bedrockClaudeAdapter; } return undefined; diff --git a/x-pack/plugins/inference/tsconfig.json b/x-pack/plugins/inference/tsconfig.json index 16d7ca041582c..593556c8f39c8 100644 --- a/x-pack/plugins/inference/tsconfig.json +++ b/x-pack/plugins/inference/tsconfig.json @@ -22,6 +22,7 @@ "@kbn/logging", "@kbn/core-http-server", "@kbn/actions-plugin", - "@kbn/config-schema" + "@kbn/config-schema", + "@kbn/stack-connectors-plugin" ] } diff --git a/x-pack/plugins/stack_connectors/common/bedrock/schema.ts b/x-pack/plugins/stack_connectors/common/bedrock/schema.ts index 093a5f9b11518..03f4f5cc01735 100644 --- a/x-pack/plugins/stack_connectors/common/bedrock/schema.ts +++ b/x-pack/plugins/stack_connectors/common/bedrock/schema.ts @@ -28,13 +28,30 @@ export const RunActionParamsSchema = schema.object({ raw: schema.maybe(schema.boolean()), }); +export const BedrockMessageSchema = schema.object( + { + role: schema.string(), + content: schema.maybe(schema.string()), + rawContent: schema.maybe(schema.arrayOf(schema.any())), + }, + { + validate: (value) => { + if (value.content === undefined && value.rawContent === undefined) { + return 'Must specify either content or rawContent'; + } else if (value.content !== undefined && value.rawContent !== undefined) { + return 'content and rawContent can not be used at the same time'; + } + }, + } +); + +export const BedrockToolChoiceSchema = schema.object({ + type: schema.oneOf([schema.literal('auto'), schema.literal('any'), schema.literal('tool')]), + name: schema.maybe(schema.string()), +}); + export const InvokeAIActionParamsSchema = schema.object({ - messages: schema.arrayOf( - schema.object({ - role: schema.string(), - content: schema.string(), - }) - ), + messages: schema.arrayOf(BedrockMessageSchema), model: schema.maybe(schema.string()), temperature: schema.maybe(schema.number()), stopSequences: schema.maybe(schema.arrayOf(schema.string())), @@ -53,6 +70,7 @@ export const InvokeAIActionParamsSchema = schema.object({ }) ) ), + toolChoice: schema.maybe(BedrockToolChoiceSchema), }); export const InvokeAIActionResponseSchema = schema.object({ @@ -60,12 +78,7 @@ export const InvokeAIActionResponseSchema = schema.object({ }); export const InvokeAIRawActionParamsSchema = schema.object({ - messages: schema.arrayOf( - schema.object({ - role: schema.string(), - content: schema.any(), - }) - ), + messages: schema.arrayOf(BedrockMessageSchema), model: schema.maybe(schema.string()), temperature: schema.maybe(schema.number()), stopSequences: schema.maybe(schema.arrayOf(schema.string())), @@ -84,6 +97,7 @@ export const InvokeAIRawActionParamsSchema = schema.object({ }) ) ), + toolChoice: schema.maybe(BedrockToolChoiceSchema), }); export const InvokeAIRawActionResponseSchema = schema.object({}, { unknowns: 'allow' }); diff --git a/x-pack/plugins/stack_connectors/common/bedrock/types.ts b/x-pack/plugins/stack_connectors/common/bedrock/types.ts index b144f78b91edd..3b02f40d2de62 100644 --- a/x-pack/plugins/stack_connectors/common/bedrock/types.ts +++ b/x-pack/plugins/stack_connectors/common/bedrock/types.ts @@ -19,6 +19,8 @@ import { InvokeAIRawActionResponseSchema, StreamingResponseSchema, RunApiLatestResponseSchema, + BedrockMessageSchema, + BedrockToolChoiceSchema, } from './schema'; export type Config = TypeOf; @@ -33,3 +35,5 @@ export type RunActionResponse = TypeOf; export type StreamingResponse = TypeOf; export type DashboardActionParams = TypeOf; export type DashboardActionResponse = TypeOf; +export type BedRockMessage = TypeOf; +export type BedrockToolChoice = TypeOf; diff --git a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts index b5ec114a9c456..c2c773bdeaf87 100644 --- a/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts +++ b/x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts @@ -22,7 +22,7 @@ import { RunActionResponseSchema, RunApiLatestResponseSchema, } from '../../../common/bedrock/schema'; -import { +import type { Config, Secrets, RunActionParams, @@ -32,6 +32,8 @@ import { InvokeAIRawActionParams, InvokeAIRawActionResponse, RunApiLatestResponse, + BedRockMessage, + BedrockToolChoice, } from '../../../common/bedrock/types'; import { SUB_ACTION, @@ -309,13 +311,14 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B signal, timeout, tools, + toolChoice, }: InvokeAIActionParams | InvokeAIRawActionParams, connectorUsageCollector: ConnectorUsageCollector ): Promise { const res = (await this.streamApi( { body: JSON.stringify( - formatBedrockBody({ messages, stopSequences, system, temperature, tools }) + formatBedrockBody({ messages, stopSequences, system, temperature, tools, toolChoice }) ), model, signal, @@ -344,13 +347,23 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B maxTokens, signal, timeout, + tools, + toolChoice, }: InvokeAIActionParams, connectorUsageCollector: ConnectorUsageCollector ): Promise { const res = (await this.runApi( { body: JSON.stringify( - formatBedrockBody({ messages, stopSequences, system, temperature, maxTokens }) + formatBedrockBody({ + messages, + stopSequences, + system, + temperature, + maxTokens, + tools, + toolChoice, + }) ), model, signal, @@ -372,6 +385,7 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B signal, timeout, tools, + toolChoice, anthropicVersion, }: InvokeAIRawActionParams, connectorUsageCollector: ConnectorUsageCollector @@ -385,6 +399,7 @@ The Kibana Connector in use may need to be reconfigured with an updated Amazon B temperature, max_tokens: maxTokens, tools, + tool_choice: toolChoice, anthropic_version: anthropicVersion, }), model, @@ -405,14 +420,16 @@ const formatBedrockBody = ({ system, maxTokens = DEFAULT_TOKEN_LIMIT, tools, + toolChoice, }: { - messages: Array<{ role: string; content?: string }>; + messages: BedRockMessage[]; stopSequences?: string[]; temperature?: number; maxTokens?: number; // optional system message to be sent to the API system?: string; tools?: Array<{ name: string; description: string }>; + toolChoice?: BedrockToolChoice; }) => ({ anthropic_version: 'bedrock-2023-05-31', ...ensureMessageFormat(messages, system), @@ -420,8 +437,14 @@ const formatBedrockBody = ({ stop_sequences: stopSequences, temperature, tools, + tool_choice: toolChoice, }); +interface FormattedBedRockMessage { + role: string; + content: string | BedRockMessage['rawContent']; +} + /** * Ensures that the messages are in the correct format for the Bedrock API * If 2 user or 2 assistant messages are sent in a row, Bedrock throws an error @@ -429,19 +452,32 @@ const formatBedrockBody = ({ * @param messages */ const ensureMessageFormat = ( - messages: Array<{ role: string; content?: string }>, + messages: BedRockMessage[], systemPrompt?: string -): { messages: Array<{ role: string; content?: string }>; system?: string } => { +): { + messages: FormattedBedRockMessage[]; + system?: string; +} => { let system = systemPrompt ? systemPrompt : ''; - const newMessages = messages.reduce((acc: Array<{ role: string; content?: string }>, m) => { - const lastMessage = acc[acc.length - 1]; + const newMessages = messages.reduce((acc, m) => { if (m.role === 'system') { system = `${system.length ? `${system}\n` : ''}${m.content}`; return acc; } - if (lastMessage && lastMessage.role === m.role) { + const messageRole = () => (['assistant', 'ai'].includes(m.role) ? 'assistant' : 'user'); + + if (m.rawContent) { + acc.push({ + role: messageRole(), + content: m.rawContent, + }); + return acc; + } + + const lastMessage = acc[acc.length - 1]; + if (lastMessage && lastMessage.role === m.role && typeof lastMessage.content === 'string') { // Bedrock only accepts assistant and user roles. // If 2 user or 2 assistant messages are sent in a row, combine the messages into a single message return [ @@ -451,11 +487,9 @@ const ensureMessageFormat = ( } // force role outside of system to ensure it is either assistant or user - return [ - ...acc, - { content: m.content, role: ['assistant', 'ai'].includes(m.role) ? 'assistant' : 'user' }, - ]; + return [...acc, { content: m.content, role: messageRole() }]; }, []); + return system.length ? { system, messages: newMessages } : { messages: newMessages }; }; From 74a551e81427160d3292e506b858e8f95102e93c Mon Sep 17 00:00:00 2001 From: Sid Date: Fri, 30 Aug 2024 13:08:29 +0200 Subject: [PATCH 12/12] Migrate Enterprise search plugin authc dependencies from security plugin to core security service (#189713) ## Summary Part of https://github.com/elastic/kibana/issues/186574 Closes https://github.com/elastic/kibana/issues/189714 Background: This PR is an example of a plugin migrating away from depending on the Security plugin, which is a high-priority effort for the last release before 9.0. The Enterprise search plugin uses authc.apiKeys.create from the security plugin's start contract on the server side. For more context, the PR which exposes the API keys service from core is here: https://github.com/elastic/kibana/pull/186910 This PR migrates the usage from the security plugin start contract to the core security service. --------- Co-authored-by: Elastic Machine Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../server/lib/indices/create_api_key.test.ts | 12 ++++-------- .../server/lib/indices/create_api_key.ts | 9 +++------ x-pack/plugins/enterprise_search/server/plugin.ts | 7 ++----- .../server/routes/enterprise_search/api_keys.ts | 13 +++++-------- x-pack/plugins/enterprise_search/tsconfig.json | 4 +++- 5 files changed, 17 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.test.ts b/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.test.ts index d89126f549a1b..43bea033a2e3d 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.test.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.test.ts @@ -5,14 +5,12 @@ * 2.0. */ -import { KibanaRequest } from '@kbn/core/server'; -import { securityMock } from '@kbn/security-plugin/server/mocks'; +import { securityServiceMock } from '@kbn/core-security-server-mocks'; import { createApiKey } from './create_api_key'; describe('createApiKey lib function', () => { - const security = securityMock.createStart(); - const request = {} as KibanaRequest; + const security = securityServiceMock.createRequestHandlerContext(); const indexName = 'my-index'; const keyName = '{indexName}-key'; @@ -31,11 +29,9 @@ describe('createApiKey lib function', () => { }); it('should create an api key via the security plugin', async () => { - await expect(createApiKey(request, security, indexName, keyName)).resolves.toEqual( - createResponse - ); + await expect(createApiKey(security, indexName, keyName)).resolves.toEqual(createResponse); - expect(security.authc.apiKeys.create).toHaveBeenCalledWith(request, { + expect(security.authc.apiKeys.create).toHaveBeenCalledWith({ name: keyName, role_descriptors: { [`${indexName}-key-role`]: { diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.ts b/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.ts index 0c1a62c4d30db..b5d7fb7cf22e9 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/create_api_key.ts @@ -5,19 +5,16 @@ * 2.0. */ -import { KibanaRequest } from '@kbn/core/server'; - -import { SecurityPluginStart } from '@kbn/security-plugin/server'; +import type { SecurityRequestHandlerContext } from '@kbn/core-security-server'; import { toAlphanumeric } from '../../../common/utils/to_alphanumeric'; export const createApiKey = async ( - request: KibanaRequest, - security: SecurityPluginStart, + security: SecurityRequestHandlerContext, indexName: string, keyName: string ) => { - return await security.authc.apiKeys.create(request, { + return await security.authc.apiKeys.create({ name: keyName, role_descriptors: { [`${toAlphanumeric(indexName)}-key-role`]: { diff --git a/x-pack/plugins/enterprise_search/server/plugin.ts b/x-pack/plugins/enterprise_search/server/plugin.ts index a03429729bf2f..d021b1b3cd634 100644 --- a/x-pack/plugins/enterprise_search/server/plugin.ts +++ b/x-pack/plugins/enterprise_search/server/plugin.ts @@ -25,7 +25,7 @@ import type { LicensingPluginStart } from '@kbn/licensing-plugin/public'; import { LogsSharedPluginSetup } from '@kbn/logs-shared-plugin/server'; import type { MlPluginSetup } from '@kbn/ml-plugin/server'; import { SearchConnectorsPluginSetup } from '@kbn/search-connectors-plugin/server'; -import { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/server'; +import { SecurityPluginSetup } from '@kbn/security-plugin/server'; import { SpacesPluginStart } from '@kbn/spaces-plugin/server'; import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; @@ -104,7 +104,6 @@ interface PluginsSetup { export interface PluginsStart { data: DataPluginStart; - security: SecurityPluginStart; spaces?: SpacesPluginStart; } @@ -284,9 +283,7 @@ export class EnterpriseSearchPlugin implements Plugin { registerAnalyticsRoutes({ ...dependencies, data, savedObjects: coreStart.savedObjects }); }); - void getStartServices().then(([, { security: securityStart }]) => { - registerApiKeysRoutes(dependencies, securityStart); - }); + registerApiKeysRoutes(dependencies); /** * Bootstrap the routes, saved objects, and collector for telemetry diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/api_keys.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/api_keys.ts index 8b94de5e6955c..71e00bed27910 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/api_keys.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/api_keys.ts @@ -7,16 +7,11 @@ import { schema } from '@kbn/config-schema'; -import { SecurityPluginStart } from '@kbn/security-plugin/server'; - import { createApiKey } from '../../lib/indices/create_api_key'; import { RouteDependencies } from '../../plugin'; import { elasticsearchErrorHandler } from '../../utils/elasticsearch_error_handler'; -export function registerApiKeysRoutes( - { log, router }: RouteDependencies, - security: SecurityPluginStart -) { +export function registerApiKeysRoutes({ log, router }: RouteDependencies) { router.post( { path: '/internal/enterprise_search/{indexName}/api_keys', @@ -32,8 +27,9 @@ export function registerApiKeysRoutes( elasticsearchErrorHandler(log, async (context, request, response) => { const indexName = decodeURIComponent(request.params.indexName); const { keyName } = request.body; + const { security: coreSecurity } = await context.core; - const createResponse = await createApiKey(request, security, indexName, keyName); + const createResponse = await createApiKey(coreSecurity, indexName, keyName); if (!createResponse) { throw new Error('Unable to create API Key'); @@ -118,7 +114,8 @@ export function registerApiKeysRoutes( }, }, async (context, request, response) => { - const result = await security.authc.apiKeys.create(request, request.body); + const { security: coreSecurity } = await context.core; + const result = await coreSecurity.authc.apiKeys.create(request.body); if (result) { const apiKey = { ...result, beats_logstash_format: `${result.id}:${result.api_key}` }; return response.ok({ body: apiKey }); diff --git a/x-pack/plugins/enterprise_search/tsconfig.json b/x-pack/plugins/enterprise_search/tsconfig.json index 86cf6c3968005..58b1526e14baf 100644 --- a/x-pack/plugins/enterprise_search/tsconfig.json +++ b/x-pack/plugins/enterprise_search/tsconfig.json @@ -81,6 +81,8 @@ "@kbn/core-chrome-browser", "@kbn/navigation-plugin", "@kbn/search-homepage", - "@kbn/security-plugin-types-common" + "@kbn/security-plugin-types-common", + "@kbn/core-security-server", + "@kbn/core-security-server-mocks" ] }