From c5447a80a93d79ebb0927a7419add98623481d7e Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Wed, 24 May 2023 18:58:41 +0200 Subject: [PATCH 01/15] Initial index bootstraping --- .../server/lib/risk_engine/configurations.ts | 82 +++++++++++ .../risk_engine/risk_engine_data_client.ts | 137 ++++++++++++++++++ .../security_solution/server/plugin.ts | 13 ++ .../server/request_context_factory.ts | 13 +- .../plugins/security_solution/server/types.ts | 2 + 5 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts create mode 100644 x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts new file mode 100644 index 0000000000000..7e910631b96c4 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const riskFieldMap = { + '@timestamp': { + type: 'date', + array: false, + required: false, + }, + identifierField: { + type: 'keyword', + array: false, + required: false, + }, + identifierValue: { + type: 'keyword', + array: false, + required: false, + }, + level: { + type: 'keyword', + array: false, + required: false, + }, + totalScore: { + type: 'float', + array: false, + required: false, + }, + totalScoreNormalized: { + type: 'float', + array: false, + required: false, + }, + alertsScore: { + type: 'float', + array: false, + required: false, + }, + otherScore: { + type: 'float', + array: false, + required: false, + }, + riskiestInputs: { + type: 'nested', + required: false, + }, + 'riskiestInputs.id': { + type: 'keyword', + array: false, + required: false, + }, + 'riskiestInputs.index': { + type: 'keyword', + array: false, + required: false, + }, + 'riskiestInputs.riskScore': { + type: 'float', + array: false, + required: false, + }, +} as const; + +export const ilmPolicyName = '.risk-engine-ilm-policy'; +export const mappingComponentName = 'risk_score_mappings'; +export const totalFieldsLimit = 1000; + +const riskScoreBaseIndexName = '.risk-score'; + +export const getIndexPattern = (namespace: string) => ({ + template: `${riskScoreBaseIndexName}.${riskScoreBaseIndexName}-${namespace}-index-template`, + pattern: `.internal.${riskScoreBaseIndexName}${riskScoreBaseIndexName}-${namespace}-*`, + basePattern: `.${riskScoreBaseIndexName}${riskScoreBaseIndexName}-*`, + name: `.internal${riskScoreBaseIndexName}${riskScoreBaseIndexName}-${namespace}-000001`, + alias: `${riskScoreBaseIndexName}${riskScoreBaseIndexName}-${namespace}`, +}); diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts new file mode 100644 index 0000000000000..2383d530523c1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { Metadata } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { + createConcreteWriteIndex, + createOrUpdateComponentTemplate, + createOrUpdateIlmPolicy, + createOrUpdateIndexTemplate, +} from '@kbn/alerting-plugin/server'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; +import { DEFAULT_NAMESPACE_STRING } from '@kbn/core-saved-objects-utils-server'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { + riskFieldMap, + getIndexPattern, + totalFieldsLimit, + mappingComponentName, + ilmPolicyName, +} from './configurations'; + +interface InitializeRiskEngineResourcesOpts { + namespace?: string; +} + +interface RiskEngineDataClientOpts { + logger: Logger; + kibanaVersion: string; + elasticsearchClientPromise: Promise; +} + +export class RiskEngineDataClient { + constructor(private readonly options: RiskEngineDataClientOpts) {} + + public async getWriter({ namespace }: { namespace: string }) { + this.initializeResources({ namespace }); + return { + bulk: async () => {}, + }; + } + + public async initializeResources({ + namespace = DEFAULT_NAMESPACE_STRING, + }: InitializeRiskEngineResourcesOpts) { + try { + const esClient = await this.options.elasticsearchClientPromise; + + const indexPatterns = getIndexPattern(namespace); + + const indexMetadata: Metadata = { + kibana: { + version: this.options.kibanaVersion, + }, + managed: true, + namespace, + }; + + await createOrUpdateIlmPolicy({ + logger: this.options.logger, + esClient, + name: ilmPolicyName, + policy: { + _meta: { + managed: true, + }, + phases: { + hot: { + actions: { + rollover: { + max_age: '30d', + max_primary_shard_size: '50gb', + }, + }, + }, + }, + }, + }); + + await createOrUpdateComponentTemplate({ + logger: this.options.logger, + esClient, + template: { + name: mappingComponentName, + _meta: { + managed: true, + }, + template: { + settings: {}, + mappings: mappingFromFieldMap(riskFieldMap, 'strict'), + }, + } as ClusterPutComponentTemplateRequest, + totalFieldsLimit, + }); + + await createOrUpdateIndexTemplate({ + logger: this.options.logger, + esClient, + template: { + name: indexPatterns.template, + body: { + index_patterns: [indexPatterns.pattern], + composed_of: [mappingComponentName], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: ilmPolicyName, + rollover_alias: indexPatterns.alias, + }, + 'index.mapping.total_fields.limit': totalFieldsLimit, + }, + mappings: { + dynamic: false, + _meta: indexMetadata, + }, + }, + _meta: indexMetadata, + }, + }, + }); + + await createConcreteWriteIndex({ + logger: this.options.logger, + esClient, + totalFieldsLimit, + indexPatterns, + }); + } catch (error) { + this.options.logger.error(`Error initializing risk engine resources: ${error}`); + } + } +} diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index f4869b211b1ea..a05021833da04 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -106,6 +106,7 @@ import { setIsElasticCloudDeployment } from './lib/telemetry/helpers'; import { artifactService } from './lib/telemetry/artifact'; import { endpointFieldsProvider } from './search_strategy/endpoint_fields'; import { ENDPOINT_FIELDS_SEARCH_STRATEGY } from '../common/endpoint/constants'; +import { RiskEngineDataClient } from './lib/risk_engine/risk_engine_data_client'; export type { SetupPlugins, StartPlugins, PluginSetup, PluginStart } from './plugin_contract'; @@ -128,6 +129,7 @@ export class Plugin implements ISecuritySolutionPlugin { private artifactsCache: LRU; private telemetryUsageCounter?: UsageCounter; private endpointContext: EndpointAppContext; + private riskEngineDataClient: RiskEngineDataClient | undefined; constructor(context: PluginInitializerContext) { const serverConfig = createConfig(context); @@ -169,6 +171,16 @@ export class Plugin implements ISecuritySolutionPlugin { const ruleExecutionLogService = createRuleExecutionLogService(config, logger, core, plugins); ruleExecutionLogService.registerEventLogProvider(); + this.riskEngineDataClient = new RiskEngineDataClient({ + logger: this.logger, + kibanaVersion: this.pluginContext.env.packageInfo.version, + elasticsearchClientPromise: core + .getStartServices() + .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), + }); + + this.riskEngineDataClient.initializeResources({}); + const requestContextFactory = new RequestContextFactory({ config, logger, @@ -178,6 +190,7 @@ export class Plugin implements ISecuritySolutionPlugin { ruleExecutionLogService, kibanaVersion: pluginContext.env.packageInfo.version, kibanaBranch: pluginContext.env.packageInfo.branch, + riskEngineDataClient: this.riskEngineDataClient, }); const router = core.http.createRouter(); diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index 3bb125cf06914..2e8f3d7628cf5 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -25,6 +25,7 @@ import type { import type { Immutable } from '../common/endpoint/types'; import type { EndpointAuthz } from '../common/endpoint/types/authz'; import type { EndpointAppContextService } from './endpoint/endpoint_app_context_services'; +import type { RiskEngineDataClient } from './lib/risk_engine/risk_engine_data_client'; export interface IRequestContextFactory { create( @@ -42,6 +43,7 @@ interface ConstructorOptions { ruleExecutionLogService: IRuleExecutionLogService; kibanaVersion: string; kibanaBranch: string; + riskEngineDataClient: RiskEngineDataClient; } export class RequestContextFactory implements IRequestContextFactory { @@ -56,7 +58,14 @@ export class RequestContextFactory implements IRequestContextFactory { request: KibanaRequest ): Promise { const { options, appClientFactory } = this; - const { config, core, plugins, endpointAppContextService, ruleExecutionLogService } = options; + const { + config, + core, + plugins, + endpointAppContextService, + ruleExecutionLogService, + riskEngineDataClient, + } = options; const { lists, ruleRegistry, security } = plugins; const [, startPlugins] = await core.getStartServices(); @@ -115,6 +124,8 @@ export class RequestContextFactory implements IRequestContextFactory { }, getInternalFleetServices: memoize(() => endpointAppContextService.getInternalFleetServices()), + + getRiskEngineDataClient: () => riskEngineDataClient, }; } } diff --git a/x-pack/plugins/security_solution/server/types.ts b/x-pack/plugins/security_solution/server/types.ts index a29193250ea28..9d0e95e4f6a63 100644 --- a/x-pack/plugins/security_solution/server/types.ts +++ b/x-pack/plugins/security_solution/server/types.ts @@ -26,6 +26,7 @@ import type { IRuleExecutionLogForRoutes } from './lib/detection_engine/rule_mon import type { FrameworkRequest } from './lib/framework'; import type { EndpointAuthz } from '../common/endpoint/types/authz'; import type { EndpointInternalFleetServicesInterface } from './endpoint/services/fleet'; +import type { RiskEngineDataClient } from './lib/risk_engine/risk_engine_data_client'; export { AppClient }; @@ -41,6 +42,7 @@ export interface SecuritySolutionApiRequestHandlerContext { getRacClient: (req: KibanaRequest) => Promise; getExceptionListClient: () => ExceptionListClient | null; getInternalFleetServices: () => EndpointInternalFleetServicesInterface; + getRiskEngineDataClient: () => RiskEngineDataClient; } export type SecuritySolutionRequestHandlerContext = CustomRequestHandlerContext<{ From 6253ba964f102652b5a1c320a4367833ee725f2f Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Thu, 25 May 2023 13:20:10 +0200 Subject: [PATCH 02/15] Change to DS --- .../server/lib/risk_engine/configurations.ts | 16 +- .../risk_engine/risk_engine_data_client.ts | 6 +- .../server/lib/risk_engine/utils/create_ds.ts | 214 ++++++++++++++++++ .../server/lib/risk_engine/utils/retry_es.ts | 58 +++++ 4 files changed, 283 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_ds.ts create mode 100644 x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_es.ts diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts index 7e910631b96c4..68cd6820d70a5 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts @@ -67,16 +67,16 @@ export const riskFieldMap = { }, } as const; -export const ilmPolicyName = '.risk-engine-ilm-policy'; -export const mappingComponentName = 'risk_score_mappings'; +export const ilmPolicyName = '.risk-score-ilm-policy'; +export const mappingComponentName = 'risk-score-mappings'; export const totalFieldsLimit = 1000; -const riskScoreBaseIndexName = '.risk-score'; +const riskScoreBaseIndexName = 'risk-score'; export const getIndexPattern = (namespace: string) => ({ - template: `${riskScoreBaseIndexName}.${riskScoreBaseIndexName}-${namespace}-index-template`, - pattern: `.internal.${riskScoreBaseIndexName}${riskScoreBaseIndexName}-${namespace}-*`, - basePattern: `.${riskScoreBaseIndexName}${riskScoreBaseIndexName}-*`, - name: `.internal${riskScoreBaseIndexName}${riskScoreBaseIndexName}-${namespace}-000001`, - alias: `${riskScoreBaseIndexName}${riskScoreBaseIndexName}-${namespace}`, + template: `.${riskScoreBaseIndexName}.${riskScoreBaseIndexName}-${namespace}-index-template`, + alias: `${riskScoreBaseIndexName}.${riskScoreBaseIndexName}-${namespace}`, + pattern: '', + basePattern: '', + name: '', }); diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts index 2383d530523c1..f5a0f551bf401 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts @@ -7,7 +7,6 @@ import type { Metadata } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; import { - createConcreteWriteIndex, createOrUpdateComponentTemplate, createOrUpdateIlmPolicy, createOrUpdateIndexTemplate, @@ -22,6 +21,7 @@ import { mappingComponentName, ilmPolicyName, } from './configurations'; +import { createConcreteWriteIndex } from './utils/create_ds'; interface InitializeRiskEngineResourcesOpts { namespace?: string; @@ -102,7 +102,8 @@ export class RiskEngineDataClient { template: { name: indexPatterns.template, body: { - index_patterns: [indexPatterns.pattern], + data_stream: { hidden: true }, + index_patterns: [indexPatterns.alias], composed_of: [mappingComponentName], template: { settings: { @@ -110,7 +111,6 @@ export class RiskEngineDataClient { hidden: true, 'index.lifecycle': { name: ilmPolicyName, - rollover_alias: indexPatterns.alias, }, 'index.mapping.total_fields.limit': totalFieldsLimit, }, diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_ds.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_ds.ts new file mode 100644 index 0000000000000..a94e701cd9a07 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_ds.ts @@ -0,0 +1,214 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// This file is a copy of x-pack/plugins/alerting/server/alerts_service/lib/create_concrete_write_index.ts +// original function create index instead of datastream, and their have plan to use datastream in the future +// so we probably should remove this file and use the original when datastream will be supported + +import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { get } from 'lodash'; +import { retryTransientEsErrors } from './retry_es'; + +export interface IIndexPatternString { + template: string; + pattern: string; + alias: string; + name: string; + basePattern: string; + secondaryAlias?: string; +} + +interface ConcreteIndexInfo { + index: string; + alias: string; + isWriteIndex: boolean; +} + +interface UpdateIndexMappingsOpts { + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; + concreteIndices: ConcreteIndexInfo[]; +} + +interface UpdateIndexOpts { + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; + concreteIndexInfo: ConcreteIndexInfo; +} + +const updateTotalFieldLimitSetting = async ({ + logger, + esClient, + totalFieldsLimit, + concreteIndexInfo, +}: UpdateIndexOpts) => { + const { index, alias } = concreteIndexInfo; + try { + await retryTransientEsErrors( + () => + esClient.indices.putSettings({ + index, + body: { 'index.mapping.total_fields.limit': totalFieldsLimit }, + }), + { logger } + ); + return; + } catch (err) { + logger.error( + `Failed to PUT index.mapping.total_fields.limit settings for alias ${alias}: ${err.message}` + ); + throw err; + } +}; + +// This will update the mappings of backing indices but *not* the settings. This +// is due to the fact settings can be classed as dynamic and static, and static +// updates will fail on an index that isn't closed. New settings *will* be applied as part +// of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654 +const updateUnderlyingMapping = async ({ + logger, + esClient, + concreteIndexInfo, +}: UpdateIndexOpts) => { + const { index, alias } = concreteIndexInfo; + let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse; + try { + simulatedIndexMapping = await retryTransientEsErrors( + () => esClient.indices.simulateIndexTemplate({ name: index }), + { logger } + ); + } catch (err) { + logger.error( + `Ignored PUT mappings for alias ${alias}; error generating simulated mappings: ${err.message}` + ); + return; + } + + const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']); + + if (simulatedMapping == null) { + logger.error(`Ignored PUT mappings for alias ${alias}; simulated mappings were empty`); + return; + } + + try { + await retryTransientEsErrors( + () => esClient.indices.putMapping({ index, body: simulatedMapping }), + { logger } + ); + + return; + } catch (err) { + logger.error(`Failed to PUT mapping for alias ${alias}: ${err.message}`); + throw err; + } +}; +/** + * Updates the underlying mapping for any existing concrete indices + */ +const updateIndexMappings = async ({ + logger, + esClient, + totalFieldsLimit, + concreteIndices, +}: UpdateIndexMappingsOpts) => { + logger.debug(`Updating underlying mappings for ${concreteIndices.length} indices.`); + + // Update total field limit setting of found indices + // Other index setting changes are not updated at this time + await Promise.all( + concreteIndices.map((index) => + updateTotalFieldLimitSetting({ logger, esClient, totalFieldsLimit, concreteIndexInfo: index }) + ) + ); + + // Update mappings of the found indices. + await Promise.all( + concreteIndices.map((index) => + updateUnderlyingMapping({ logger, esClient, totalFieldsLimit, concreteIndexInfo: index }) + ) + ); +}; + +interface CreateConcreteWriteIndexOpts { + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; + indexPatterns: IIndexPatternString; +} +/** + * Installs index template that uses installed component template + * Prior to installation, simulates the installation to check for possible + * conflicts. Simulate should return an empty mapping if a template + * conflicts with an already installed template. + */ +export const createConcreteWriteIndex = async ({ + logger, + esClient, + indexPatterns, + totalFieldsLimit, +}: CreateConcreteWriteIndexOpts) => { + logger.info(`Creating data stream - ${indexPatterns.alias}`); + + // check if a concrete write index already exists + let concreteIndices: ConcreteIndexInfo[] = []; + try { + // Specify both the index pattern for the backing indices and their aliases + // The alias prevents the request from finding other namespaces that could match the -* pattern + const response = await retryTransientEsErrors( + () => esClient.indices.getDataStream({ name: indexPatterns.alias, expand_wildcards: 'all' }), + { logger } + ); + + concreteIndices = response.data_streams.map((dataStream) => ({ + index: dataStream.name, + alias: dataStream.name, + isWriteIndex: true, + })); + + logger.debug( + `Found ${concreteIndices.length} concrete indices for ${ + indexPatterns.alias + } - ${JSON.stringify(concreteIndices)}` + ); + } catch (error) { + // 404 is expected if no concrete write indices have been created + if (error.statusCode !== 404) { + logger.error( + `Error fetching concrete indices for ${indexPatterns.alias} pattern - ${error.message}` + ); + throw error; + } + } + + // let concreteWriteIndicesExist = false; + const concreteWriteIndicesExist = concreteIndices.length > 0; + + // if a concrete write ds already exists, update the underlying mapping + if (concreteIndices.length > 0) { + await updateIndexMappings({ logger, esClient, totalFieldsLimit, concreteIndices }); + } + + // check if a concrete write ds already exists + if (!concreteWriteIndicesExist) { + try { + await retryTransientEsErrors( + () => + esClient.indices.createDataStream({ + name: indexPatterns.alias, + }), + { logger } + ); + } catch (error) { + logger.error(`Error creating concrete write index - ${error.message}`); + throw error; + } + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_es.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_es.ts new file mode 100644 index 0000000000000..7a3839ad3c5bc --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_es.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { errors as EsErrors } from '@elastic/elasticsearch'; + +const MAX_ATTEMPTS = 3; + +const retryResponseStatuses = [ + 503, // ServiceUnavailable + 408, // RequestTimeout + 410, // Gone +]; + +const isRetryableError = (e: Error) => + e instanceof EsErrors.NoLivingConnectionsError || + e instanceof EsErrors.ConnectionError || + e instanceof EsErrors.TimeoutError || + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + (e instanceof EsErrors.ResponseError && retryResponseStatuses.includes(e?.statusCode!)); + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const retryTransientEsErrors = async ( + esCall: () => Promise, + { + logger, + attempt = 0, + }: { + logger: Logger; + attempt?: number; + } +): Promise => { + try { + return await esCall(); + } catch (e) { + if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + const retryCount = attempt + 1; + const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s... + + logger.warn( + `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ + e.stack + }` + ); + + // delay with some randomness + await delay(retryDelaySec * 1000 * Math.random()); + return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); + } + + throw e; + } +}; From c9183866e5babb824d71c1d1dd3cfa9d6468d9df Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Fri, 26 May 2023 14:02:27 +0200 Subject: [PATCH 03/15] Fix mocks --- .../routes/__mocks__/request_context.ts | 3 +++ .../__mocks__/risk_engine_data_client_mock.ts | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 x-pack/plugins/security_solution/server/lib/risk_engine/__mocks__/risk_engine_data_client_mock.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts index bfda85063f2b9..e3a16c048fa8a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts @@ -34,6 +34,7 @@ import type { import { getEndpointAuthzInitialStateMock } from '../../../../../common/endpoint/service/authz/mocks'; import type { EndpointAuthz } from '../../../../../common/endpoint/types/authz'; +import { riskEngineDataClientMock } from '../../../risk_engine/__mocks__/risk_engine_data_client_mock'; export const createMockClients = () => { const core = coreMock.createRequestHandlerContext(); @@ -59,6 +60,7 @@ export const createMockClients = () => { config: createMockConfig(), appClient: siemMock.createClient(), ruleExecutionLog: ruleExecutionLogMock.forRoutes.create(), + riskEngineDataClient: riskEngineDataClientMock.create(), }; }; @@ -135,6 +137,7 @@ const createSecuritySolutionRequestContextMock = ( // TODO: Mock EndpointInternalFleetServicesInterface and return the mocked object. throw new Error('Not implemented'); }), + getRiskEngineDataClient: jest.fn(() => clients.riskEngineDataClient), }; }; diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/__mocks__/risk_engine_data_client_mock.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/__mocks__/risk_engine_data_client_mock.ts new file mode 100644 index 0000000000000..0e9f1fade7bb6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/__mocks__/risk_engine_data_client_mock.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RiskEngineDataClient } from '../risk_engine_data_client'; + +const createRiskEngineDataClientMock = () => + ({ + getWriter: jest.fn(), + initializeResources: jest.fn(), + } as unknown as jest.Mocked); + +export const riskEngineDataClientMock = { create: createRiskEngineDataClientMock }; From 5f12c176d7773504d45c0d8621314715804a5a8f Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Mon, 5 Jun 2023 16:00:48 +0200 Subject: [PATCH 04/15] Add integrations tests --- .../security_and_spaces/group10/index.ts | 1 + .../group10/risk_engine_install_resources.ts | 150 ++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/group10/risk_engine_install_resources.ts diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/index.ts index 4449e9ca07800..7abf4c536e1e9 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/index.ts @@ -37,5 +37,6 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./throttle')); loadTestFile(require.resolve('./ignore_fields')); loadTestFile(require.resolve('./migrations')); + loadTestFile(require.resolve('./risk_engine_install_resources')); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/risk_engine_install_resources.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/risk_engine_install_resources.ts new file mode 100644 index 0000000000000..a7cae20fa8b34 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group10/risk_engine_install_resources.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const es = getService('es'); + + describe('install risk engine resources', () => { + it('should install resources on startup', async () => { + const ilmPolicyName = '.risk-score-ilm-policy'; + const componentTemplateName = '.risk-score-mappings'; + const indexTemplateName = '.risk-score.risk-score-default-index-template'; + const indexName = 'risk-score.risk-score-default'; + + const ilmPolicy = await es.ilm.getLifecycle({ + name: ilmPolicyName, + }); + + expect(ilmPolicy[ilmPolicyName].policy).to.eql({ + _meta: { + managed: true, + }, + phases: { + hot: { + min_age: '0ms', + actions: { + rollover: { + max_age: '30d', + max_primary_shard_size: '50gb', + }, + }, + }, + }, + }); + + const { component_templates: componentTemplates1 } = await es.cluster.getComponentTemplate({ + name: componentTemplateName, + }); + + expect(componentTemplates1.length).to.eql(1); + const componentTemplate = componentTemplates1[0]; + + expect(componentTemplate.name).to.eql(componentTemplateName); + expect(componentTemplate.component_template.template.mappings).to.eql({ + dynamic: 'strict', + properties: { + '@timestamp': { + type: 'date', + }, + alertsScore: { + type: 'float', + }, + identifierField: { + type: 'keyword', + }, + identifierValue: { + type: 'keyword', + }, + level: { + type: 'keyword', + }, + otherScore: { + type: 'float', + }, + riskiestInputs: { + properties: { + id: { + type: 'keyword', + }, + index: { + type: 'keyword', + }, + riskScore: { + type: 'float', + }, + }, + type: 'nested', + }, + totalScore: { + type: 'float', + }, + totalScoreNormalized: { + type: 'float', + }, + }, + }); + + const { index_templates: indexTemplates } = await es.indices.getIndexTemplate({ + name: indexTemplateName, + }); + expect(indexTemplates.length).to.eql(1); + const indexTemplate = indexTemplates[0]; + expect(indexTemplate.name).to.eql(indexTemplateName); + expect(indexTemplate.index_template.index_patterns).to.eql(['risk-score.risk-score-default']); + expect(indexTemplate.index_template.composed_of).to.eql(['.risk-score-mappings']); + expect(indexTemplate.index_template.template!.mappings?.dynamic).to.eql(false); + expect(indexTemplate.index_template.template!.mappings?._meta?.managed).to.eql(true); + expect(indexTemplate.index_template.template!.mappings?._meta?.namespace).to.eql('default'); + expect(indexTemplate.index_template.template!.mappings?._meta?.kibana?.version).to.be.a( + 'string' + ); + expect(indexTemplate.index_template.template!.settings).to.eql({ + index: { + lifecycle: { + name: '.risk-score-ilm-policy', + }, + mapping: { + total_fields: { + limit: '1000', + }, + }, + hidden: 'true', + auto_expand_replicas: '0-1', + }, + }); + + const dsResponse = await es.indices.get({ + index: indexName, + }); + + const dataStream = Object.values(dsResponse).find((ds) => ds.data_stream === indexName); + + expect(dataStream?.mappings?._meta?.managed).to.eql(true); + expect(dataStream?.mappings?._meta?.namespace).to.eql('default'); + expect(dataStream?.mappings?._meta?.kibana?.version).to.be.a('string'); + expect(dataStream?.mappings?.dynamic).to.eql('false'); + + expect(dataStream?.settings?.index?.lifecycle).to.eql({ + name: '.risk-score-ilm-policy', + }); + + expect(dataStream?.settings?.index?.mapping).to.eql({ + total_fields: { + limit: '1000', + }, + }); + + expect(dataStream?.settings?.index?.hidden).to.eql('true'); + expect(dataStream?.settings?.index?.number_of_shards).to.eql(1); + expect(dataStream?.settings?.index?.auto_expand_replicas).to.eql('0-1'); + }); + }); +}; From 87f19bb0fc38e2408ebb2cce3b1f29a094b127ce Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Mon, 5 Jun 2023 16:45:04 +0200 Subject: [PATCH 05/15] Add tests --- .../server/lib/risk_engine/configurations.ts | 2 +- .../risk_engine_data_client.test.ts | 221 ++++++++++++++++++ .../risk_engine/risk_engine_data_client.ts | 21 +- 3 files changed, 241 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts index 68cd6820d70a5..93ddb0cd4c14f 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts @@ -68,7 +68,7 @@ export const riskFieldMap = { } as const; export const ilmPolicyName = '.risk-score-ilm-policy'; -export const mappingComponentName = 'risk-score-mappings'; +export const mappingComponentName = '.risk-score-mappings'; export const totalFieldsLimit = 1000; const riskScoreBaseIndexName = 'risk-score'; diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts new file mode 100644 index 0000000000000..92d8564f5be8f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts @@ -0,0 +1,221 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + createOrUpdateComponentTemplate, + createOrUpdateIlmPolicy, + createOrUpdateIndexTemplate, +} from '@kbn/alerting-plugin/server'; +import { loggingSystemMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { RiskEngineDataClient } from './risk_engine_data_client'; +import { createConcreteWriteIndex } from './utils/create_ds'; + +jest.mock('@kbn/alerting-plugin/server', () => ({ + createOrUpdateComponentTemplate: jest.fn(), + createOrUpdateIlmPolicy: jest.fn(), + createOrUpdateIndexTemplate: jest.fn(), +})); + +jest.mock('./utils/create_ds', () => ({ + createConcreteWriteIndex: jest.fn(), +})); + +describe('RiskEngineDataClient', () => { + let riskEngineDataClient: RiskEngineDataClient; + let logger: ReturnType; + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const totalFieldsLimit = 1000; + + beforeEach(() => { + logger = loggingSystemMock.createLogger(); + const options = { + logger, + kibanaVersion: '8.9.0', + elasticsearchClientPromise: Promise.resolve(esClient), + }; + riskEngineDataClient = new RiskEngineDataClient(options); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('getWriter', () => { + it('should return a writer object', async () => { + const writer = await riskEngineDataClient.getWriter({ namespace: 'default' }); + expect(writer).toBeDefined(); + expect(typeof writer?.bulk).toBe('function'); + }); + + it('should cache and return the same writer for the same namespace', async () => { + const writer1 = await riskEngineDataClient.getWriter({ namespace: 'default' }); + const writer2 = await riskEngineDataClient.getWriter({ namespace: 'default' }); + const writer3 = await riskEngineDataClient.getWriter({ namespace: 'space-1' }); + + expect(writer1).toEqual(writer2); + expect(writer2).not.toEqual(writer3); + }); + + it('should cache writer and not call initializeResources for a second tme', async () => { + const initializeResourcesSpy = jest.spyOn(riskEngineDataClient, 'initializeResources'); + await riskEngineDataClient.getWriter({ namespace: 'default' }); + await riskEngineDataClient.getWriter({ namespace: 'default' }); + expect(initializeResourcesSpy).toHaveBeenCalledTimes(1); + }); + }); + + describe('initializeResources succes', () => { + it('should initialize risk engine resources', async () => { + await riskEngineDataClient.initializeResources({ namespace: 'default' }); + + expect(createOrUpdateIlmPolicy).toHaveBeenCalledWith({ + logger, + esClient, + name: '.risk-score-ilm-policy', + policy: { + _meta: { + managed: true, + }, + phases: { + hot: { + actions: { + rollover: { + max_age: '30d', + max_primary_shard_size: '50gb', + }, + }, + }, + }, + }, + }); + + expect(createOrUpdateComponentTemplate).toHaveBeenCalledWith({ + logger, + esClient, + template: { + name: '.risk-score-mappings', + _meta: { + managed: true, + }, + template: { + settings: {}, + mappings: { + dynamic: 'strict', + properties: { + '@timestamp': { + type: 'date', + }, + alertsScore: { + type: 'float', + }, + identifierField: { + type: 'keyword', + }, + identifierValue: { + type: 'keyword', + }, + level: { + type: 'keyword', + }, + otherScore: { + type: 'float', + }, + riskiestInputs: { + properties: { + id: { + type: 'keyword', + }, + index: { + type: 'keyword', + }, + riskScore: { + type: 'float', + }, + }, + type: 'nested', + }, + totalScore: { + type: 'float', + }, + totalScoreNormalized: { + type: 'float', + }, + }, + }, + }, + }, + totalFieldsLimit, + }); + + expect(createOrUpdateIndexTemplate).toHaveBeenCalledWith({ + logger, + esClient, + template: { + name: '.risk-score.risk-score-default-index-template', + body: { + data_stream: { hidden: true }, + index_patterns: ['risk-score.risk-score-default'], + composed_of: ['.risk-score-mappings'], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: '.risk-score-ilm-policy', + }, + 'index.mapping.total_fields.limit': totalFieldsLimit, + }, + mappings: { + dynamic: false, + _meta: { + kibana: { + version: '8.9.0', + }, + managed: true, + namespace: 'default', + }, + }, + }, + _meta: { + kibana: { + version: '8.9.0', + }, + managed: true, + namespace: 'default', + }, + }, + }, + }); + + expect(createConcreteWriteIndex).toHaveBeenCalledWith({ + logger, + esClient, + totalFieldsLimit, + indexPatterns: { + template: `.risk-score.risk-score-default-index-template`, + alias: `risk-score.risk-score-default`, + pattern: '', + basePattern: '', + name: '', + }, + }); + }); + }); + + describe('initializeResources error', () => { + it('should handle errors during initialization', async () => { + const error = new Error('There error'); + (createOrUpdateIlmPolicy as jest.Mock).mockRejectedValue(error); + + await riskEngineDataClient.initializeResources({ namespace: 'default' }); + + expect(logger.error).toHaveBeenCalledWith( + `Error initializing risk engine resources: ${error}` + ); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts index f5a0f551bf401..56863c55f2f4a 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts @@ -33,14 +33,29 @@ interface RiskEngineDataClientOpts { elasticsearchClientPromise: Promise; } +interface Writer { + bulk: () => Promise; +} + export class RiskEngineDataClient { + private writerCache: Map = new Map(); constructor(private readonly options: RiskEngineDataClientOpts) {} public async getWriter({ namespace }: { namespace: string }) { - this.initializeResources({ namespace }); - return { + if (this.writerCache.get(namespace)) { + return this.writerCache.get(namespace); + } + + await this.initializeResources({ namespace }); + return this.writerCache.get(namespace); + } + + private async initialiseWriter(namespace: string) { + const writer: Writer = { bulk: async () => {}, }; + this.writerCache.set(namespace, writer); + return this.writerCache.get(namespace); } public async initializeResources({ @@ -130,6 +145,8 @@ export class RiskEngineDataClient { totalFieldsLimit, indexPatterns, }); + + this.initialiseWriter(namespace); } catch (error) { this.options.logger.error(`Error initializing risk engine resources: ${error}`); } From 6aeaa7c459ce4222c9c5c572c51579ccf9bbba48 Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Wed, 7 Jun 2023 19:43:29 +0200 Subject: [PATCH 06/15] Add feature flag --- .../security_solution/common/experimental_features.ts | 5 +++++ x-pack/plugins/security_solution/server/plugin.ts | 4 +++- .../test/detection_engine_api_integration/common/config.ts | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index e313796fe9559..d96bc8b33aadd 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -148,6 +148,11 @@ export const allowedExperimentalValues = Object.freeze({ * The flag doesn't have to be documented and has to be removed after the feature is ready to release. */ detectionsCoverageOverview: false, + + /** + * Enable risk engine client and initialisation of datastream, component templates and mappings + */ + riskScoringPersistence: false, }); type ExperimentalConfigKeys = Array; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 7b93d8bd6ffdb..90cd9b121372f 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -167,7 +167,9 @@ export class Plugin implements ISecuritySolutionPlugin { .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), }); - this.riskEngineDataClient.initializeResources({}); + if(experimentalFeatures.riskScoringPersistence) { + this.riskEngineDataClient.initializeResources({}); + } const requestContextFactory = new RequestContextFactory({ config, diff --git a/x-pack/test/detection_engine_api_integration/common/config.ts b/x-pack/test/detection_engine_api_integration/common/config.ts index a1a71bf907b86..7869cbf07ce06 100644 --- a/x-pack/test/detection_engine_api_integration/common/config.ts +++ b/x-pack/test/detection_engine_api_integration/common/config.ts @@ -76,6 +76,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'previewTelemetryUrlEnabled', + 'riskScoringPersistence' ])}`, '--xpack.task_manager.poll_interval=1000', `--xpack.actions.preconfigured=${JSON.stringify({ From c06e01b0648b527588c3f8c8a1364caafb79601f Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Wed, 7 Jun 2023 18:09:05 +0000 Subject: [PATCH 07/15] [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix' --- x-pack/plugins/security_solution/server/plugin.ts | 2 +- x-pack/test/detection_engine_api_integration/common/config.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 90cd9b121372f..0c078f751180a 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -167,7 +167,7 @@ export class Plugin implements ISecuritySolutionPlugin { .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), }); - if(experimentalFeatures.riskScoringPersistence) { + if (experimentalFeatures.riskScoringPersistence) { this.riskEngineDataClient.initializeResources({}); } diff --git a/x-pack/test/detection_engine_api_integration/common/config.ts b/x-pack/test/detection_engine_api_integration/common/config.ts index 7869cbf07ce06..ace942536f105 100644 --- a/x-pack/test/detection_engine_api_integration/common/config.ts +++ b/x-pack/test/detection_engine_api_integration/common/config.ts @@ -76,7 +76,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'previewTelemetryUrlEnabled', - 'riskScoringPersistence' + 'riskScoringPersistence', ])}`, '--xpack.task_manager.poll_interval=1000', `--xpack.actions.preconfigured=${JSON.stringify({ From 0ebb07b4ad452725dcb4ebbb5cbac755bfd4090a Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Thu, 15 Jun 2023 05:53:00 +0000 Subject: [PATCH 08/15] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../security_solution/server/request_context_factory.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index 9b768af688097..405a6cbae01aa 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -58,7 +58,14 @@ export class RequestContextFactory implements IRequestContextFactory { request: KibanaRequest ): Promise { const { options, appClientFactory } = this; - const { config, core, plugins, endpointAppContextService, ruleMonitoringService, riskEngineDataClient } = options; + const { + config, + core, + plugins, + endpointAppContextService, + ruleMonitoringService, + riskEngineDataClient, + } = options; const { lists, ruleRegistry, security } = plugins; From db8ac4d6b875c41e85657aca3848e43261401113 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 19 Jun 2023 09:28:14 +0000 Subject: [PATCH 09/15] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../security_solution/common/experimental_features.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index 513f2a066f6d8..dd00c9b3ab190 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -116,17 +116,16 @@ export const allowedExperimentalValues = Object.freeze({ * The flag doesn't have to be documented and has to be removed after the feature is ready to release. */ detectionsCoverageOverview: false, - + /** * Enable risk engine client and initialisation of datastream, component templates and mappings */ riskScoringPersistence: false, - + /** * Enables experimental Entity Analytics HTTP endpoints */ riskScoringRoutesEnabled: false, - }); type ExperimentalConfigKeys = Array; From b5529227251cff342ddd6514acf56b75bb601e07 Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Mon, 19 Jun 2023 14:22:25 +0200 Subject: [PATCH 10/15] PR fixes --- .../server/lib/risk_engine/configurations.ts | 25 +++++-- .../risk_engine_data_client.test.ts | 6 +- .../risk_engine/risk_engine_data_client.ts | 68 ++++++++----------- .../{create_ds.ts => create_datastream.ts} | 36 ++++------ ...try_es.ts => retry_transient_es_errors.ts} | 0 5 files changed, 66 insertions(+), 69 deletions(-) rename x-pack/plugins/security_solution/server/lib/risk_engine/utils/{create_ds.ts => create_datastream.ts} (83%) rename x-pack/plugins/security_solution/server/lib/risk_engine/utils/{retry_es.ts => retry_transient_es_errors.ts} (100%) diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts index 93ddb0cd4c14f..2b15cd373baf0 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts @@ -4,8 +4,26 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import type { FieldMap } from '@kbn/alerts-as-data-utils'; +import { IIndexPatternString } from './utils/create_datastream'; -export const riskFieldMap = { +export const ilmPolicy = { + _meta: { + managed: true, + }, + phases: { + hot: { + actions: { + rollover: { + max_age: '30d', + max_primary_shard_size: '50gb', + }, + }, + }, + }, +}; + +export const riskFieldMap: FieldMap = { '@timestamp': { type: 'date', array: false, @@ -73,10 +91,7 @@ export const totalFieldsLimit = 1000; const riskScoreBaseIndexName = 'risk-score'; -export const getIndexPattern = (namespace: string) => ({ +export const getIndexPattern = (namespace: string): IIndexPatternString => ({ template: `.${riskScoreBaseIndexName}.${riskScoreBaseIndexName}-${namespace}-index-template`, alias: `${riskScoreBaseIndexName}.${riskScoreBaseIndexName}-${namespace}`, - pattern: '', - basePattern: '', - name: '', }); diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts index 92d8564f5be8f..713c17efe5d3c 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts @@ -12,7 +12,7 @@ import { } from '@kbn/alerting-plugin/server'; import { loggingSystemMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; import { RiskEngineDataClient } from './risk_engine_data_client'; -import { createConcreteWriteIndex } from './utils/create_ds'; +import { createDataStream } from './utils/create_datastream'; jest.mock('@kbn/alerting-plugin/server', () => ({ createOrUpdateComponentTemplate: jest.fn(), @@ -21,7 +21,7 @@ jest.mock('@kbn/alerting-plugin/server', () => ({ })); jest.mock('./utils/create_ds', () => ({ - createConcreteWriteIndex: jest.fn(), + createDataStream: jest.fn(), })); describe('RiskEngineDataClient', () => { @@ -191,7 +191,7 @@ describe('RiskEngineDataClient', () => { }, }); - expect(createConcreteWriteIndex).toHaveBeenCalledWith({ + expect(createDataStream).toHaveBeenCalledWith({ logger, esClient, totalFieldsLimit, diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts index 56863c55f2f4a..772046a6b4438 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts @@ -20,8 +20,9 @@ import { totalFieldsLimit, mappingComponentName, ilmPolicyName, + ilmPolicy } from './configurations'; -import { createConcreteWriteIndex } from './utils/create_ds'; +import { createDataStream } from './utils/create_datastream'; interface InitializeRiskEngineResourcesOpts { namespace?: string; @@ -50,12 +51,12 @@ export class RiskEngineDataClient { return this.writerCache.get(namespace); } - private async initialiseWriter(namespace: string) { + private async initializeWriter(namespace: string) { const writer: Writer = { bulk: async () => {}, }; this.writerCache.set(namespace, writer); - return this.writerCache.get(namespace); + return writer; } public async initializeResources({ @@ -74,42 +75,31 @@ export class RiskEngineDataClient { namespace, }; - await createOrUpdateIlmPolicy({ - logger: this.options.logger, - esClient, - name: ilmPolicyName, - policy: { - _meta: { - managed: true, - }, - phases: { - hot: { - actions: { - rollover: { - max_age: '30d', - max_primary_shard_size: '50gb', - }, - }, + await Promise.all([ + createOrUpdateIlmPolicy({ + logger: this.options.logger, + esClient, + name: ilmPolicyName, + policy: ilmPolicy, + }), + createOrUpdateComponentTemplate({ + logger: this.options.logger, + esClient, + template: { + name: mappingComponentName, + _meta: { + managed: true, }, - }, - }, - }); + template: { + settings: {}, + mappings: mappingFromFieldMap(riskFieldMap, 'strict'), + }, + } as ClusterPutComponentTemplateRequest, + totalFieldsLimit, + }) + ]) + - await createOrUpdateComponentTemplate({ - logger: this.options.logger, - esClient, - template: { - name: mappingComponentName, - _meta: { - managed: true, - }, - template: { - settings: {}, - mappings: mappingFromFieldMap(riskFieldMap, 'strict'), - }, - } as ClusterPutComponentTemplateRequest, - totalFieldsLimit, - }); await createOrUpdateIndexTemplate({ logger: this.options.logger, @@ -139,14 +129,14 @@ export class RiskEngineDataClient { }, }); - await createConcreteWriteIndex({ + await createDataStream({ logger: this.options.logger, esClient, totalFieldsLimit, indexPatterns, }); - this.initialiseWriter(namespace); + this.initializeWriter(namespace); } catch (error) { this.options.logger.error(`Error initializing risk engine resources: ${error}`); } diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_ds.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts similarity index 83% rename from x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_ds.ts rename to x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts index a94e701cd9a07..81b8deffa90c2 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_ds.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts @@ -12,15 +12,11 @@ import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { Logger, ElasticsearchClient } from '@kbn/core/server'; import { get } from 'lodash'; -import { retryTransientEsErrors } from './retry_es'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; export interface IIndexPatternString { template: string; - pattern: string; alias: string; - name: string; - basePattern: string; - secondaryAlias?: string; } interface ConcreteIndexInfo { @@ -144,12 +140,9 @@ interface CreateConcreteWriteIndexOpts { indexPatterns: IIndexPatternString; } /** - * Installs index template that uses installed component template - * Prior to installation, simulates the installation to check for possible - * conflicts. Simulate should return an empty mapping if a template - * conflicts with an already installed template. + * Create a data stream */ -export const createConcreteWriteIndex = async ({ +export const createDataStream = async ({ logger, esClient, indexPatterns, @@ -157,8 +150,8 @@ export const createConcreteWriteIndex = async ({ }: CreateConcreteWriteIndexOpts) => { logger.info(`Creating data stream - ${indexPatterns.alias}`); - // check if a concrete write index already exists - let concreteIndices: ConcreteIndexInfo[] = []; + // check if a ds already exists + let dataStreams: ConcreteIndexInfo[] = []; try { // Specify both the index pattern for the backing indices and their aliases // The alias prevents the request from finding other namespaces that could match the -* pattern @@ -167,19 +160,19 @@ export const createConcreteWriteIndex = async ({ { logger } ); - concreteIndices = response.data_streams.map((dataStream) => ({ + dataStreams = response.data_streams.map((dataStream) => ({ index: dataStream.name, alias: dataStream.name, isWriteIndex: true, })); logger.debug( - `Found ${concreteIndices.length} concrete indices for ${ + `Found ${dataStreams.length} concrete indices for ${ indexPatterns.alias - } - ${JSON.stringify(concreteIndices)}` + } - ${JSON.stringify(dataStreams)}` ); } catch (error) { - // 404 is expected if no concrete write indices have been created + // 404 is expected if no ds have been created if (error.statusCode !== 404) { logger.error( `Error fetching concrete indices for ${indexPatterns.alias} pattern - ${error.message}` @@ -188,16 +181,15 @@ export const createConcreteWriteIndex = async ({ } } - // let concreteWriteIndicesExist = false; - const concreteWriteIndicesExist = concreteIndices.length > 0; + const isDataStreamsExist = dataStreams.length > 0; // if a concrete write ds already exists, update the underlying mapping - if (concreteIndices.length > 0) { - await updateIndexMappings({ logger, esClient, totalFieldsLimit, concreteIndices }); + if (dataStreams.length > 0) { + await updateIndexMappings({ logger, esClient, totalFieldsLimit, concreteIndices: dataStreams }); } // check if a concrete write ds already exists - if (!concreteWriteIndicesExist) { + if (!isDataStreamsExist) { try { await retryTransientEsErrors( () => @@ -207,7 +199,7 @@ export const createConcreteWriteIndex = async ({ { logger } ); } catch (error) { - logger.error(`Error creating concrete write index - ${error.message}`); + logger.error(`Error creating ds - ${error.message}`); throw error; } } diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_es.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_transient_es_errors.ts similarity index 100% rename from x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_es.ts rename to x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_transient_es_errors.ts From 431af67324c52a58e8b7dfc4b6477f059b0e319e Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 19 Jun 2023 17:07:05 +0000 Subject: [PATCH 11/15] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../server/lib/risk_engine/configurations.ts | 2 +- .../server/lib/risk_engine/risk_engine_data_client.ts | 8 +++----- .../server/lib/risk_engine/utils/create_datastream.ts | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts index 2b15cd373baf0..64b31b2c705a5 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/configurations.ts @@ -5,7 +5,7 @@ * 2.0. */ import type { FieldMap } from '@kbn/alerts-as-data-utils'; -import { IIndexPatternString } from './utils/create_datastream'; +import type { IIndexPatternString } from './utils/create_datastream'; export const ilmPolicy = { _meta: { diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts index 772046a6b4438..90c4dd205ebe9 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts @@ -20,7 +20,7 @@ import { totalFieldsLimit, mappingComponentName, ilmPolicyName, - ilmPolicy + ilmPolicy, } from './configurations'; import { createDataStream } from './utils/create_datastream'; @@ -96,10 +96,8 @@ export class RiskEngineDataClient { }, } as ClusterPutComponentTemplateRequest, totalFieldsLimit, - }) - ]) - - + }), + ]); await createOrUpdateIndexTemplate({ logger: this.options.logger, diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts index 81b8deffa90c2..8dead4fc954e4 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts @@ -167,9 +167,9 @@ export const createDataStream = async ({ })); logger.debug( - `Found ${dataStreams.length} concrete indices for ${ - indexPatterns.alias - } - ${JSON.stringify(dataStreams)}` + `Found ${dataStreams.length} concrete indices for ${indexPatterns.alias} - ${JSON.stringify( + dataStreams + )}` ); } catch (error) { // 404 is expected if no ds have been created From 6eda6eddc50baa9529eacaff0422a41d6201fda7 Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Mon, 19 Jun 2023 19:39:04 +0200 Subject: [PATCH 12/15] Error handling --- .../server/lib/risk_engine/risk_engine_data_client.ts | 2 +- .../server/lib/risk_engine/utils/create_datastream.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts index 90c4dd205ebe9..b1ce40dabb4c8 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts @@ -136,7 +136,7 @@ export class RiskEngineDataClient { this.initializeWriter(namespace); } catch (error) { - this.options.logger.error(`Error initializing risk engine resources: ${error}`); + this.options.logger.error(`Error initializing risk engine resources: ${error.message}`); } } } diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts index 8dead4fc954e4..db1dee65feb75 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts @@ -150,7 +150,7 @@ export const createDataStream = async ({ }: CreateConcreteWriteIndexOpts) => { logger.info(`Creating data stream - ${indexPatterns.alias}`); - // check if a ds already exists + // check if a datastream already exists let dataStreams: ConcreteIndexInfo[] = []; try { // Specify both the index pattern for the backing indices and their aliases @@ -172,7 +172,7 @@ export const createDataStream = async ({ )}` ); } catch (error) { - // 404 is expected if no ds have been created + // 404 is expected if no datastream have been created if (error.statusCode !== 404) { logger.error( `Error fetching concrete indices for ${indexPatterns.alias} pattern - ${error.message}` @@ -183,12 +183,12 @@ export const createDataStream = async ({ const isDataStreamsExist = dataStreams.length > 0; - // if a concrete write ds already exists, update the underlying mapping + // if a concrete write datastream already exists, update the underlying mapping if (dataStreams.length > 0) { await updateIndexMappings({ logger, esClient, totalFieldsLimit, concreteIndices: dataStreams }); } - // check if a concrete write ds already exists + // check if a concrete write datastream already exists if (!isDataStreamsExist) { try { await retryTransientEsErrors( @@ -199,7 +199,7 @@ export const createDataStream = async ({ { logger } ); } catch (error) { - logger.error(`Error creating ds - ${error.message}`); + logger.error(`Error creating datastream - ${error.message}`); throw error; } } From 4e8c558c0740a9de1c070bb908cd48737f215679 Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Mon, 19 Jun 2023 20:38:54 +0200 Subject: [PATCH 13/15] Import fix --- .../server/lib/risk_engine/risk_engine_data_client.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts index 713c17efe5d3c..b8999e0ecbfd6 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts @@ -20,7 +20,7 @@ jest.mock('@kbn/alerting-plugin/server', () => ({ createOrUpdateIndexTemplate: jest.fn(), })); -jest.mock('./utils/create_ds', () => ({ +jest.mock('./utils/create_datastream', () => ({ createDataStream: jest.fn(), })); From 0b36d21335970bcbb41c3e36a7b7f6993f78db6a Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Tue, 20 Jun 2023 11:36:05 +0200 Subject: [PATCH 14/15] Fix tests --- .../server/lib/risk_engine/risk_engine_data_client.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts index b8999e0ecbfd6..391d89b2ebf8b 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.test.ts @@ -198,9 +198,6 @@ describe('RiskEngineDataClient', () => { indexPatterns: { template: `.risk-score.risk-score-default-index-template`, alias: `risk-score.risk-score-default`, - pattern: '', - basePattern: '', - name: '', }, }); }); @@ -214,7 +211,7 @@ describe('RiskEngineDataClient', () => { await riskEngineDataClient.initializeResources({ namespace: 'default' }); expect(logger.error).toHaveBeenCalledWith( - `Error initializing risk engine resources: ${error}` + `Error initializing risk engine resources: ${error.message}` ); }); }); From 49421ff2df9f3fc74070e2547b9b4f5fb0b09a14 Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Wed, 21 Jun 2023 12:27:15 +0200 Subject: [PATCH 15/15] Add types and tests --- .../risk_engine/risk_engine_data_client.ts | 8 +- .../risk_engine/utils/create_datastream.ts | 4 +- .../utils/retry_transient_es_errors.test.ts | 95 +++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_transient_es_errors.test.ts diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts index b1ce40dabb4c8..9b77741bb164a 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/risk_engine_data_client.ts @@ -42,16 +42,16 @@ export class RiskEngineDataClient { private writerCache: Map = new Map(); constructor(private readonly options: RiskEngineDataClientOpts) {} - public async getWriter({ namespace }: { namespace: string }) { + public async getWriter({ namespace }: { namespace: string }): Promise { if (this.writerCache.get(namespace)) { - return this.writerCache.get(namespace); + return this.writerCache.get(namespace) as Writer; } await this.initializeResources({ namespace }); - return this.writerCache.get(namespace); + return this.writerCache.get(namespace) as Writer; } - private async initializeWriter(namespace: string) { + private async initializeWriter(namespace: string): Promise { const writer: Writer = { bulk: async () => {}, }; diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts index db1dee65feb75..910ba5e887046 100644 --- a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/create_datastream.ts @@ -181,7 +181,7 @@ export const createDataStream = async ({ } } - const isDataStreamsExist = dataStreams.length > 0; + const dataStreamsExist = dataStreams.length > 0; // if a concrete write datastream already exists, update the underlying mapping if (dataStreams.length > 0) { @@ -189,7 +189,7 @@ export const createDataStream = async ({ } // check if a concrete write datastream already exists - if (!isDataStreamsExist) { + if (!dataStreamsExist) { try { await retryTransientEsErrors( () => diff --git a/x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_transient_es_errors.test.ts b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_transient_es_errors.test.ts new file mode 100644 index 0000000000000..2501c57776d80 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/risk_engine/utils/retry_transient_es_errors.test.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { loggerMock } from '@kbn/logging-mocks'; +import { errors as EsErrors } from '@elastic/elasticsearch'; + +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +const logger = loggerMock.create(); +const randomDelayMultiplier = 0.01; + +describe('retryTransientErrors', () => { + beforeEach(() => { + jest.resetAllMocks(); + jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier); + }); + + it("doesn't retry if operation is successful", async () => { + const esCallMock = jest.fn().mockResolvedValue('success'); + expect(await retryTransientEsErrors(esCallMock, { logger })).toEqual('success'); + expect(esCallMock).toHaveBeenCalledTimes(1); + }); + + it('logs a warning message on retry', async () => { + const esCallMock = jest + .fn() + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockResolvedValue('success'); + + await retryTransientEsErrors(esCallMock, { logger }); + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn.mock.calls[0][0]).toMatch( + `Retrying Elasticsearch operation after [2s] due to error: ConnectionError: foo ConnectionError: foo` + ); + }); + + it('retries with an exponential backoff', async () => { + let attempt = 0; + const esCallMock = jest.fn(async () => { + attempt++; + if (attempt < 4) { + throw new EsErrors.ConnectionError('foo'); + } else { + return 'success'; + } + }); + + expect(await retryTransientEsErrors(esCallMock, { logger })).toEqual('success'); + expect(esCallMock).toHaveBeenCalledTimes(4); + expect(logger.warn).toHaveBeenCalledTimes(3); + expect(logger.warn.mock.calls[0][0]).toMatch( + `Retrying Elasticsearch operation after [2s] due to error: ConnectionError: foo ConnectionError: foo` + ); + expect(logger.warn.mock.calls[1][0]).toMatch( + `Retrying Elasticsearch operation after [4s] due to error: ConnectionError: foo ConnectionError: foo` + ); + expect(logger.warn.mock.calls[2][0]).toMatch( + `Retrying Elasticsearch operation after [8s] due to error: ConnectionError: foo ConnectionError: foo` + ); + }); + + it('retries each supported error type', async () => { + const errors = [ + new EsErrors.NoLivingConnectionsError('no living connection', { + warnings: [], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + meta: {} as any, + }), + new EsErrors.ConnectionError('no connection'), + new EsErrors.TimeoutError('timeout'), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + new EsErrors.ResponseError({ statusCode: 503, meta: {} as any, warnings: [] }), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + new EsErrors.ResponseError({ statusCode: 408, meta: {} as any, warnings: [] }), + // eslint-disable-next-line @typescript-eslint/no-explicit-any + new EsErrors.ResponseError({ statusCode: 410, meta: {} as any, warnings: [] }), + ]; + + for (const error of errors) { + const esCallMock = jest.fn().mockRejectedValueOnce(error).mockResolvedValue('success'); + expect(await retryTransientEsErrors(esCallMock, { logger })).toEqual('success'); + expect(esCallMock).toHaveBeenCalledTimes(2); + } + }); + + it('does not retry unsupported errors', async () => { + const error = new Error('foo!'); + const esCallMock = jest.fn().mockRejectedValueOnce(error).mockResolvedValue('success'); + await expect(retryTransientEsErrors(esCallMock, { logger })).rejects.toThrow(error); + expect(esCallMock).toHaveBeenCalledTimes(1); + }); +});