From cf18e4637e9a35ff5434bbd8162588eeef24d33e Mon Sep 17 00:00:00 2001 From: ncheckin <68351161+ncheckin@users.noreply.github.com> Date: Fri, 25 Sep 2020 16:03:32 -0400 Subject: [PATCH 01/11] Update tutorial-define-index.asciidoc (#75754) adds windows alternative for curl Co-authored-by: Kaarina Tungseth --- docs/getting-started/tutorial-define-index.asciidoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/getting-started/tutorial-define-index.asciidoc b/docs/getting-started/tutorial-define-index.asciidoc index fbe7450683dbc..cb3f6c9ff0c9b 100644 --- a/docs/getting-started/tutorial-define-index.asciidoc +++ b/docs/getting-started/tutorial-define-index.asciidoc @@ -47,5 +47,11 @@ contains the time series data. [role="screenshot"] image::images/tutorial_index_patterns.png[All tutorial index patterns] +NOTE: When you define an index pattern, the indices that match that pattern must +exist in Elasticsearch and they must contain data. To check if the indices are +available, open the menu, go to *Dev Tools > Console*, then enter `GET _cat/indices`. Alternately, use +`curl -XGET "http://localhost:9200/_cat/indices"`. +For Windows, run `Invoke-RestMethod -Uri "http://localhost:9200/_cat/indices"` in Powershell. + From 500ad8baf038b1fe29c6b11cf305cdfa6f2c1542 Mon Sep 17 00:00:00 2001 From: ncheckin <68351161+ncheckin@users.noreply.github.com> Date: Fri, 25 Sep 2020 16:03:57 -0400 Subject: [PATCH 02/11] Update tutorial-full-experience.asciidoc (#75836) add powershell alternative for curl -O commands at beginning of doc Co-authored-by: Kaarina Tungseth --- docs/getting-started/tutorial-full-experience.asciidoc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/getting-started/tutorial-full-experience.asciidoc b/docs/getting-started/tutorial-full-experience.asciidoc index 1e6fe39dbd013..a7d5412ae0632 100644 --- a/docs/getting-started/tutorial-full-experience.asciidoc +++ b/docs/getting-started/tutorial-full-experience.asciidoc @@ -25,7 +25,14 @@ curl -O https://download.elastic.co/demos/kibana/gettingstarted/8.x/shakespeare. curl -O https://download.elastic.co/demos/kibana/gettingstarted/8.x/accounts.zip curl -O https://download.elastic.co/demos/kibana/gettingstarted/8.x/logs.jsonl.gz -Two of the data sets are compressed. To extract the files, use the following commands: +Alternatively, for Windows users, run the following commands in Powershell: + +[source,shell] +Invoke-RestMethod https://download.elastic.co/demos/kibana/gettingstarted/8.x/shakespeare.json -OutFile shakespeare.json +Invoke-RestMethod https://download.elastic.co/demos/kibana/gettingstarted/8.x/accounts.zip -OutFile accounts.zip +Invoke-RestMethod https://download.elastic.co/demos/kibana/gettingstarted/8.x/logs.jsonl.gz -OutFile logs.jsonl.gz + +Two of the data sets are compressed. To extract the files, use these commands: [source,shell] unzip accounts.zip From faf4b040042918fe804bc3f80e42011a073c2d8c Mon Sep 17 00:00:00 2001 From: Marshall Main <55718608+marshallmain@users.noreply.github.com> Date: Fri, 25 Sep 2020 16:23:25 -0400 Subject: [PATCH 03/11] [Detections][EQL] EQL rule execution in detection engine (#77419) * First draft of EQL rules in detection engine * Reorganize functions to separate files * Start adding eventCategoryOverride option for EQL rules * Add building block alerts for each event within sequence * Use eql instead of eql_query for rule type * Remove unused imports * Fix tests * Add basic tests for buildEqlSearchRequest * Add rulesSchema tests for eql * Add buildSignalFromSequence test * Add threat rule fields to buildRuleWithoutOverrides * Fix buildSignalFromSequence typecheck error * Add more tests * Add tests for wrapBuildingBlock and generateSignalId * Use isEqlRule function and fix import error * delete frank * Move sequence interface to types.ts * Fix import * Remove EQL execution placeholder, add back language to eql rule type * allow no indices for eql search * Fix unit tests for language update * Fix buildEqlSearchRequest tests * Replace signal.child with signal.group * remove unused import * Move sequence signal group building to separate testable function * Unbork the merge conflict resolution Co-authored-by: Elastic Machine --- .../detection_engine/get_query_filter.test.ts | 136 +++++++++++- .../detection_engine/get_query_filter.ts | 79 ++++++- .../schemas/common/schemas.ts | 6 + .../request/add_prepackaged_rules_schema.ts | 2 + .../schemas/request/create_rules_schema.ts | 2 + .../schemas/request/import_rules_schema.ts | 2 + .../schemas/request/patch_rules_schema.ts | 2 + .../schemas/request/update_rules_schema.ts | 2 + .../schemas/response/rules_schema.mocks.ts | 13 +- .../schemas/response/rules_schema.test.ts | 33 ++- .../schemas/response/rules_schema.ts | 23 +- .../routes/__mocks__/request_responses.ts | 1 + .../routes/index/signals_mapping.json | 10 + .../routes/rules/create_rules_bulk_route.ts | 2 + .../routes/rules/create_rules_route.ts | 7 +- .../routes/rules/import_rules_route.ts | 3 + .../routes/rules/patch_rules_bulk_route.ts | 2 + .../routes/rules/patch_rules_route.ts | 2 + .../routes/rules/update_rules_bulk_route.ts | 2 + .../routes/rules/update_rules_route.ts | 2 + .../detection_engine/routes/rules/utils.ts | 1 + .../rules/create_rules.mock.ts | 2 + .../detection_engine/rules/create_rules.ts | 2 + .../rules/install_prepacked_rules.ts | 2 + .../rules/patch_rules.mock.ts | 2 + .../lib/detection_engine/rules/patch_rules.ts | 2 + .../lib/detection_engine/rules/types.ts | 4 + .../rules/update_prepacked_rules.ts | 2 + .../rules/update_rules.mock.ts | 2 + .../detection_engine/rules/update_rules.ts | 2 + .../lib/detection_engine/rules/utils.test.ts | 3 + .../lib/detection_engine/rules/utils.ts | 2 + .../scripts/rules/queries/query_eql.json | 68 ++++++ .../signals/__mocks__/es_results.ts | 108 +++++++++- .../signals/build_bulk_body.test.ts | 201 +++++++++++++++++- .../signals/build_bulk_body.ts | 97 ++++++++- .../signals/build_event_type_signal.ts | 4 +- .../signals/build_rule.test.ts | 124 +++++++++-- .../detection_engine/signals/build_rule.ts | 72 ++++++- .../signals/build_signal.test.ts | 90 +++++--- .../detection_engine/signals/build_signal.ts | 10 +- .../detection_engine/signals/get_filter.ts | 4 +- .../signals/signal_params_schema.mock.ts | 1 + .../signals/signal_params_schema.ts | 1 + .../signals/signal_rule_alert_type.ts | 60 +++++- .../signals/single_bulk_create.ts | 62 +++++- .../lib/detection_engine/signals/types.ts | 19 +- .../detection_engine/signals/utils.test.ts | 53 +++++ .../lib/detection_engine/signals/utils.ts | 57 +++++ .../server/lib/detection_engine/types.ts | 2 + .../security_solution/server/lib/types.ts | 55 +++-- 51 files changed, 1337 insertions(+), 108 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts index 72ef230a42342..0224caafb41a8 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getQueryFilter, buildExceptionFilter } from './get_query_filter'; +import { getQueryFilter, buildExceptionFilter, buildEqlSearchRequest } from './get_query_filter'; import { Filter, EsQueryConfig } from 'src/plugins/data/public'; import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock'; @@ -1085,4 +1085,138 @@ describe('get_filter', () => { }); }); }); + + describe('buildEqlSearchRequest', () => { + test('should build a basic request with time range', () => { + const request = buildEqlSearchRequest( + 'process where true', + ['testindex1', 'testindex2'], + 'now-5m', + 'now', + 100, + undefined, + [], + undefined + ); + expect(request).toEqual({ + method: 'POST', + path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, + body: { + size: 100, + query: 'process where true', + filter: { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'now', + }, + }, + }, + }, + }); + }); + + test('should build a request with timestamp and event category overrides', () => { + const request = buildEqlSearchRequest( + 'process where true', + ['testindex1', 'testindex2'], + 'now-5m', + 'now', + 100, + 'event.ingested', + [], + 'event.other_category' + ); + expect(request).toEqual({ + method: 'POST', + path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, + event_category_field: 'event.other_category', + body: { + size: 100, + query: 'process where true', + filter: { + range: { + 'event.ingested': { + gte: 'now-5m', + lte: 'now', + }, + }, + }, + }, + }); + }); + + test('should build a request with exceptions', () => { + const request = buildEqlSearchRequest( + 'process where true', + ['testindex1', 'testindex2'], + 'now-5m', + 'now', + 100, + undefined, + [getExceptionListItemSchemaMock()], + undefined + ); + expect(request).toEqual({ + method: 'POST', + path: `/testindex1,testindex2/_eql/search?allow_no_indices=true`, + body: { + size: 100, + query: 'process where true', + filter: { + range: { + '@timestamp': { + gte: 'now-5m', + lte: 'now', + }, + }, + bool: { + must_not: { + bool: { + should: [ + { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], + }, + }, + score_mode: 'none', + }, + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + }, + }, + }, + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 466a004c14c66..05c706164ab44 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -17,7 +17,12 @@ import { CreateExceptionListItemSchema, } from '../../../lists/common/schemas'; import { buildExceptionListQueries } from './build_exceptions_query'; -import { Query as QueryString, Language, Index } from './schemas/common/schemas'; +import { + Query as QueryString, + Language, + Index, + TimestampOverrideOrUndefined, +} from './schemas/common/schemas'; export const getQueryFilter = ( query: QueryString, @@ -67,6 +72,78 @@ export const getQueryFilter = ( return buildEsQuery(indexPattern, initialQuery, enabledFilters, config); }; +interface EqlSearchRequest { + method: string; + path: string; + body: object; + event_category_field?: string; +} + +export const buildEqlSearchRequest = ( + query: string, + index: string[], + from: string, + to: string, + size: number, + timestampOverride: TimestampOverrideOrUndefined, + exceptionLists: ExceptionListItemSchema[], + eventCategoryOverride: string | undefined +): EqlSearchRequest => { + const timestamp = timestampOverride ?? '@timestamp'; + const indexPattern: IIndexPattern = { + fields: [], + title: index.join(), + }; + const config: EsQueryConfig = { + allowLeadingWildcards: true, + queryStringOptions: { analyze_wildcard: true }, + ignoreFilterIfFieldNotInIndex: false, + dateFormatTZ: 'Zulu', + }; + const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists }); + let exceptionFilter: Filter | undefined; + if (exceptionQueries.length > 0) { + // Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value), + // allowing us to make 1024-item chunks of exception list items. + // Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a + // very conservative value. + exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024); + } + const indexString = index.join(); + const baseRequest = { + method: 'POST', + path: `/${indexString}/_eql/search?allow_no_indices=true`, + body: { + size, + query, + filter: { + range: { + [timestamp]: { + gte: from, + lte: to, + }, + }, + bool: + exceptionFilter !== undefined + ? { + must_not: { + bool: exceptionFilter?.query.bool, + }, + } + : undefined, + }, + }, + }; + if (eventCategoryOverride) { + return { + ...baseRequest, + event_category_field: eventCategoryOverride, + }; + } else { + return baseRequest; + } +}; + export const buildExceptionFilter = ( exceptionQueries: Query[], indexPattern: IIndexPattern, diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index 5fbba84467ecf..e8d7f409de20a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -40,6 +40,12 @@ export type Enabled = t.TypeOf; export const enabledOrUndefined = t.union([enabled, t.undefined]); export type EnabledOrUndefined = t.TypeOf; +export const event_category_override = t.string; +export type EventCategoryOverride = t.TypeOf; + +export const eventCategoryOverrideOrUndefined = t.union([event_category_override, t.undefined]); +export type EventCategoryOverrideOrUndefined = t.TypeOf; + export const false_positives = t.array(t.string); export type FalsePositives = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts index 69538f025d95d..3f338c57dd930 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts @@ -44,6 +44,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { threat_index, @@ -96,6 +97,7 @@ export const addPrepackagedRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanFalse, // defaults to false if not set during decode + event_category_override, // defaults to "undefined" if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts index c024ba1c48f8d..2489210a26c8f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts @@ -45,6 +45,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { threat_index, @@ -88,6 +89,7 @@ export const createRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode + event_category_override, // defaults to "undefined" if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts index b63d70783b7b5..a411b3d439a1f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts @@ -51,6 +51,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { threat_index, @@ -107,6 +108,7 @@ export const importRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode + event_category_override, // defaults to "undefined" if not set during decode false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts index a674ac86af87b..40e79d96a9e6b 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts @@ -46,6 +46,7 @@ import { timestamp_override, risk_score_mapping, severity_mapping, + event_category_override, } from '../common/schemas'; import { listArrayOrUndefined } from '../types/lists'; @@ -65,6 +66,7 @@ export const patchRulesSchema = t.exact( actions, anomaly_threshold, enabled, + event_category_override, false_positives, filters, from, diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts index 1299dada065e1..8a13dd2f4e908 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts @@ -47,6 +47,7 @@ import { Author, RiskScoreMapping, SeverityMapping, + event_category_override, } from '../common/schemas'; import { @@ -90,6 +91,7 @@ export const updateRulesSchema = t.intersection([ author: DefaultStringArray, // defaults to empty array of strings if not set during decode building_block_type, // defaults to undefined if not set during decode enabled: DefaultBooleanTrue, // defaults to true if not set during decode + event_category_override, false_positives: DefaultStringArray, // defaults to empty string array if not set during decode filters, // defaults to undefined if not set during decode from: DefaultFromString, // defaults to "now-6m" if not set during decode diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts index a462b297d37f8..aaa246c82d9d7 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks.ts @@ -52,7 +52,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem severity: 'high', severity_mapping: [], updated_by: 'elastic_kibana', - tags: [], + tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', threat: [], @@ -61,7 +61,7 @@ export const getRulesSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchem status_date: '2020-02-22T16:47:50.047Z', last_success_at: '2020-02-22T16:47:50.047Z', last_success_message: 'succeeded', - output_index: '.siem-signals-hassanabad-frank-default', + output_index: '.siem-signals-default', max_signals: 100, risk_score: 55, risk_score_mapping: [], @@ -110,3 +110,12 @@ export const getThreatMatchingSchemaMock = (anchorDate: string = ANCHOR_DATE): R ], }; }; + +export const getRulesEqlSchemaMock = (anchorDate: string = ANCHOR_DATE): RulesSchema => { + return { + ...getRulesSchemaMock(anchorDate), + language: 'eql', + type: 'eql', + query: 'process where true', + }; +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts index 3a47d4af6ac14..c5bad3c55066b 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts @@ -18,6 +18,7 @@ import { addTimelineTitle, addMlFields, addThreatMatchFields, + addEqlFields, } from './rules_schema'; import { exactCheck } from '../../../exact_check'; import { foldLeftRight, getPaths } from '../../../test_utils'; @@ -26,6 +27,7 @@ import { getRulesSchemaMock, getRulesMlSchemaMock, getThreatMatchingSchemaMock, + getRulesEqlSchemaMock, } from './rules_schema.mocks'; import { ListArray } from '../types/lists'; @@ -628,6 +630,19 @@ describe('rules_schema', () => { ]); expect(message.schema).toEqual({}); }); + + test('it validates an eql rule response', () => { + const payload = getRulesEqlSchemaMock(); + + const dependents = getDependents(payload); + const decoded = dependents.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + const expected = getRulesEqlSchemaMock(); + + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(expected); + }); }); describe('addSavedId', () => { @@ -668,11 +683,6 @@ describe('rules_schema', () => { expect(fields.length).toEqual(2); }); - test('should return two fields for a rule of type "eql"', () => { - const fields = addQueryFields({ type: 'eql' }); - expect(fields.length).toEqual(2); - }); - test('should return two fields for a rule of type "threshold"', () => { const fields = addQueryFields({ type: 'threshold' }); expect(fields.length).toEqual(2); @@ -757,4 +767,17 @@ describe('rules_schema', () => { expect(fields.length).toEqual(5); }); }); + + describe('addEqlFields', () => { + test('should return empty array if type is not "eql"', () => { + const fields = addEqlFields({ type: 'query' }); + const expected: t.Mixed[] = []; + expect(fields).toEqual(expected); + }); + + test('should return 3 fields for a rule of type "eql"', () => { + const fields = addEqlFields({ type: 'eql' }); + expect(fields.length).toEqual(3); + }); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index 1c2254f9f8f09..908425a7496d0 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -16,6 +16,7 @@ import { anomaly_threshold, description, enabled, + event_category_override, false_positives, from, id, @@ -121,6 +122,9 @@ export const dependentRulesSchema = t.partial({ language, query, + // eql only fields + event_category_override, + // when type = saved_query, saved_id is required saved_id, @@ -219,9 +223,7 @@ export const addTimelineTitle = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mi }; export const addQueryFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { - if ( - ['eql', 'query', 'saved_query', 'threshold', 'threat_match'].includes(typeAndTimelineOnly.type) - ) { + if (['query', 'saved_query', 'threshold', 'threat_match'].includes(typeAndTimelineOnly.type)) { return [ t.exact(t.type({ query: dependentRulesSchema.props.query })), t.exact(t.type({ language: dependentRulesSchema.props.language })), @@ -255,6 +257,20 @@ export const addThresholdFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t. } }; +export const addEqlFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { + if (typeAndTimelineOnly.type === 'eql') { + return [ + t.exact( + t.partial({ event_category_override: dependentRulesSchema.props.event_category_override }) + ), + t.exact(t.type({ query: dependentRulesSchema.props.query })), + t.exact(t.type({ language: dependentRulesSchema.props.language })), + ]; + } else { + return []; + } +}; + export const addThreatMatchFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed[] => { if (typeAndTimelineOnly.type === 'threat_match') { return [ @@ -278,6 +294,7 @@ export const getDependents = (typeAndTimelineOnly: TypeAndTimelineOnly): t.Mixed ...addQueryFields(typeAndTimelineOnly), ...addMlFields(typeAndTimelineOnly), ...addThresholdFields(typeAndTimelineOnly), + ...addEqlFields(typeAndTimelineOnly), ...addThreatMatchFields(typeAndTimelineOnly), ]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index fb01f92255516..5d9cfb4bb4492 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -348,6 +348,7 @@ export const getResult = (): RuleAlertType => ({ description: 'Detecting root and admin users', ruleId: 'rule-1', index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + eventCategoryOverride: undefined, falsePositives: [], from: 'now-6m', immutable: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index cfce019910071..7255325358baf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -60,6 +60,16 @@ } } }, + "group": { + "properties": { + "id": { + "type": "keyword" + }, + "index": { + "type": "integer" + } + } + }, "rule": { "properties": { "id": { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts index dd887233c36a3..067a4352e1080 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts @@ -69,6 +69,7 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -153,6 +154,7 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 26ab89ad8ea7c..54df87ca17787 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { isEqlRule } from '../../../../../common/detection_engine/utils'; import { createRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/create_rules_type_dependents'; import { RuleAlertAction } from '../../../../../common/detection_engine/types'; import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; @@ -53,6 +54,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -94,7 +96,9 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void const query = !isMlRule(type) && queryOrUndefined == null ? '' : queryOrUndefined; const language = - !isMlRule(type) && languageOrUndefined == null ? 'kuery' : languageOrUndefined; + !isMlRule(type) && !isEqlRule(type) && languageOrUndefined == null + ? 'kuery' + : languageOrUndefined; // TODO: Fix these either with an is conversion or by better typing them within io-ts const actions: RuleAlertAction[] = actionsRest as RuleAlertAction[]; @@ -138,6 +142,7 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts index 0f5d0304f5ca0..4dbca5df0041c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -135,6 +135,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, immutable, @@ -194,6 +195,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable, @@ -242,6 +244,7 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP savedObjectsClient, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts index 5099cf5de958f..39bbe9ee686a4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts @@ -59,6 +59,7 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -119,6 +120,7 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts index 3b3efd2ed166d..879bd8d5b8a1d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts @@ -50,6 +50,7 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -117,6 +118,7 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts index 0e414e130849a..4df0773f86317 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts @@ -62,6 +62,7 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -127,6 +128,7 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts index 553d084b62633..ef698db008d80 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts @@ -52,6 +52,7 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query: queryOrUndefined, @@ -117,6 +118,7 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index 556ea209152e6..c75b32b614e07 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -114,6 +114,7 @@ export const transformAlertToRule = ( description: alert.params.description, enabled: alert.enabled, anomaly_threshold: alert.params.anomalyThreshold, + event_category_override: alert.params.eventCategoryOverride, false_positives: alert.params.falsePositives, filters: alert.params.filters, from: alert.params.from, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts index 95067e57868d1..a6034f3d7b7b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts @@ -14,6 +14,7 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ anomalyThreshold: undefined, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: 'user.name: root or user.name: admin', @@ -61,6 +62,7 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ anomalyThreshold: 55, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts index 9ed94cd7bff2e..3a311d03e3c89 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts @@ -17,6 +17,7 @@ export const createRules = async ({ buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, query, @@ -69,6 +70,7 @@ export const createRules = async ({ description, ruleId, index, + eventCategoryOverride, falsePositives, from, immutable, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts index 59e14dcffc3c0..38adc03c00d50 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts @@ -22,6 +22,7 @@ export const installPrepackagedRules = ( building_block_type: buildingBlockType, description, enabled, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -70,6 +71,7 @@ export const installPrepackagedRules = ( buildingBlockType, description, enabled, + eventCategoryOverride, falsePositives, from, immutable: true, // At the moment we force all prepackaged rules to be immutable diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts index cfb40056eb85d..aeb136a969aa1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts @@ -120,6 +120,7 @@ export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({ anomalyThreshold: undefined, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: 'user.name: root or user.name: admin', @@ -163,6 +164,7 @@ export const getPatchMlRulesOptionsMock = (): PatchRulesOptions => ({ anomalyThreshold: 55, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts index e0814647b4c39..852ff06bdc736 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts @@ -18,6 +18,7 @@ export const patchRules = async ({ buildingBlockType, savedObjectsClient, description, + eventCategoryOverride, falsePositives, enabled, query, @@ -62,6 +63,7 @@ export const patchRules = async ({ author, buildingBlockType, description, + eventCategoryOverride, falsePositives, query, language, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 6b851351f27f2..d688e1b338e21 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -84,6 +84,7 @@ import { TimestampOverrideOrUndefined, BuildingBlockTypeOrUndefined, RuleNameOverrideOrUndefined, + EventCategoryOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; import { ThreatIndexOrUndefined, @@ -187,6 +188,7 @@ export interface CreateRulesOptions { buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; enabled: Enabled; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositives; from: From; query: QueryOrUndefined; @@ -236,6 +238,7 @@ export interface UpdateRulesOptions { buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; enabled: Enabled; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositives; from: From; query: QueryOrUndefined; @@ -279,6 +282,7 @@ export interface PatchRulesOptions { buildingBlockType: BuildingBlockTypeOrUndefined; description: DescriptionOrUndefined; enabled: EnabledOrUndefined; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositivesOrUndefined; from: FromOrUndefined; query: QueryOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts index bf97784e8d917..01a481ed7b2d9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts @@ -23,6 +23,7 @@ export const updatePrepackagedRules = async ( author, building_block_type: buildingBlockType, description, + event_category_override: eventCategoryOverride, false_positives: falsePositives, from, query, @@ -69,6 +70,7 @@ export const updatePrepackagedRules = async ( author, buildingBlockType, description, + eventCategoryOverride, falsePositives, from, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts index 650b59fb85bc0..8cdc904a861c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts @@ -17,6 +17,7 @@ export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({ anomalyThreshold: undefined, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: 'user.name: root or user.name: admin', @@ -61,6 +62,7 @@ export const getUpdateMlRulesOptionsMock = (): UpdateRulesOptions => ({ anomalyThreshold: 55, description: 'some description', enabled: true, + eventCategoryOverride: undefined, falsePositives: ['false positive 1', 'false positive 2'], from: 'now-6m', query: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts index 494a4e221d862..08df785884b76 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts @@ -18,6 +18,7 @@ export const updateRules = async ({ buildingBlockType, savedObjectsClient, description, + eventCategoryOverride, falsePositives, enabled, query, @@ -64,6 +65,7 @@ export const updateRules = async ({ author, buildingBlockType, description, + eventCategoryOverride, falsePositives, query, language, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts index 17505a4478261..227f574bc4e4b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts @@ -31,6 +31,7 @@ describe('utils', () => { author: [], buildingBlockType: undefined, description: 'some description change', + eventCategoryOverride: undefined, falsePositives: undefined, query: undefined, language: undefined, @@ -73,6 +74,7 @@ describe('utils', () => { author: [], buildingBlockType: undefined, description: 'some description change', + eventCategoryOverride: undefined, falsePositives: undefined, query: undefined, language: undefined, @@ -115,6 +117,7 @@ describe('utils', () => { author: [], buildingBlockType: undefined, description: 'some description change', + eventCategoryOverride: undefined, falsePositives: undefined, query: undefined, language: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts index 49c02f92ff336..d9f953f2803a6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts @@ -39,6 +39,7 @@ import { RuleNameOverrideOrUndefined, SeverityMappingOrUndefined, TimestampOverrideOrUndefined, + EventCategoryOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; import { PartialFilter } from '../types'; import { ListArrayOrUndefined } from '../../../../common/detection_engine/schemas/types'; @@ -60,6 +61,7 @@ export interface UpdateProperties { author: AuthorOrUndefined; buildingBlockType: BuildingBlockTypeOrUndefined; description: DescriptionOrUndefined; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositivesOrUndefined; from: FromOrUndefined; query: QueryOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json new file mode 100644 index 0000000000000..598f2182002c1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_eql.json @@ -0,0 +1,68 @@ +{ + "name": "EQL query rule", + "description": "Rule with an eql query", + "false_positives": [ + "https://www.example.com/some-article-about-a-false-positive", + "some text string about why another condition could be a false positive" + ], + "rule_id": "rule-id-eql", + "enabled": false, + "index": [".ds-logs-endpoint.events.process-default-000001"], + "interval": "30s", + "query": "sequence [process where process.name = \"mimikatz.exe\"] [process where process.name = \"explorer.exe\"]", + "output_index": ".siem-signals-default", + "meta": { + "anything_you_want_ui_related_or_otherwise": { + "as_deep_structured_as_you_need": { + "any_data_type": {} + } + } + }, + "risk_score": 1, + "max_signals": 100, + "tags": ["tag 1", "tag 2", "any tag you want"], + "to": "now", + "from": "now-300m", + "severity": "high", + "type": "eql", + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0040", + "name": "impact", + "reference": "https://attack.mitre.org/tactics/TA0040/" + }, + "technique": [ + { + "id": "T1499", + "name": "endpoint denial of service", + "reference": "https://attack.mitre.org/techniques/T1499/" + } + ] + }, + { + "framework": "Some other Framework you want", + "tactic": { + "id": "some-other-id", + "name": "Some other name", + "reference": "https://example.com" + }, + "technique": [ + { + "id": "some-other-id", + "name": "some other technique name", + "reference": "https://example.com" + } + ] + } + ], + "references": [ + "http://www.example.com/some-article-about-attack", + "Some plain text string here explaining why this is a valid thing to look out for" + ], + "timeline_id": "timeline_id", + "timeline_title": "timeline_title", + "note": "# note markdown", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 9ee8c5cf298a1..b37bc7d0fab69 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -4,7 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalSearchResponse, BulkResponse, BulkItem } from '../types'; +import { + SignalSourceHit, + SignalSearchResponse, + BulkResponse, + BulkItem, + RuleAlertAttributes, + SignalHit, +} from '../types'; import { Logger, SavedObject, @@ -24,6 +31,7 @@ export const sampleRuleAlertParams = ( buildingBlockType: 'default', ruleId: 'rule-1', description: 'Detecting root and admin users', + eventCategoryOverride: undefined, falsePositives: [], immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], @@ -60,6 +68,30 @@ export const sampleRuleAlertParams = ( exceptionsList: getListArrayMock(), }); +export const sampleRuleSO = (): SavedObject => { + return { + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'alert', + version: '1', + updated_at: '2020-03-27T22:55:59.577Z', + attributes: { + actions: [], + enabled: true, + name: 'rule-name', + tags: ['some fake tag 1', 'some fake tag 2'], + createdBy: 'sample user', + createdAt: '2020-03-27T22:55:59.577Z', + updatedBy: 'sample user', + schedule: { + interval: '5m', + }, + throttle: 'no_actions', + params: sampleRuleAlertParams(), + }, + references: [], + }; +}; + export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): SignalSourceHit => ({ _index: 'myFakeSignalIndex', _type: 'doc', @@ -189,6 +221,80 @@ export const sampleDocWithAncestors = (): SignalSearchResponse => { }; }; +export const sampleSignalHit = (): SignalHit => ({ + '@timestamp': '2020-04-20T21:27:45+0000', + event: { + kind: 'signal', + }, + signal: { + parents: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + status: 'open', + rule: { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: '2020-04-20T21:27:45+0000', + updated_at: '2020-04-20T21:27:45+0000', + created_by: 'elastic', + description: 'some description', + enabled: true, + false_positives: ['false positive 1', 'false positive 2'], + from: 'now-6m', + immutable: false, + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + references: ['test 1', 'test 2'], + severity: 'high', + severity_mapping: [], + updated_by: 'elastic_kibana', + tags: ['some fake tag 1', 'some fake tag 2'], + to: 'now', + type: 'query', + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), + }, + depth: 1, + }, +}); + export const sampleBulkCreateDuplicateResult = { took: 60, errors: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 967dc5331e46b..f45a408cd32b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -9,8 +9,10 @@ import { sampleDocNoSortId, sampleRuleGuid, sampleIdGuid, + sampleDocWithAncestors, + sampleRuleSO, } from './__mocks__/es_results'; -import { buildBulkBody } from './build_bulk_body'; +import { buildBulkBody, buildSignalFromSequence, buildSignalFromEvent } from './build_bulk_body'; import { SignalHit } from './types'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; @@ -433,3 +435,200 @@ describe('buildBulkBody', () => { expect(fakeSignalSourceHit).toEqual(expected); }); }); + +describe('buildSignalFromSequence', () => { + test('builds a basic signal from a sequence of building blocks', () => { + const blocks = [sampleDocWithAncestors().hits.hits[0], sampleDocWithAncestors().hits.hits[0]]; + const ruleSO = sampleRuleSO(); + const signal = buildSignalFromSequence(blocks, ruleSO); + // Timestamp will potentially always be different so remove it for the test + // @ts-expect-error + delete signal['@timestamp']; + const expected: Omit = { + event: { + kind: 'signal', + }, + signal: { + parents: [ + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + ancestors: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 2, + group: { + id: '269c1f5754bff92fb8040283b687258e99b03e8b2ab1262cc20c82442e5de5ea', + }, + }, + }; + expect(signal).toEqual(expected); + }); +}); + +describe('buildSignalFromEvent', () => { + test('builds a basic signal from a single event', () => { + const ancestor = sampleDocWithAncestors().hits.hits[0]; + delete ancestor._source.source; + const ruleSO = sampleRuleSO(); + const signal = buildSignalFromEvent(ancestor, ruleSO); + // Timestamp will potentially always be different so remove it for the test + // @ts-expect-error + delete signal['@timestamp']; + const expected: Omit & { someKey: 'someValue' } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + original_time: '2020-04-20T21:27:45+0000', + parent: { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + parents: [ + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + ancestors: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: sampleIdGuid, + rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + type: 'signal', + index: 'myFakeSignalIndex', + depth: 1, + }, + ], + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 2, + }, + }; + expect(signal).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index 7be97e46f91f2..01a6b0e7aefad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -4,12 +4,22 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalHit, Signal } from './types'; -import { buildRule } from './build_rule'; +import { SavedObject } from 'src/core/types'; +import { + SignalSourceHit, + SignalHit, + Signal, + RuleAlertAttributes, + BaseSignalHit, + SignalSource, +} from './types'; +import { buildRule, buildRuleWithoutOverrides } from './build_rule'; import { additionalSignalFields, buildSignal } from './build_signal'; import { buildEventTypeSignal } from './build_event_type_signal'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; +import { generateSignalId, wrapBuildingBlocks, wrapSignal } from './utils'; +import { EqlSequence } from '../../types'; interface BuildBulkBodyParams { doc: SignalSourceHit; @@ -71,3 +81,86 @@ export const buildBulkBody = ({ }; return signalHit; }; + +/** + * Takes N raw documents from ES that form a sequence and builds them into N+1 signals ready to be indexed - + * one signal for each event in the sequence, and a "shell" signal that ties them all together. All N+1 signals + * share the same signal.group.id to make it easy to query them. + * @param sequence The raw ES documents that make up the sequence + * @param ruleSO SavedObject representing the rule that found the sequence + * @param outputIndex Index to write the resulting signals to + */ +export const buildSignalGroupFromSequence = ( + sequence: EqlSequence, + ruleSO: SavedObject, + outputIndex: string +): BaseSignalHit[] => { + const wrappedBuildingBlocks = wrapBuildingBlocks( + sequence.events.map((event) => { + const signal = buildSignalFromEvent(event, ruleSO); + signal.signal.rule.building_block_type = 'default'; + return signal; + }), + outputIndex + ); + + // Now that we have an array of building blocks for the events in the sequence, + // we can build the signal that links the building blocks together + // and also insert the group id (which is also the "shell" signal _id) in each building block + const sequenceSignal = wrapSignal( + buildSignalFromSequence(wrappedBuildingBlocks, ruleSO), + outputIndex + ); + wrappedBuildingBlocks.forEach((block, idx) => { + // TODO: fix type of blocks so we don't have to check existence of _source.signal + if (block._source.signal) { + block._source.signal.group = { + id: sequenceSignal._id, + index: idx, + }; + } + }); + return [...wrappedBuildingBlocks, sequenceSignal]; +}; + +export const buildSignalFromSequence = ( + events: BaseSignalHit[], + ruleSO: SavedObject +): SignalHit => { + const rule = buildRuleWithoutOverrides(ruleSO); + const signal: Signal = buildSignal(events, rule); + return { + '@timestamp': new Date().toISOString(), + event: { + kind: 'signal', + }, + signal: { + ...signal, + group: { + // This is the same function that is used later to generate the _id for the sequence signal document, + // so _id should equal signal.group.id for the "shell" document + id: generateSignalId(signal), + }, + }, + }; +}; + +export const buildSignalFromEvent = ( + event: BaseSignalHit, + ruleSO: SavedObject +): SignalHit => { + const rule = buildRuleWithoutOverrides(ruleSO); + const signal = { + ...buildSignal([event], rule), + ...additionalSignalFields(event), + }; + const eventFields = buildEventTypeSignal(event); + // TODO: better naming for SignalHit - it's really a new signal to be inserted + const signalHit: SignalHit = { + ...event._source, + '@timestamp': new Date().toISOString(), + event: eventFields, + signal, + }; + return signalHit; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts index 59cdc020c611d..81c9d1dedcc56 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit } from './types'; +import { BaseSignalHit } from './types'; -export const buildEventTypeSignal = (doc: SignalSourceHit): object => { +export const buildEventTypeSignal = (doc: BaseSignalHit): object => { if (doc._source.event != null && doc._source.event instanceof Object) { return { ...doc._source.event, kind: 'signal' }; } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index ba815a0b62f0d..62e5854037d9e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -4,12 +4,17 @@ * you may not use this file except in compliance with the Elastic License. */ -import { buildRule, removeInternalTagsFromRule } from './build_rule'; -import { sampleDocNoSortId, sampleRuleAlertParams, sampleRuleGuid } from './__mocks__/es_results'; +import { buildRule, removeInternalTagsFromRule, buildRuleWithoutOverrides } from './build_rule'; +import { + sampleDocNoSortId, + sampleRuleAlertParams, + sampleRuleGuid, + sampleRuleSO, +} from './__mocks__/es_results'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY } from '../../../../common/constants'; -import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { getRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; describe('buildRule', () => { beforeEach(() => { @@ -272,9 +277,11 @@ describe('buildRule', () => { }; expect(rule).toEqual(expected); }); +}); +describe('removeInternalTagsFromRule', () => { test('it removes internal tags from a typical rule', () => { - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); rule.tags = [ 'some fake tag 1', 'some fake tag 2', @@ -282,30 +289,113 @@ describe('buildRule', () => { `${INTERNAL_IMMUTABLE_KEY}:true`, ]; const noInternals = removeInternalTagsFromRule(rule); - expect(noInternals).toEqual(getPartialRulesSchemaMock()); + expect(noInternals).toEqual(getRulesSchemaMock()); }); test('it works with an empty array', () => { - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); rule.tags = []; const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); + const expected = getRulesSchemaMock(); expected.tags = []; expect(noInternals).toEqual(expected); }); - test('it works if tags does not exist', () => { - const rule = getPartialRulesSchemaMock(); - delete rule.tags; - const noInternals = removeInternalTagsFromRule(rule); - const expected = getPartialRulesSchemaMock(); - delete expected.tags; - expect(noInternals).toEqual(expected); - }); - test('it works if tags contains normal values and no internal values', () => { - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); const noInternals = removeInternalTagsFromRule(rule); expect(noInternals).toEqual(rule); }); }); + +describe('buildRuleWithoutOverrides', () => { + test('builds a rule using rule SO', () => { + const ruleSO = sampleRuleSO(); + const rule = buildRuleWithoutOverrides(ruleSO); + expect(rule).toEqual({ + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }); + }); + + test('builds a rule using rule SO and removes internal tags', () => { + const ruleSO = sampleRuleSO(); + ruleSO.attributes.tags = [ + 'some fake tag 1', + 'some fake tag 2', + `${INTERNAL_RULE_ID_KEY}:rule-1`, + `${INTERNAL_IMMUTABLE_KEY}:true`, + ]; + const rule = buildRuleWithoutOverrides(ruleSO); + expect(rule).toEqual({ + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'sample user', + updated_by: 'sample user', + version: 1, + updated_at: ruleSO.updated_at ?? '', + created_at: ruleSO.attributes.createdAt, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts index aacf9b8be31b4..e5370735333bc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts @@ -4,12 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { pickBy } from 'lodash/fp'; +import { SavedObject } from 'src/core/types'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams } from '../types'; import { buildRiskScoreFromMapping } from './mappings/build_risk_score_from_mapping'; -import { SignalSourceHit } from './types'; +import { SignalSourceHit, RuleAlertAttributes } from './types'; import { buildSeverityFromMapping } from './mappings/build_severity_from_mapping'; import { buildRuleNameFromMapping } from './mappings/build_rule_name_from_mapping'; import { INTERNAL_IDENTIFIER } from '../../../../common/constants'; @@ -44,7 +44,7 @@ export const buildRule = ({ interval, tags, throttle, -}: BuildRuleParams): Partial => { +}: BuildRuleParams): RulesSchema => { const { riskScore, riskScoreMeta } = buildRiskScoreFromMapping({ doc, riskScore: ruleParams.riskScore, @@ -65,7 +65,7 @@ export const buildRule = ({ const meta = { ...ruleParams.meta, ...riskScoreMeta, ...severityMeta, ...ruleNameMeta }; - const rule = pickBy((value: unknown) => value != null, { + const rule = { id, rule_id: ruleParams.ruleId ?? '(unknown rule_id)', actions, @@ -111,15 +111,73 @@ export const buildRule = ({ machine_learning_job_id: ruleParams.machineLearningJobId, anomaly_threshold: ruleParams.anomalyThreshold, threshold: ruleParams.threshold, - }); + }; + return removeInternalTagsFromRule(rule); +}; + +export const buildRuleWithoutOverrides = ( + ruleSO: SavedObject +): RulesSchema => { + const ruleParams = ruleSO.attributes.params; + const rule: RulesSchema = { + id: ruleSO.id, + rule_id: ruleParams.ruleId, + actions: ruleSO.attributes.actions, + author: ruleParams.author ?? [], + building_block_type: ruleParams.buildingBlockType, + false_positives: ruleParams.falsePositives, + saved_id: ruleParams.savedId, + timeline_id: ruleParams.timelineId, + timeline_title: ruleParams.timelineTitle, + meta: ruleParams.meta, + max_signals: ruleParams.maxSignals, + risk_score: ruleParams.riskScore, + risk_score_mapping: ruleParams.riskScoreMapping ?? [], + output_index: ruleParams.outputIndex, + description: ruleParams.description, + note: ruleParams.note, + from: ruleParams.from, + immutable: ruleParams.immutable, + index: ruleParams.index, + interval: ruleSO.attributes.schedule.interval, + language: ruleParams.language, + license: ruleParams.license, + name: ruleSO.attributes.name, + query: ruleParams.query, + references: ruleParams.references, + rule_name_override: ruleParams.ruleNameOverride, + severity: ruleParams.severity, + severity_mapping: ruleParams.severityMapping ?? [], + tags: ruleSO.attributes.tags, + type: ruleParams.type, + to: ruleParams.to, + enabled: ruleSO.attributes.enabled, + filters: ruleParams.filters, + created_by: ruleSO.attributes.createdBy, + updated_by: ruleSO.attributes.updatedBy, + threat: ruleParams.threat ?? [], + timestamp_override: ruleParams.timestampOverride, // TODO: Timestamp Override via timestamp_override + throttle: ruleSO.attributes.throttle, + version: ruleParams.version, + created_at: ruleSO.attributes.createdAt, + updated_at: ruleSO.updated_at ?? '', + exceptions_list: ruleParams.exceptionsList ?? [], + machine_learning_job_id: ruleParams.machineLearningJobId, + anomaly_threshold: ruleParams.anomalyThreshold, + threshold: ruleParams.threshold, + threat_filters: ruleParams.threatFilters, + threat_index: ruleParams.threatIndex, + threat_query: ruleParams.threatQuery, + threat_mapping: ruleParams.threatMapping, + }; return removeInternalTagsFromRule(rule); }; -export const removeInternalTagsFromRule = (rule: Partial): Partial => { +export const removeInternalTagsFromRule = (rule: RulesSchema): RulesSchema => { if (rule.tags == null) { return rule; } else { - const ruleWithoutInternalTags: Partial = { + const ruleWithoutInternalTags: RulesSchema = { ...rule, tags: rule.tags.filter((tag) => !tag.startsWith(INTERNAL_IDENTIFIER)), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index d684807a09126..d0c451bbdf2e2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -7,7 +7,11 @@ import { sampleDocNoSortId } from './__mocks__/es_results'; import { buildSignal, buildParent, buildAncestors, additionalSignalFields } from './build_signal'; import { Signal, Ancestor } from './types'; -import { getPartialRulesSchemaMock } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { + getRulesSchemaMock, + ANCHOR_DATE, +} from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; +import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; describe('buildSignal', () => { beforeEach(() => { @@ -17,7 +21,7 @@ describe('buildSignal', () => { test('it builds a signal as expected without original_event if event does not exist', () => { const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); delete doc._source.event; - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); const signal = { ...buildSignal([doc], rule), ...additionalSignalFields(doc), @@ -48,31 +52,39 @@ describe('buildSignal', () => { original_time: '2020-04-20T21:27:45+0000', status: 'open', rule: { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: new Date(ANCHOR_DATE).toISOString(), + updated_at: new Date(ANCHOR_DATE).toISOString(), created_by: 'elastic', - description: 'Detecting root and admin users', + description: 'some description', enabled: true, - false_positives: [], + false_positives: ['false positive 1', 'false positive 2'], from: 'now-6m', - id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', immutable: false, - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - interval: '5m', - risk_score: 50, - rule_id: 'rule-1', - language: 'kuery', - max_signals: 100, - name: 'Detect Root/Admin Users', - output_index: '.siem-signals', + name: 'Query with a rule id', query: 'user.name: root or user.name: admin', - references: ['http://www.example.com', 'https://ww.example.com'], + references: ['test 1', 'test 2'], severity: 'high', - updated_by: 'elastic', + severity_mapping: [], + updated_by: 'elastic_kibana', tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', - note: '', - updated_at: signal.rule.updated_at, - created_at: signal.rule.created_at, + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), }, depth: 1, }; @@ -87,7 +99,7 @@ describe('buildSignal', () => { kind: 'event', module: 'system', }; - const rule = getPartialRulesSchemaMock(); + const rule = getRulesSchemaMock(); const signal = { ...buildSignal([doc], rule), ...additionalSignalFields(doc), @@ -124,31 +136,39 @@ describe('buildSignal', () => { }, status: 'open', rule: { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: new Date(ANCHOR_DATE).toISOString(), + updated_at: new Date(ANCHOR_DATE).toISOString(), created_by: 'elastic', - description: 'Detecting root and admin users', + description: 'some description', enabled: true, - false_positives: [], + false_positives: ['false positive 1', 'false positive 2'], from: 'now-6m', - id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', immutable: false, - index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], - interval: '5m', - risk_score: 50, - rule_id: 'rule-1', - language: 'kuery', - max_signals: 100, - name: 'Detect Root/Admin Users', - output_index: '.siem-signals', + name: 'Query with a rule id', query: 'user.name: root or user.name: admin', - references: ['http://www.example.com', 'https://ww.example.com'], + references: ['test 1', 'test 2'], severity: 'high', - updated_by: 'elastic', + severity_mapping: [], + updated_by: 'elastic_kibana', tags: ['some fake tag 1', 'some fake tag 2'], to: 'now', type: 'query', - note: '', - updated_at: signal.rule.updated_at, - created_at: signal.rule.created_at, + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), }, depth: 1, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 78818779dd661..947938de6caca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -5,14 +5,14 @@ */ import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; -import { SignalSourceHit, Signal, Ancestor } from './types'; +import { Signal, Ancestor, BaseSignalHit } from './types'; /** * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child * signal's `signal.parents` array. * @param doc The parent signal or event */ -export const buildParent = (doc: SignalSourceHit): Ancestor => { +export const buildParent = (doc: BaseSignalHit): Ancestor => { if (doc._source.signal != null) { return { rule: doc._source.signal.rule.id, @@ -38,7 +38,7 @@ export const buildParent = (doc: SignalSourceHit): Ancestor => { * creating an array of N+1 ancestors. * @param doc The parent signal/event for which to extend the ancestry. */ -export const buildAncestors = (doc: SignalSourceHit): Ancestor[] => { +export const buildAncestors = (doc: BaseSignalHit): Ancestor[] => { const newAncestor = buildParent(doc); const existingAncestors = doc._source.signal?.ancestors; if (existingAncestors != null) { @@ -53,7 +53,7 @@ export const buildAncestors = (doc: SignalSourceHit): Ancestor[] => { * @param docs The parent signals/events of the new signal to be built. * @param rule The rule that is generating the new signal. */ -export const buildSignal = (docs: SignalSourceHit[], rule: Partial): Signal => { +export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => { const parents = docs.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; const ancestors = docs.reduce((acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), []); @@ -70,7 +70,7 @@ export const buildSignal = (docs: SignalSourceHit[], rule: Partial) * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. * @param doc The parent signal/event of the new signal to be built. */ -export const additionalSignalFields = (doc: SignalSourceHit) => { +export const additionalSignalFields = (doc: BaseSignalHit) => { return { parent: buildParent(doc), original_time: doc._source['@timestamp'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts index 6ce0be54a9e7b..522f4bfa5ef98 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.ts @@ -90,7 +90,6 @@ export const getFilter = async ({ }; switch (type) { - case 'eql': case 'threat_match': case 'threshold': { return savedId != null ? savedQueryFilter() : queryFilter(); @@ -106,6 +105,9 @@ export const getFilter = async ({ 'Unsupported Rule of type "machine_learning" supplied to getFilter' ); } + case 'eql': { + throw new BadRequestError('Unsupported Rule of type "eql" supplied to getFilter'); + } default: { return assertUnreachable(type); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts index c8f8341392553..922fadb13a298 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.mock.ts @@ -22,6 +22,7 @@ export const getSignalParamsSchemaDecodedMock = (): SignalParamsSchema => ({ author: [], buildingBlockType: null, description: 'Detecting root and admin users', + eventCategoryOverride: undefined, falsePositives: [], filters: null, from: 'now-6m', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index dbb48d59d3a3f..4006345b24385 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -14,6 +14,7 @@ const signalSchema = schema.object({ buildingBlockType: schema.nullable(schema.string()), description: schema.string(), note: schema.nullable(schema.string()), + eventCategoryOverride: schema.maybe(schema.string()), falsePositives: schema.arrayOf(schema.string(), { defaultValue: [] }), from: schema.string(), ruleId: schema.string(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 97ab12f905358..f7b56f42755ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -24,13 +24,19 @@ import { SetupPlugins } from '../../../plugin'; import { getInputIndex } from './get_input_output_index'; import { searchAfterAndBulkCreate } from './search_after_bulk_create'; import { getFilter } from './get_filter'; -import { SignalRuleAlertTypeDefinition, RuleAlertAttributes } from './types'; +import { + SignalRuleAlertTypeDefinition, + RuleAlertAttributes, + EqlSignalSearchResponse, + BaseSignalHit, +} from './types'; import { getGapBetweenRuns, getListsClient, getExceptions, getGapMaxCatchupRatio, MAX_RULE_GAP_RATIO, + wrapSignal, createErrorsFromShard, createSearchAfterReturnType, mergeReturns, @@ -50,6 +56,9 @@ import { ruleStatusServiceFactory } from './rule_status_service'; import { buildRuleMessageFactory } from './rule_messages'; import { ruleStatusSavedObjectsClientFactory } from './rule_status_saved_objects_client'; import { getNotificationResultsLink } from '../notifications/utils'; +import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_query_filter'; +import { bulkInsertSignals } from './single_bulk_create'; +import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk_body'; import { createThreatSignals } from './threat_mapping/create_threat_signals'; export const signalRulesAlertType = ({ @@ -265,8 +274,6 @@ export const signalRulesAlertType = ({ bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], }), ]); - } else if (isEqlRule(type)) { - throw new Error('EQL Rules are under development, execution is not yet implemented'); } else if (isThresholdRule(type) && threshold) { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ @@ -378,7 +385,7 @@ export const signalRulesAlertType = ({ buildRuleMessage, threatIndex, }); - } else { + } else if (type === 'query' || type === 'saved_query') { const inputIndex = await getInputIndex(services, version, index); const esFilter = await getFilter({ type, @@ -417,6 +424,51 @@ export const signalRulesAlertType = ({ throttle, buildRuleMessage, }); + } else if (isEqlRule(type)) { + if (query === undefined) { + throw new Error('eql query rule must have a query defined'); + } + const inputIndex = await getInputIndex(services, version, index); + const request = buildEqlSearchRequest( + query, + inputIndex, + params.from, + params.to, + searchAfterSize, + params.timestampOverride, + exceptionItems ?? [], + params.eventCategoryOverride + ); + const response: EqlSignalSearchResponse = await services.callCluster( + 'transport.request', + request + ); + let newSignals: BaseSignalHit[] | undefined; + if (response.hits.sequences !== undefined) { + newSignals = response.hits.sequences.reduce( + (acc: BaseSignalHit[], sequence) => + acc.concat(buildSignalGroupFromSequence(sequence, savedObject, outputIndex)), + [] + ); + } else if (response.hits.events !== undefined) { + newSignals = response.hits.events.map((event) => + wrapSignal(buildSignalFromEvent(event, savedObject), outputIndex) + ); + } else { + throw new Error( + 'eql query response should have either `sequences` or `events` but had neither' + ); + } + // TODO: replace with code that filters out recursive rule signals while allowing sequences and their building blocks + // const filteredSignals = filterDuplicateSignals(alertId, newSignals); + if (newSignals.length > 0) { + const insertResult = await bulkInsertSignals(newSignals, logger, services, refresh); + result.bulkCreateTimes.push(insertResult.bulkCreateDuration); + result.createdSignalsCount += insertResult.createdItemsCount; + } + result.success = true; + } else { + throw new Error(`unknown rule type ${type}`); } if (result.success) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index e8f254e6a8966..e3c3c940b3225 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -7,7 +7,7 @@ import { countBy, isEmpty } from 'lodash'; import { performance } from 'perf_hooks'; import { AlertServices } from '../../../../../alerts/server'; -import { SignalSearchResponse, BulkResponse } from './types'; +import { SignalSearchResponse, BulkResponse, SignalHit, BaseSignalHit } from './types'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { generateId, makeFloatString, errorAggregator } from './utils'; @@ -59,6 +59,18 @@ export const filterDuplicateRules = ( }); }; +/** + * Similar to filterDuplicateRules, but operates on candidate signal documents rather than events that matched + * the detection query. This means we only have to compare the ruleId against the ancestors array. + * @param ruleId The rule id + * @param signals The candidate new signals + */ +export const filterDuplicateSignals = (ruleId: string, signals: SignalHit[]) => { + return signals.filter( + (doc) => !doc.signal.ancestors.some((ancestor) => ancestor.rule === ruleId) + ); +}; + export interface SingleBulkCreateResponse { success: boolean; bulkCreateDuration?: string; @@ -66,6 +78,11 @@ export interface SingleBulkCreateResponse { errors: string[]; } +export interface BulkInsertSignalsResponse { + bulkCreateDuration: string; + createdItemsCount: number; +} + // Bulk Index documents. export const singleBulkCreate = async ({ filteredEvents, @@ -167,3 +184,46 @@ export const singleBulkCreate = async ({ }; } }; + +// Bulk Index new signals. +export const bulkInsertSignals = async ( + signals: BaseSignalHit[], + logger: Logger, + services: AlertServices, + refresh: RefreshTypes +): Promise => { + // index documents after creating an ID based on the + // id and index of each parent and the rule ID + const bulkBody = signals.flatMap((doc) => [ + { + create: { + _index: doc._index, + _id: doc._id, + }, + }, + doc._source, + ]); + const start = performance.now(); + const response: BulkResponse = await services.callCluster('bulk', { + refresh, + body: bulkBody, + }); + const end = performance.now(); + logger.debug(`individual bulk process time took: ${makeFloatString(end - start)} milliseconds`); + logger.debug(`took property says bulk took: ${response.took} milliseconds`); + + if (response.errors) { + const duplicateSignalsCount = countBy(response.items, 'create.status')['409']; + logger.debug(`ignored ${duplicateSignalsCount} duplicate signals`); + const errorCountByMessage = errorAggregator(response, [409]); + if (!isEmpty(errorCountByMessage)) { + logger.error( + `[-] bulkResponse had errors with responses of: ${JSON.stringify(errorCountByMessage)}` + ); + } + } + + const createdItemsCount = countBy(response.items, 'create.status')['201'] ?? 0; + logger.debug(`bulk created ${createdItemsCount} signals`); + return { bulkCreateDuration: makeFloatString(end - start), createdItemsCount }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 6ebdca0764e9d..2f6ed0c1e3a8e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -16,7 +16,7 @@ import { } from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; -import { SearchResponse } from '../../types'; +import { SearchResponse, EqlSearchResponse, BaseHit } from '../../types'; import { ListClient } from '../../../../../lists/server'; import { Logger } from '../../../../../../../src/core/server'; import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; @@ -53,6 +53,8 @@ export type SearchTypes = export interface SignalSource { [key: string]: SearchTypes; + // TODO: SignalSource is being used as the type for documents matching detection engine queries, but they may not + // actually have @timestamp if a timestamp override is used '@timestamp': string; signal?: { // parent is deprecated: new signals should populate parents instead @@ -60,6 +62,10 @@ export interface SignalSource { parent?: Ancestor; parents?: Ancestor[]; ancestors: Ancestor[]; + group?: { + id: string; + index?: number; + }; rule: { id: string; }; @@ -116,6 +122,9 @@ export interface GetResponse { export type EventSearchResponse = SearchResponse; export type SignalSearchResponse = SearchResponse; export type SignalSourceHit = SignalSearchResponse['hits']['hits'][number]; +export type BaseSignalHit = BaseHit; + +export type EqlSignalSearchResponse = EqlSearchResponse; export type RuleExecutorOptions = Omit & { params: RuleTypeParams; @@ -140,11 +149,15 @@ export interface Ancestor { } export interface Signal { - rule: Partial; + rule: RulesSchema; // DEPRECATED: use parents instead of parent parent?: Ancestor; parents: Ancestor[]; ancestors: Ancestor[]; + group?: { + id: string; + index?: number; + }; original_time?: string; original_event?: SearchTypes; status: Status; @@ -155,7 +168,7 @@ export interface Signal { export interface SignalHit { '@timestamp': string; event: object; - signal: Partial; + signal: Signal; } export interface AlertAttributes { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 97f3dbeaf4489..14e12b2ea4632 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -25,6 +25,8 @@ import { getListsClient, getSignalTimeTuples, getExceptions, + wrapBuildingBlocks, + generateSignalId, createErrorsFromShard, createSearchAfterReturnTypeFromResponse, createSearchAfterReturnType, @@ -38,6 +40,7 @@ import { sampleBulkError, sampleBulkErrorItem, mockLogger, + sampleSignalHit, sampleDocSearchResultsWithSortId, sampleEmptyDocSearchResults, sampleDocSearchResultsNoSortIdNoHits, @@ -794,6 +797,56 @@ describe('utils', () => { }); }); + describe('wrapBuildingBlocks', () => { + it('should generate a unique id for each building block', () => { + const wrappedBlocks = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + const blockIds: string[] = []; + wrappedBlocks.forEach((block) => { + expect(blockIds.includes(block._id)).toEqual(false); + blockIds.push(block._id); + }); + }); + + it('should generate different ids for identical documents in different sequences', () => { + const wrappedBlockSequence1 = wrapBuildingBlocks([sampleSignalHit()], 'test-index'); + const wrappedBlockSequence2 = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + const blockId = wrappedBlockSequence1[0]._id; + wrappedBlockSequence2.forEach((block) => { + expect(block._id).not.toEqual(blockId); + }); + }); + + it('should generate the same ids when given the same sequence twice', () => { + const wrappedBlockSequence1 = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + const wrappedBlockSequence2 = wrapBuildingBlocks( + [sampleSignalHit(), sampleSignalHit()], + 'test-index' + ); + wrappedBlockSequence1.forEach((block, idx) => { + expect(block._id).toEqual(wrappedBlockSequence2[idx]._id); + }); + }); + }); + + describe('generateSignalId', () => { + it('generates a unique signal id for same signal with different rule id', () => { + const signalId1 = generateSignalId(sampleSignalHit().signal); + const modifiedSignal = sampleSignalHit(); + modifiedSignal.signal.rule.id = 'some other rule id'; + const signalIdModified = generateSignalId(modifiedSignal.signal); + expect(signalId1).not.toEqual(signalIdModified); + }); + }); + describe('createErrorsFromShard', () => { test('empty errors will return an empty array', () => { const createdErrors = createErrorsFromShard({ errors: [] }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 2eabc03dccad7..53089b7f1ca2b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -16,8 +16,11 @@ import { BulkResponse, BulkResponseErrorAggregation, isValidUnit, + SignalHit, + BaseSignalHit, SearchAfterAndBulkCreateReturnType, SignalSearchResponse, + Signal, } from './types'; import { BuildRuleMessage } from './rule_messages'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; @@ -212,6 +215,60 @@ export const generateId = ( ruleId: string ): string => createHash('sha256').update(docIndex.concat(docId, version, ruleId)).digest('hex'); +// TODO: do we need to include version in the id? If it does matter then we should include it in signal.parents as well +export const generateSignalId = (signal: Signal) => + createHash('sha256') + .update( + signal.parents + .reduce((acc, parent) => acc.concat(parent.id, parent.index), '') + .concat(signal.rule.id) + ) + .digest('hex'); + +/** + * Generates unique doc ids for each building block signal within a sequence. The id of each building block + * depends on the parents of every building block, so that a signal which appears in multiple different sequences + * (e.g. if multiple rules build sequences that share a common event/signal) will get a unique id per sequence. + * @param buildingBlocks The full list of building blocks in the sequence. + */ +export const generateBuildingBlockIds = (buildingBlocks: SignalHit[]): string[] => { + const baseHashString = buildingBlocks.reduce( + (baseString, block) => + baseString + .concat( + block.signal.parents.reduce((acc, parent) => acc.concat(parent.id, parent.index), '') + ) + .concat(block.signal.rule.id), + '' + ); + return buildingBlocks.map((block, idx) => + createHash('sha256').update(baseHashString).update(String(idx)).digest('hex') + ); +}; + +export const wrapBuildingBlocks = (buildingBlocks: SignalHit[], index: string): BaseSignalHit[] => { + const blockIds = generateBuildingBlockIds(buildingBlocks); + return buildingBlocks.map((block, idx) => { + return { + _id: blockIds[idx], + _index: index, + _source: { + ...block, + }, + }; + }); +}; + +export const wrapSignal = (signal: SignalHit, index: string): BaseSignalHit => { + return { + _id: generateSignalId(signal.signal), + _index: index, + _source: { + ...signal, + }, + }; +}; + export const parseInterval = (intervalString: string): moment.Duration | null => { try { return moment.duration(parseDuration(intervalString)); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts index b0554adcc46b0..728f5b1dd867f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts @@ -37,6 +37,7 @@ import { SeverityMappingOrUndefined, TimestampOverrideOrUndefined, Type, + EventCategoryOverrideOrUndefined, } from '../../../common/detection_engine/schemas/common/schemas'; import { ThreatIndexOrUndefined, @@ -56,6 +57,7 @@ export interface RuleTypeParams { buildingBlockType: BuildingBlockTypeOrUndefined; description: Description; note: NoteOrUndefined; + eventCategoryOverride: EventCategoryOverrideOrUndefined; falsePositives: FalsePositives; from: From; ruleId: RuleId; diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts index 7e59280cd1358..117cffd844cfb 100644 --- a/x-pack/plugins/security_solution/server/lib/types.ts +++ b/x-pack/plugins/security_solution/server/lib/types.ts @@ -16,6 +16,7 @@ import { Sources } from './sources'; import { Note } from './note/saved_object'; import { PinnedEvent } from './pinned_event/saved_object'; import { Timeline } from './timeline/saved_object'; +import { SearchTypes } from './detection_engine/signals/types'; export * from './hosts'; @@ -44,6 +45,12 @@ export interface TotalValue { relation: string; } +export interface BaseHit { + _index: string; + _id: string; + _source: T; +} + export interface SearchResponse { took: number; timed_out: boolean; @@ -52,27 +59,43 @@ export interface SearchResponse { hits: { total: TotalValue | number; max_score: number; - hits: Array<{ - _index: string; - _type: string; - _id: string; - _score: number; - _source: T; - _version?: number; - _explanation?: Explanation; - fields?: string[]; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - highlight?: any; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - inner_hits?: any; - matched_queries?: string[]; - sort?: string[]; - }>; + hits: Array< + BaseHit & { + _type: string; + _score: number; + _version?: number; + _explanation?: Explanation; + fields?: string[]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + highlight?: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + inner_hits?: any; + matched_queries?: string[]; + sort?: string[]; + } + >; }; // eslint-disable-next-line @typescript-eslint/no-explicit-any aggregations?: any; } +export interface EqlSequence { + join_keys: SearchTypes[]; + events: Array>; +} + +export interface EqlSearchResponse { + is_partial: boolean; + is_running: boolean; + took: number; + timed_out: boolean; + hits: { + total: TotalValue; + sequences?: Array>; + events?: Array>; + }; +} + export interface ShardsResponse { total: number; successful: number; From 3f6c0d688cf38d5b20db788b7a51231ebd95e7c6 Mon Sep 17 00:00:00 2001 From: Daniil Suleiman <31325372+sulemanof@users.noreply.github.com> Date: Mon, 28 Sep 2020 12:43:34 +0300 Subject: [PATCH 04/11] Lazy load metric & mardown visualizations (#78391) * Lazy load metrics vis * Use common chart spinner * Simplify markdown renderer * Update tests * Update types for metric vis * Fix tests * Fix merge conflict Co-authored-by: Elastic Machine --- src/plugins/vis_type_markdown/kibana.json | 2 +- .../__snapshots__/markdown_fn.test.ts.snap | 2 +- .../vis_type_markdown/public/index.scss | 8 -- .../vis_type_markdown/public/markdown_fn.ts | 8 +- .../public/markdown_renderer.tsx | 50 ++++------ .../{_markdown_vis.scss => markdown_vis.scss} | 7 ++ .../public/markdown_vis_controller.test.tsx | 46 ++-------- .../public/markdown_vis_controller.tsx | 92 +++++-------------- .../vis_type_markdown/public/plugin.ts | 8 +- .../metric_vis.scss} | 7 ++ .../components/metric_vis_component.test.tsx | 2 +- .../components/metric_vis_component.tsx | 10 +- src/plugins/vis_type_metric/public/index.scss | 8 -- src/plugins/vis_type_metric/public/index.ts | 1 - .../vis_type_metric/public/metric_vis_fn.ts | 4 +- .../public/metric_vis_renderer.tsx | 34 +++---- src/plugins/vis_type_metric/public/plugin.ts | 3 +- .../vis_type_metric/public/services.ts | 3 - .../public/components/_visualization.scss | 7 ++ .../components/visualization_container.tsx | 29 +++++- 20 files changed, 132 insertions(+), 199 deletions(-) delete mode 100644 src/plugins/vis_type_markdown/public/index.scss rename src/plugins/vis_type_markdown/public/{_markdown_vis.scss => markdown_vis.scss} (55%) rename src/plugins/vis_type_metric/public/{_metric_vis.scss => components/metric_vis.scss} (78%) delete mode 100644 src/plugins/vis_type_metric/public/index.scss diff --git a/src/plugins/vis_type_markdown/kibana.json b/src/plugins/vis_type_markdown/kibana.json index 5723fdefe1e4c..c0afcb0e99d13 100644 --- a/src/plugins/vis_type_markdown/kibana.json +++ b/src/plugins/vis_type_markdown/kibana.json @@ -4,5 +4,5 @@ "ui": true, "server": true, "requiredPlugins": ["expressions", "visualizations"], - "requiredBundles": ["kibanaUtils", "kibanaReact", "charts", "visualizations", "expressions", "visDefaultEditor"] + "requiredBundles": ["kibanaReact", "charts", "visualizations", "expressions", "visDefaultEditor"] } diff --git a/src/plugins/vis_type_markdown/public/__snapshots__/markdown_fn.test.ts.snap b/src/plugins/vis_type_markdown/public/__snapshots__/markdown_fn.test.ts.snap index 473e2cba742b7..9983f67d4be4d 100644 --- a/src/plugins/vis_type_markdown/public/__snapshots__/markdown_fn.test.ts.snap +++ b/src/plugins/vis_type_markdown/public/__snapshots__/markdown_fn.test.ts.snap @@ -5,7 +5,7 @@ Object { "as": "markdown_vis", "type": "render", "value": Object { - "visConfig": Object { + "visParams": Object { "fontSize": 12, "markdown": "## hello _markdown_", "openLinksInNewTab": true, diff --git a/src/plugins/vis_type_markdown/public/index.scss b/src/plugins/vis_type_markdown/public/index.scss deleted file mode 100644 index ddb7fe3a6b0d9..0000000000000 --- a/src/plugins/vis_type_markdown/public/index.scss +++ /dev/null @@ -1,8 +0,0 @@ -// Prefix all styles with "mkd" to avoid conflicts. -// Examples -// mkdChart -// mkdChart__legend -// mkdChart__legend--small -// mkdChart__legend-isLoading - -@import './markdown_vis'; diff --git a/src/plugins/vis_type_markdown/public/markdown_fn.ts b/src/plugins/vis_type_markdown/public/markdown_fn.ts index 4b3c9989431f9..eaa2c840f8046 100644 --- a/src/plugins/vis_type_markdown/public/markdown_fn.ts +++ b/src/plugins/vis_type_markdown/public/markdown_fn.ts @@ -21,16 +21,16 @@ import { i18n } from '@kbn/i18n'; import { ExpressionFunctionDefinition, Render } from '../../expressions/public'; import { Arguments, MarkdownVisParams } from './types'; -interface RenderValue { +export interface MarkdownVisRenderValue { visType: 'markdown'; - visConfig: MarkdownVisParams; + visParams: MarkdownVisParams; } export type MarkdownVisExpressionFunctionDefinition = ExpressionFunctionDefinition< 'markdownVis', unknown, Arguments, - Render + Render >; export const createMarkdownVisFn = (): MarkdownVisExpressionFunctionDefinition => ({ @@ -70,7 +70,7 @@ export const createMarkdownVisFn = (): MarkdownVisExpressionFunctionDefinition = as: 'markdown_vis', value: { visType: 'markdown', - visConfig: { + visParams: { markdown: args.markdown, openLinksInNewTab: args.openLinksInNewTab, fontSize: parseInt(args.font.spec.fontSize || '12', 10), diff --git a/src/plugins/vis_type_markdown/public/markdown_renderer.tsx b/src/plugins/vis_type_markdown/public/markdown_renderer.tsx index 5950a762635b2..8071196c6a213 100644 --- a/src/plugins/vis_type_markdown/public/markdown_renderer.tsx +++ b/src/plugins/vis_type_markdown/public/markdown_renderer.tsx @@ -17,41 +17,29 @@ * under the License. */ -import React from 'react'; +import React, { lazy } from 'react'; import { render, unmountComponentAtNode } from 'react-dom'; import { VisualizationContainer } from '../../visualizations/public'; import { ExpressionRenderDefinition } from '../../expressions/common/expression_renderers'; -import { MarkdownVisWrapper } from './markdown_vis_controller'; -import { StartServicesGetter } from '../../kibana_utils/public'; +import { MarkdownVisRenderValue } from './markdown_fn'; -export const getMarkdownRenderer = (start: StartServicesGetter) => { - const markdownVisRenderer: () => ExpressionRenderDefinition = () => ({ - name: 'markdown_vis', - displayName: 'markdown visualization', - reuseDomNode: true, - render: async (domNode: HTMLElement, config: any, handlers: any) => { - const { visConfig } = config; +// @ts-ignore +const MarkdownVisComponent = lazy(() => import('./markdown_vis_controller')); - const I18nContext = await start().core.i18n.Context; +export const markdownVisRenderer: ExpressionRenderDefinition = { + name: 'markdown_vis', + displayName: 'markdown visualization', + reuseDomNode: true, + render: async (domNode, { visParams }, handlers) => { + handlers.onDestroy(() => { + unmountComponentAtNode(domNode); + }); - handlers.onDestroy(() => { - unmountComponentAtNode(domNode); - }); - - render( - - - - - , - domNode - ); - }, - }); - - return markdownVisRenderer; + render( + + + , + domNode + ); + }, }; diff --git a/src/plugins/vis_type_markdown/public/_markdown_vis.scss b/src/plugins/vis_type_markdown/public/markdown_vis.scss similarity index 55% rename from src/plugins/vis_type_markdown/public/_markdown_vis.scss rename to src/plugins/vis_type_markdown/public/markdown_vis.scss index fb0a3d05e5e85..2356562a86ed0 100644 --- a/src/plugins/vis_type_markdown/public/_markdown_vis.scss +++ b/src/plugins/vis_type_markdown/public/markdown_vis.scss @@ -1,3 +1,10 @@ +// Prefix all styles with "mkd" to avoid conflicts. +// Examples +// mkdChart +// mkdChart__legend +// mkdChart__legend--small +// mkdChart__legend-isLoading + .mkdVis { padding: $euiSizeS; width: 100%; diff --git a/src/plugins/vis_type_markdown/public/markdown_vis_controller.test.tsx b/src/plugins/vis_type_markdown/public/markdown_vis_controller.test.tsx index 6df205b21d910..36850fc820ded 100644 --- a/src/plugins/vis_type_markdown/public/markdown_vis_controller.test.tsx +++ b/src/plugins/vis_type_markdown/public/markdown_vis_controller.test.tsx @@ -20,7 +20,7 @@ import React from 'react'; import { wait } from '@testing-library/dom'; import { render, cleanup } from '@testing-library/react/pure'; -import { MarkdownVisWrapper } from './markdown_vis_controller'; +import MarkdownVisComponent from './markdown_vis_controller'; afterEach(cleanup); @@ -36,7 +36,7 @@ describe('markdown vis controller', () => { }; const { getByTestId, getByText } = render( - + ); await wait(() => getByTestId('markdownBody')); @@ -60,7 +60,7 @@ describe('markdown vis controller', () => { }; const { getByTestId, getByText } = render( - + ); await wait(() => getByTestId('markdownBody')); @@ -82,7 +82,7 @@ describe('markdown vis controller', () => { }; const { getByTestId, getByText, rerender } = render( - + ); await wait(() => getByTestId('markdownBody')); @@ -90,9 +90,7 @@ describe('markdown vis controller', () => { expect(getByText(/initial/i)).toBeInTheDocument(); vis.params.markdown = 'Updated'; - rerender( - - ); + rerender(); expect(getByText(/Updated/i)).toBeInTheDocument(); }); @@ -114,11 +112,7 @@ describe('markdown vis controller', () => { it('should be called on initial rendering', async () => { const { getByTestId } = render( - + ); await wait(() => getByTestId('markdownBody')); @@ -128,11 +122,7 @@ describe('markdown vis controller', () => { it('should be called on successive render when params change', async () => { const { getByTestId, rerender } = render( - + ); await wait(() => getByTestId('markdownBody')); @@ -142,24 +132,14 @@ describe('markdown vis controller', () => { renderComplete.mockClear(); vis.params.markdown = 'changed'; - rerender( - - ); + rerender(); expect(renderComplete).toHaveBeenCalledTimes(1); }); it('should be called on successive render even without data change', async () => { const { getByTestId, rerender } = render( - + ); await wait(() => getByTestId('markdownBody')); @@ -168,13 +148,7 @@ describe('markdown vis controller', () => { renderComplete.mockClear(); - rerender( - - ); + rerender(); expect(renderComplete).toHaveBeenCalledTimes(1); }); diff --git a/src/plugins/vis_type_markdown/public/markdown_vis_controller.tsx b/src/plugins/vis_type_markdown/public/markdown_vis_controller.tsx index e1155ca42df72..a2387b96eab6d 100644 --- a/src/plugins/vis_type_markdown/public/markdown_vis_controller.tsx +++ b/src/plugins/vis_type_markdown/public/markdown_vis_controller.tsx @@ -17,83 +17,35 @@ * under the License. */ -import React from 'react'; +import React, { useEffect } from 'react'; import { Markdown } from '../../kibana_react/public'; import { MarkdownVisParams } from './types'; +import './markdown_vis.scss'; + interface MarkdownVisComponentProps extends MarkdownVisParams { renderComplete: () => void; } -/** - * The MarkdownVisComponent renders markdown to HTML and presents it. - */ -class MarkdownVisComponent extends React.Component { - /** - * Will be called after the first render when the component is present in the DOM. - * - * We call renderComplete here, to signal, that we are done with rendering. - */ - componentDidMount() { - this.props.renderComplete(); - } - - /** - * Will be called after the component has been updated and the changes has been - * flushed into the DOM. - * - * We will use this to signal that we are done rendering by calling the - * renderComplete property. - */ - componentDidUpdate() { - this.props.renderComplete(); - } +const MarkdownVisComponent = ({ + fontSize, + markdown, + openLinksInNewTab, + renderComplete, +}: MarkdownVisComponentProps) => { + useEffect(renderComplete); // renderComplete will be called after each render to signal, that we are done with rendering. - /** - * Render the actual HTML. - * Note: if only fontSize parameter has changed, this method will be called - * and return the appropriate JSX, but React will detect, that only the - * style argument has been updated, and thus only set this attribute to the DOM. - */ - render() { - return ( -
- -
- ); - } -} - -/** - * This is a wrapper component, that is actually used as the visualization. - * The sole purpose of this component is to extract all required parameters from - * the properties and pass them down as separate properties to the actual component. - * That way the actual (MarkdownVisComponent) will properly trigger it's prop update - * callback (componentWillReceiveProps) if one of these params change. It wouldn't - * trigger otherwise (e.g. it doesn't for this wrapper), since it only triggers - * if the reference to the prop changes (in this case the reference to vis). - * - * The way React works, this wrapper nearly brings no overhead, but allows us - * to use proper lifecycle methods in the actual component. - */ - -export interface MarkdownVisWrapperProps { - visParams: MarkdownVisParams; - fireEvent: (event: any) => void; - renderComplete: () => void; -} - -export function MarkdownVisWrapper(props: MarkdownVisWrapperProps) { return ( - +
+ +
); -} +}; + +// default export required for React.Lazy +// eslint-disable-next-line import/no-default-export +export { MarkdownVisComponent as default }; diff --git a/src/plugins/vis_type_markdown/public/plugin.ts b/src/plugins/vis_type_markdown/public/plugin.ts index c117df7e0fa33..790b19876d366 100644 --- a/src/plugins/vis_type_markdown/public/plugin.ts +++ b/src/plugins/vis_type_markdown/public/plugin.ts @@ -24,10 +24,7 @@ import { VisualizationsSetup } from '../../visualizations/public'; import { markdownVisDefinition } from './markdown_vis'; import { createMarkdownVisFn } from './markdown_fn'; import { ConfigSchema } from '../config'; - -import './index.scss'; -import { getMarkdownRenderer } from './markdown_renderer'; -import { createStartServicesGetter } from '../../kibana_utils/public'; +import { markdownVisRenderer } from './markdown_renderer'; /** @internal */ export interface MarkdownPluginSetupDependencies { @@ -44,9 +41,8 @@ export class MarkdownPlugin implements Plugin { } public setup(core: CoreSetup, { expressions, visualizations }: MarkdownPluginSetupDependencies) { - const start = createStartServicesGetter(core.getStartServices); visualizations.createBaseVisualization(markdownVisDefinition); - expressions.registerRenderer(getMarkdownRenderer(start)); + expressions.registerRenderer(markdownVisRenderer); expressions.registerFunction(createMarkdownVisFn); } diff --git a/src/plugins/vis_type_metric/public/_metric_vis.scss b/src/plugins/vis_type_metric/public/components/metric_vis.scss similarity index 78% rename from src/plugins/vis_type_metric/public/_metric_vis.scss rename to src/plugins/vis_type_metric/public/components/metric_vis.scss index b1f04cc93c4b7..5665ba8e8d099 100644 --- a/src/plugins/vis_type_metric/public/_metric_vis.scss +++ b/src/plugins/vis_type_metric/public/components/metric_vis.scss @@ -1,3 +1,10 @@ +// Prefix all styles with "mtr" to avoid conflicts. +// Examples +// mtrChart +// mtrChart__legend +// mtrChart__legend--small +// mtrChart__legend-isLoading + .mtrVis { width: 100%; display: flex; diff --git a/src/plugins/vis_type_metric/public/components/metric_vis_component.test.tsx b/src/plugins/vis_type_metric/public/components/metric_vis_component.test.tsx index b56d4e4f62e41..7f82c6adb5694 100644 --- a/src/plugins/vis_type_metric/public/components/metric_vis_component.test.tsx +++ b/src/plugins/vis_type_metric/public/components/metric_vis_component.test.tsx @@ -20,7 +20,7 @@ import React from 'react'; import { shallow } from 'enzyme'; -import { MetricVisComponent, MetricVisComponentProps } from './metric_vis_component'; +import MetricVisComponent, { MetricVisComponentProps } from './metric_vis_component'; jest.mock('../services', () => ({ getFormatService: () => ({ diff --git a/src/plugins/vis_type_metric/public/components/metric_vis_component.tsx b/src/plugins/vis_type_metric/public/components/metric_vis_component.tsx index 9ce3820ee4e23..e5c7db65c09a8 100644 --- a/src/plugins/vis_type_metric/public/components/metric_vis_component.tsx +++ b/src/plugins/vis_type_metric/public/components/metric_vis_component.tsx @@ -30,14 +30,16 @@ import { getFormatService } from '../services'; import { SchemaConfig } from '../../../visualizations/public'; import { Range } from '../../../expressions/public'; +import './metric_vis.scss'; + export interface MetricVisComponentProps { - visParams: VisParams; + visParams: Pick; visData: Input; fireEvent: (event: any) => void; renderComplete: () => void; } -export class MetricVisComponent extends Component { +class MetricVisComponent extends Component { private getLabels() { const config = this.props.visParams.metric; const isPercentageMode = config.percentageMode; @@ -209,3 +211,7 @@ export class MetricVisComponent extends Component { return metricsHtml; } } + +// default export required for React.Lazy +// eslint-disable-next-line import/no-default-export +export { MetricVisComponent as default }; diff --git a/src/plugins/vis_type_metric/public/index.scss b/src/plugins/vis_type_metric/public/index.scss deleted file mode 100644 index 638f9ac1ef93a..0000000000000 --- a/src/plugins/vis_type_metric/public/index.scss +++ /dev/null @@ -1,8 +0,0 @@ -// Prefix all styles with "mtr" to avoid conflicts. -// Examples -// mtrChart -// mtrChart__legend -// mtrChart__legend--small -// mtrChart__legend-isLoading - -@import 'metric_vis'; diff --git a/src/plugins/vis_type_metric/public/index.ts b/src/plugins/vis_type_metric/public/index.ts index 3d3e1879a51d9..ac541a9577cfc 100644 --- a/src/plugins/vis_type_metric/public/index.ts +++ b/src/plugins/vis_type_metric/public/index.ts @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -import './index.scss'; import { PluginInitializerContext } from 'kibana/public'; import { MetricVisPlugin as Plugin } from './plugin'; diff --git a/src/plugins/vis_type_metric/public/metric_vis_fn.ts b/src/plugins/vis_type_metric/public/metric_vis_fn.ts index b58be63581724..97b1e6822333e 100644 --- a/src/plugins/vis_type_metric/public/metric_vis_fn.ts +++ b/src/plugins/vis_type_metric/public/metric_vis_fn.ts @@ -46,7 +46,7 @@ interface Arguments { bucket: any; // these aren't typed yet } -interface RenderValue { +export interface MetricVisRenderValue { visType: typeof visType; visData: Input; visConfig: Pick; @@ -57,7 +57,7 @@ export type MetricVisExpressionFunctionDefinition = ExpressionFunctionDefinition 'metricVis', Input, Arguments, - Render + Render >; export const createMetricVisFn = (): MetricVisExpressionFunctionDefinition => ({ diff --git a/src/plugins/vis_type_metric/public/metric_vis_renderer.tsx b/src/plugins/vis_type_metric/public/metric_vis_renderer.tsx index 2bae668b080ea..bf0d6da9fba05 100644 --- a/src/plugins/vis_type_metric/public/metric_vis_renderer.tsx +++ b/src/plugins/vis_type_metric/public/metric_vis_renderer.tsx @@ -17,37 +17,33 @@ * under the License. */ -import React from 'react'; +import React, { lazy } from 'react'; import { render, unmountComponentAtNode } from 'react-dom'; -import { MetricVisComponent } from './components/metric_vis_component'; -import { getI18n } from './services'; + import { VisualizationContainer } from '../../visualizations/public'; import { ExpressionRenderDefinition } from '../../expressions/common/expression_renderers'; +import { MetricVisRenderValue } from './metric_vis_fn'; +// @ts-ignore +const MetricVisComponent = lazy(() => import('./components/metric_vis_component')); -export const metricVisRenderer: () => ExpressionRenderDefinition = () => ({ +export const metricVisRenderer: () => ExpressionRenderDefinition = () => ({ name: 'metric_vis', displayName: 'metric visualization', reuseDomNode: true, - render: async (domNode: HTMLElement, config: any, handlers: any) => { - const { visData, visConfig } = config; - - const I18nContext = getI18n().Context; - + render: async (domNode, { visData, visConfig }, handlers) => { handlers.onDestroy(() => { unmountComponentAtNode(domNode); }); render( - - - - - , + + + , domNode ); }, diff --git a/src/plugins/vis_type_metric/public/plugin.ts b/src/plugins/vis_type_metric/public/plugin.ts index b9e094aa76889..c653d1bdaf965 100644 --- a/src/plugins/vis_type_metric/public/plugin.ts +++ b/src/plugins/vis_type_metric/public/plugin.ts @@ -25,7 +25,7 @@ import { createMetricVisFn } from './metric_vis_fn'; import { createMetricVisTypeDefinition } from './metric_vis_type'; import { ChartsPluginSetup } from '../../charts/public'; import { DataPublicPluginStart } from '../../data/public'; -import { setFormatService, setI18n } from './services'; +import { setFormatService } from './services'; import { ConfigSchema } from '../config'; import { metricVisRenderer } from './metric_vis_renderer'; @@ -59,7 +59,6 @@ export class MetricVisPlugin implements Plugin { } public start(core: CoreStart, { data }: MetricVisPluginStartDependencies) { - setI18n(core.i18n); setFormatService(data.fieldFormats); } } diff --git a/src/plugins/vis_type_metric/public/services.ts b/src/plugins/vis_type_metric/public/services.ts index 0e19cfdce228d..681afbaf0b268 100644 --- a/src/plugins/vis_type_metric/public/services.ts +++ b/src/plugins/vis_type_metric/public/services.ts @@ -17,12 +17,9 @@ * under the License. */ -import { I18nStart } from 'kibana/public'; import { createGetterSetter } from '../../kibana_utils/common'; import { DataPublicPluginStart } from '../../data/public'; export const [getFormatService, setFormatService] = createGetterSetter< DataPublicPluginStart['fieldFormats'] >('metric data.fieldFormats'); - -export const [getI18n, setI18n] = createGetterSetter('I18n'); diff --git a/src/plugins/visualizations/public/components/_visualization.scss b/src/plugins/visualizations/public/components/_visualization.scss index 785968d2883e7..f5e2d4fcf2862 100644 --- a/src/plugins/visualizations/public/components/_visualization.scss +++ b/src/plugins/visualizations/public/components/_visualization.scss @@ -70,3 +70,10 @@ flex-direction: column; } +.visChart__spinner { + display: flex; + flex: 1 1 auto; + justify-content: center; + align-items: center; +} + diff --git a/src/plugins/visualizations/public/components/visualization_container.tsx b/src/plugins/visualizations/public/components/visualization_container.tsx index d6f87d4cea123..007a9e6e9dde4 100644 --- a/src/plugins/visualizations/public/components/visualization_container.tsx +++ b/src/plugins/visualizations/public/components/visualization_container.tsx @@ -17,14 +17,35 @@ * under the License. */ -import React, { ReactNode } from 'react'; +import React, { ReactNode, Suspense } from 'react'; +import { EuiLoadingChart } from '@elastic/eui'; +import classNames from 'classnames'; +import { VisualizationNoResults } from './visualization_noresults'; interface VisualizationContainerProps { className?: string; children: ReactNode; + showNoResult?: boolean; } -export const VisualizationContainer = (props: VisualizationContainerProps) => { - const classes = `visualization ${props.className}`; - return
{props.children}
; +export const VisualizationContainer = ({ + className, + children, + showNoResult = false, +}: VisualizationContainerProps) => { + const classes = classNames('visualization', className); + + const fallBack = ( +
+ +
+ ); + + return ( +
+ + {showNoResult ? : children} + +
+ ); }; From a71d0693dc45d1e9eb186eb59557107c85f1f393 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Mon, 28 Sep 2020 14:33:48 +0200 Subject: [PATCH 05/11] TypeScript cleanup in visualizations plugin (#78428) Co-authored-by: Elastic Machine --- .../public/input_control_vis_type.ts | 5 ++++- .../input_control_vis/public/vis_controller.tsx | 12 ++++++------ .../vis_type_metric/public/metric_vis_type.ts | 3 ++- .../public/table_vis_controller.test.ts | 2 +- .../vis_type_table/public/table_vis_type.ts | 8 +++++--- .../vis_type_table/public/vis_controller.ts | 2 +- .../vis_type_vislib/public/vis_controller.tsx | 2 +- src/plugins/visualizations/public/index.ts | 2 +- src/plugins/visualizations/public/types.ts | 9 +++++++-- .../public/vis_types/base_vis_type.ts | 16 +++++++++++++--- .../visualizations/public/vis_types/index.ts | 2 ++ .../public/vis_types/react_vis_controller.tsx | 15 +++++---------- .../public/vis_types/react_vis_type.ts | 8 +++++++- .../public/vis_types/types_service.ts | 14 ++++++-------- 14 files changed, 61 insertions(+), 39 deletions(-) diff --git a/src/plugins/input_control_vis/public/input_control_vis_type.ts b/src/plugins/input_control_vis/public/input_control_vis_type.ts index 9f415f2100004..782df67f5c58a 100644 --- a/src/plugins/input_control_vis/public/input_control_vis_type.ts +++ b/src/plugins/input_control_vis/public/input_control_vis_type.ts @@ -19,12 +19,15 @@ import { i18n } from '@kbn/i18n'; +import { BaseVisTypeOptions } from 'src/plugins/visualizations/public'; import { createInputControlVisController } from './vis_controller'; import { getControlsTab } from './components/editor/controls_tab'; import { OptionsTab } from './components/editor/options_tab'; import { InputControlVisDependencies } from './plugin'; -export function createInputControlVisTypeDefinition(deps: InputControlVisDependencies) { +export function createInputControlVisTypeDefinition( + deps: InputControlVisDependencies +): BaseVisTypeOptions { const InputControlVisController = createInputControlVisController(deps); const ControlsTab = getControlsTab(deps); diff --git a/src/plugins/input_control_vis/public/vis_controller.tsx b/src/plugins/input_control_vis/public/vis_controller.tsx index faea98b792291..6f35e17866120 100644 --- a/src/plugins/input_control_vis/public/vis_controller.tsx +++ b/src/plugins/input_control_vis/public/vis_controller.tsx @@ -31,12 +31,12 @@ import { RangeControl } from './control/range_control_factory'; import { ListControl } from './control/list_control_factory'; import { InputControlVisDependencies } from './plugin'; import { FilterManager, Filter } from '../../data/public'; -import { VisParams, Vis } from '../../visualizations/public'; +import { VisParams, ExprVis } from '../../visualizations/public'; export const createInputControlVisController = (deps: InputControlVisDependencies) => { return class InputControlVisController { private I18nContext?: I18nStart['Context']; - private isLoaded = false; + private _isLoaded = false; controls: Array; queryBarUpdateHandler: () => void; @@ -45,7 +45,7 @@ export const createInputControlVisController = (deps: InputControlVisDependencie timeFilterSubscription: Subscription; visParams?: VisParams; - constructor(public el: Element, public vis: Vis) { + constructor(public el: Element, public vis: ExprVis) { this.controls = []; this.queryBarUpdateHandler = this.updateControlsFromKbn.bind(this); @@ -58,7 +58,7 @@ export const createInputControlVisController = (deps: InputControlVisDependencie .getTimeUpdate$() .subscribe(() => { if (this.visParams?.useTimeFilter) { - this.isLoaded = false; + this._isLoaded = false; } }); } @@ -68,11 +68,11 @@ export const createInputControlVisController = (deps: InputControlVisDependencie const [{ i18n }] = await deps.core.getStartServices(); this.I18nContext = i18n.Context; } - if (!this.isLoaded || !isEqual(visParams, this.visParams)) { + if (!this._isLoaded || !isEqual(visParams, this.visParams)) { this.visParams = visParams; this.controls = []; this.controls = await this.initControls(); - this.isLoaded = true; + this._isLoaded = true; } this.drawVis(); } diff --git a/src/plugins/vis_type_metric/public/metric_vis_type.ts b/src/plugins/vis_type_metric/public/metric_vis_type.ts index 6b4d6e151693f..1c5afd396c2c3 100644 --- a/src/plugins/vis_type_metric/public/metric_vis_type.ts +++ b/src/plugins/vis_type_metric/public/metric_vis_type.ts @@ -18,13 +18,14 @@ */ import { i18n } from '@kbn/i18n'; +import { BaseVisTypeOptions } from 'src/plugins/visualizations/public'; import { MetricVisOptions } from './components/metric_vis_options'; import { ColorSchemas, colorSchemas, ColorModes } from '../../charts/public'; import { AggGroupNames } from '../../data/public'; import { Schemas } from '../../vis_default_editor/public'; import { toExpressionAst } from './to_ast'; -export const createMetricVisTypeDefinition = () => ({ +export const createMetricVisTypeDefinition = (): BaseVisTypeOptions => ({ name: 'metric', title: i18n.translate('visTypeMetric.metricTitle', { defaultMessage: 'Metric' }), icon: 'visMetric', diff --git a/src/plugins/vis_type_table/public/table_vis_controller.test.ts b/src/plugins/vis_type_table/public/table_vis_controller.test.ts index 56d17c187bd3f..7535e98d391c6 100644 --- a/src/plugins/vis_type_table/public/table_vis_controller.test.ts +++ b/src/plugins/vis_type_table/public/table_vis_controller.test.ts @@ -121,7 +121,7 @@ describe('Table Vis - Controller', () => { function getRangeVis(params?: object) { return ({ type: tableVisTypeDefinition, - params: Object.assign({}, tableVisTypeDefinition.visConfig.defaults, params), + params: Object.assign({}, tableVisTypeDefinition.visConfig?.defaults, params), data: { aggs: createAggConfigs(stubIndexPattern, [ { type: 'count', schema: 'metric' }, diff --git a/src/plugins/vis_type_table/public/table_vis_type.ts b/src/plugins/vis_type_table/public/table_vis_type.ts index 80d53021b7866..c1419a4847458 100644 --- a/src/plugins/vis_type_table/public/table_vis_type.ts +++ b/src/plugins/vis_type_table/public/table_vis_type.ts @@ -20,7 +20,7 @@ import { CoreSetup, PluginInitializerContext } from 'kibana/public'; import { i18n } from '@kbn/i18n'; import { AggGroupNames } from '../../data/public'; import { Schemas } from '../../vis_default_editor/public'; -import { Vis } from '../../visualizations/public'; +import { BaseVisTypeOptions, Vis } from '../../visualizations/public'; import { tableVisResponseHandler } from './table_vis_response_handler'; // @ts-ignore import tableVisTemplate from './table_vis.html'; @@ -28,9 +28,11 @@ import { TableOptions } from './components/table_vis_options_lazy'; import { getTableVisualizationControllerClass } from './vis_controller'; import { VIS_EVENT_TO_TRIGGER } from '../../../plugins/visualizations/public'; -export function getTableVisTypeDefinition(core: CoreSetup, context: PluginInitializerContext) { +export function getTableVisTypeDefinition( + core: CoreSetup, + context: PluginInitializerContext +): BaseVisTypeOptions { return { - type: 'table', name: 'table', title: i18n.translate('visTypeTable.tableVisTitle', { defaultMessage: 'Data Table', diff --git a/src/plugins/vis_type_table/public/vis_controller.ts b/src/plugins/vis_type_table/public/vis_controller.ts index d87812b9f5d69..5e82796e66339 100644 --- a/src/plugins/vis_type_table/public/vis_controller.ts +++ b/src/plugins/vis_type_table/public/vis_controller.ts @@ -64,7 +64,7 @@ export function getTableVisualizationControllerClass( } } - async render(esResponse: object, visParams: VisParams) { + async render(esResponse: object, visParams: VisParams): Promise { getKibanaLegacy().loadFontAwesome(); await this.initLocalAngular(); diff --git a/src/plugins/vis_type_vislib/public/vis_controller.tsx b/src/plugins/vis_type_vislib/public/vis_controller.tsx index 86ef98de045d7..c422e9f4f3a0a 100644 --- a/src/plugins/vis_type_vislib/public/vis_controller.tsx +++ b/src/plugins/vis_type_vislib/public/vis_controller.tsx @@ -68,7 +68,7 @@ export const createVislibVisController = (deps: VisTypeVislibDependencies) => { this.container.appendChild(this.legendEl); } - render(esResponse: any, visParams: VisParams) { + render(esResponse: any, visParams: VisParams): Promise { if (this.vislibVis) { this.destroy(); } diff --git a/src/plugins/visualizations/public/index.ts b/src/plugins/visualizations/public/index.ts index 17c292a1b183b..3e3926bc5c8d8 100644 --- a/src/plugins/visualizations/public/index.ts +++ b/src/plugins/visualizations/public/index.ts @@ -36,7 +36,7 @@ export { getSchemas as getVisSchemas } from './legacy/build_pipeline'; /** @public types */ export { VisualizationsSetup, VisualizationsStart }; -export { VisTypeAlias, VisType } from './vis_types'; +export { VisTypeAlias, VisType, BaseVisTypeOptions, ReactVisTypeOptions } from './vis_types'; export { VisParams, SerializedVis, SerializedVisData, VisData } from './vis'; export type VisualizeEmbeddableFactoryContract = PublicContract; export type VisualizeEmbeddableContract = PublicContract; diff --git a/src/plugins/visualizations/public/types.ts b/src/plugins/visualizations/public/types.ts index f47ffbbe921a2..897a8c1e32319 100644 --- a/src/plugins/visualizations/public/types.ts +++ b/src/plugins/visualizations/public/types.ts @@ -17,6 +17,7 @@ * under the License. */ +import { ExpressionAstExpression } from 'src/plugins/expressions'; import { SavedObject } from '../../../plugins/saved_objects/public'; import { AggConfigOptions, @@ -24,6 +25,7 @@ import { TimefilterContract, } from '../../../plugins/data/public'; import { SerializedVis, Vis, VisParams } from './vis'; +import { ExprVis } from './expressions/vis'; export { Vis, SerializedVis, VisParams }; @@ -35,7 +37,7 @@ export interface VisualizationController { export type VisualizationControllerConstructor = new ( el: HTMLElement, - vis: Vis + vis: ExprVis ) => VisualizationController; export interface SavedVisState { @@ -71,4 +73,7 @@ export interface VisToExpressionAstParams { abortSignal?: AbortSignal; } -export type VisToExpressionAst = (vis: Vis, params: VisToExpressionAstParams) => string; +export type VisToExpressionAst = ( + vis: Vis, + params: VisToExpressionAstParams +) => ExpressionAstExpression; diff --git a/src/plugins/visualizations/public/vis_types/base_vis_type.ts b/src/plugins/visualizations/public/vis_types/base_vis_type.ts index fa0bbfc5e250a..283286648ff16 100644 --- a/src/plugins/visualizations/public/vis_types/base_vis_type.ts +++ b/src/plugins/visualizations/public/vis_types/base_vis_type.ts @@ -22,7 +22,7 @@ import { VisToExpressionAst, VisualizationControllerConstructor } from '../types import { TriggerContextMapping } from '../../../ui_actions/public'; import { Adapters } from '../../../inspector/public'; -export interface BaseVisTypeOptions { +interface CommonBaseVisTypeOptions { name: string; title: string; description?: string; @@ -31,7 +31,6 @@ export interface BaseVisTypeOptions { image?: string; stage?: 'experimental' | 'beta' | 'production'; options?: Record; - visualization: VisualizationControllerConstructor | undefined; visConfig?: Record; editor?: any; editorConfig?: Record; @@ -42,9 +41,20 @@ export interface BaseVisTypeOptions { setup?: unknown; useCustomNoDataScreen?: boolean; inspectorAdapters?: Adapters | (() => Adapters); - toExpressionAst?: VisToExpressionAst; } +interface ExpressionBaseVisTypeOptions extends CommonBaseVisTypeOptions { + toExpressionAst: VisToExpressionAst; + visualization?: undefined; +} + +interface VisualizationBaseVisTypeOptions extends CommonBaseVisTypeOptions { + toExpressionAst?: undefined; + visualization: VisualizationControllerConstructor | undefined; +} + +export type BaseVisTypeOptions = ExpressionBaseVisTypeOptions | VisualizationBaseVisTypeOptions; + export class BaseVisType { name: string; title: string; diff --git a/src/plugins/visualizations/public/vis_types/index.ts b/src/plugins/visualizations/public/vis_types/index.ts index 625c8be414c74..8f38e33569162 100644 --- a/src/plugins/visualizations/public/vis_types/index.ts +++ b/src/plugins/visualizations/public/vis_types/index.ts @@ -18,3 +18,5 @@ */ export * from './types_service'; +export type { BaseVisTypeOptions } from './base_vis_type'; +export type { ReactVisTypeOptions } from './react_vis_type'; diff --git a/src/plugins/visualizations/public/vis_types/react_vis_controller.tsx b/src/plugins/visualizations/public/vis_types/react_vis_controller.tsx index 643e6ffcb730b..ceb6435dce27e 100644 --- a/src/plugins/visualizations/public/vis_types/react_vis_controller.tsx +++ b/src/plugins/visualizations/public/vis_types/react_vis_controller.tsx @@ -19,17 +19,12 @@ import React from 'react'; import { render, unmountComponentAtNode } from 'react-dom'; -import { Vis, VisualizationController } from '../types'; +import { VisualizationController } from '../types'; import { getI18n, getUISettings } from '../services'; +import { ExprVis } from '../expressions/vis'; export class ReactVisController implements VisualizationController { - private el: HTMLElement; - private vis: Vis; - - constructor(element: HTMLElement, vis: Vis) { - this.el = element; - this.vis = vis; - } + constructor(private element: HTMLElement, private vis: ExprVis) {} public render(visData: any, visParams: any): Promise { const I18nContext = getI18n().Context; @@ -51,12 +46,12 @@ export class ReactVisController implements VisualizationController { renderComplete={resolve} /> , - this.el + this.element ); }); } public destroy() { - unmountComponentAtNode(this.el); + unmountComponentAtNode(this.element); } } diff --git a/src/plugins/visualizations/public/vis_types/react_vis_type.ts b/src/plugins/visualizations/public/vis_types/react_vis_type.ts index 68979abe52a3c..047d36d804111 100644 --- a/src/plugins/visualizations/public/vis_types/react_vis_type.ts +++ b/src/plugins/visualizations/public/vis_types/react_vis_type.ts @@ -20,8 +20,14 @@ import { BaseVisType, BaseVisTypeOptions } from './base_vis_type'; import { ReactVisController } from './react_vis_controller'; +export type ReactVisTypeOptions = Omit; + +/** + * This class should only be used for visualizations not using the `toExpressionAst` with a custom renderer. + * If you implement a custom renderer you should just mount a react component inside this. + */ export class ReactVisType extends BaseVisType { - constructor(opts: Omit) { + constructor(opts: ReactVisTypeOptions) { super({ ...opts, visualization: ReactVisController, diff --git a/src/plugins/visualizations/public/vis_types/types_service.ts b/src/plugins/visualizations/public/vis_types/types_service.ts index 14c2a9c50ab0e..157dbd41ce8a2 100644 --- a/src/plugins/visualizations/public/vis_types/types_service.ts +++ b/src/plugins/visualizations/public/vis_types/types_service.ts @@ -19,10 +19,8 @@ import { IconType } from '@elastic/eui'; import { visTypeAliasRegistry, VisTypeAlias } from './vis_type_alias_registry'; -// @ts-ignore -import { BaseVisType } from './base_vis_type'; -// @ts-ignore -import { ReactVisType } from './react_vis_type'; +import { BaseVisType, BaseVisTypeOptions } from './base_vis_type'; +import { ReactVisType, ReactVisTypeOptions } from './react_vis_type'; import { TriggerContextMapping } from '../../../ui_actions/public'; export interface VisType { @@ -71,17 +69,17 @@ export class TypesService { return { /** * registers a visualization type - * @param {VisType} config - visualization type definition + * @param config - visualization type definition */ - createBaseVisualization: (config: any) => { + createBaseVisualization: (config: BaseVisTypeOptions): void => { const vis = new BaseVisType(config); registerVisualization(() => vis); }, /** * registers a visualization which uses react for rendering - * @param {VisType} config - visualization type definition + * @param config - visualization type definition */ - createReactVisualization: (config: any) => { + createReactVisualization: (config: ReactVisTypeOptions): void => { const vis = new ReactVisType(config); registerVisualization(() => vis); }, From 88df93bed56ea02c9a84f78eac32c9653829c5eb Mon Sep 17 00:00:00 2001 From: Alison Goryachev Date: Mon, 28 Sep 2020 08:47:05 -0400 Subject: [PATCH 06/11] [Ingest pipelines] Upload indexed document to test a pipeline (#77939) --- .../helpers/setup_environment.tsx | 5 + .../__jest__/http_requests.helpers.ts | 12 + .../__jest__/test_pipeline.helpers.tsx | 23 ++ .../__jest__/test_pipeline.test.tsx | 86 ++++++- .../test_pipeline_flyout.container.tsx | 7 + .../test_pipeline/test_pipeline_flyout.tsx | 18 +- .../add_document_form.tsx | 209 ++++++++++++++++++ .../add_documents_accordion.scss | 4 + .../add_documents_accordion.tsx | 111 ++++++++++ .../add_documents_accordion/index.ts | 7 + .../documents_schema.tsx | 24 ++ .../tab_documents.tsx | 179 +++++++++------ .../context/test_pipeline_context.tsx | 8 +- .../use_is_mounted.ts | 21 ++ .../public/application/index.tsx | 2 + .../application/mount_management_section.ts | 6 +- .../public/application/services/api.ts | 9 + .../plugins/ingest_pipelines/public/plugin.ts | 7 +- .../ingest_pipelines/public/shared_imports.ts | 2 + .../plugins/ingest_pipelines/public/types.ts | 8 +- .../ingest_pipelines/public/url_generator.ts | 6 +- .../server/routes/api/documents.ts | 56 +++++ .../server/routes/api/index.ts | 2 + .../ingest_pipelines/server/routes/index.ts | 2 + .../ingest_pipelines/ingest_pipelines.ts | 62 +++++- .../ingest_pipelines/lib/elasticsearch.ts | 10 + 26 files changed, 783 insertions(+), 103 deletions(-) create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_document_form.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.scss create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/index.ts create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/use_is_mounted.ts create mode 100644 x-pack/plugins/ingest_pipelines/server/routes/api/documents.ts diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx index c380032bd9482..d9a0ac4115389 100644 --- a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx +++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx @@ -44,6 +44,11 @@ const appServices = { api: apiService, notifications: notificationServiceMock.createSetupContract(), history, + urlGenerators: { + getUrlGenerator: jest.fn().mockReturnValue({ + createUrl: jest.fn(), + }), + }, }; export const setupEnvironment = () => { diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/http_requests.helpers.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/http_requests.helpers.ts index 541a6853a99b3..c89b07ae0192f 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/http_requests.helpers.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/http_requests.helpers.ts @@ -21,8 +21,20 @@ const registerHttpRequestMockHelpers = (server: SinonFakeServer) => { ]); }; + const setFetchDocumentsResponse = (response?: HttpResponse, error?: any) => { + const status = error ? error.status || 400 : 200; + const body = error ? JSON.stringify(error.body) : JSON.stringify(response); + + server.respondWith('GET', '/api/ingest_pipelines/documents/:index/:id', [ + status, + { 'Content-Type': 'application/json' }, + body, + ]); + }; + return { setSimulatePipelineResponse, + setFetchDocumentsResponse, }; }; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.helpers.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.helpers.tsx index f4c89d7a1058a..215ef63d9782e 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.helpers.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.helpers.tsx @@ -94,6 +94,11 @@ const appServices = { notifications: notificationServiceMock.createSetupContract(), history, uiSettings: {}, + urlGenerators: { + getUrlGenerator: jest.fn().mockReturnValue({ + createUrl: jest.fn(), + }), + }, }; const testBedSetup = registerTestBed( @@ -180,6 +185,20 @@ const createActions = (testBed: TestBed) => { }); component.update(); }, + + async toggleDocumentsAccordion() { + await act(async () => { + find('addDocumentsAccordion').simulate('click'); + }); + component.update(); + }, + + async clickAddDocumentButton() { + await act(async () => { + find('addDocumentButton').simulate('click'); + }); + component.update(); + }, }; }; @@ -229,4 +248,8 @@ type TestSubject = | 'configurationTab' | 'outputTab' | 'processorOutputTabContent' + | 'addDocumentsAccordion' + | 'addDocumentButton' + | 'addDocumentError' + | 'addDocumentSuccess' | string; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.test.tsx index e5118a6e465af..47f05602799e4 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.test.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/__jest__/test_pipeline.test.tsx @@ -141,10 +141,9 @@ describe('Test pipeline', () => { const { actions, find, exists } = testBed; const error = { - status: 400, - error: 'Bad Request', - message: - '"[parse_exception] [_source] required property is missing, with { property_name="_source" }"', + status: 500, + error: 'Internal server error', + message: 'Internal server error', }; httpRequestsMockHelpers.setSimulatePipelineResponse(undefined, { body: error }); @@ -153,13 +152,90 @@ describe('Test pipeline', () => { actions.clickAddDocumentsButton(); // Add invalid sample documents array and run the pipeline - actions.addDocumentsJson(JSON.stringify([{}])); + actions.addDocumentsJson( + JSON.stringify([ + { + _index: 'test', + _id: '1', + _version: 1, + _seq_no: 0, + _primary_term: 1, + _source: { + name: 'John Doe', + }, + }, + ]) + ); await actions.clickRunPipelineButton(); // Verify error rendered expect(exists('pipelineExecutionError')).toBe(true); expect(find('pipelineExecutionError').text()).toContain(error.message); }); + + describe('Add indexed documents', () => { + test('should successfully add an indexed document', async () => { + const { actions, form, exists } = testBed; + + const { _index: index, _id: documentId } = DOCUMENTS[0]; + + httpRequestsMockHelpers.setFetchDocumentsResponse(DOCUMENTS[0]); + + // Open flyout + actions.clickAddDocumentsButton(); + + // Open documents accordion, click run without required fields, and verify error messages + await actions.toggleDocumentsAccordion(); + await actions.clickAddDocumentButton(); + expect(form.getErrorsMessages()).toEqual([ + 'An index name is required.', + 'A document ID is required.', + ]); + + // Add required fields, and click run + form.setInputValue('indexField.input', index); + form.setInputValue('idField.input', documentId); + await actions.clickAddDocumentButton(); + + // Verify request + const latestRequest = server.requests[server.requests.length - 1]; + expect(latestRequest.status).toEqual(200); + expect(latestRequest.url).toEqual(`/api/ingest_pipelines/documents/${index}/${documentId}`); + // Verify success callout + expect(exists('addDocumentSuccess')).toBe(true); + }); + + test('should surface API errors from the request', async () => { + const { actions, form, exists, find } = testBed; + + const nonExistentDoc = { + index: 'foo', + id: '1', + }; + + const error = { + status: 404, + error: 'Not found', + message: '[index_not_found_exception] no such index', + }; + + httpRequestsMockHelpers.setFetchDocumentsResponse(undefined, { body: error }); + + // Open flyout + actions.clickAddDocumentsButton(); + + // Open documents accordion, add required fields, and click run + await actions.toggleDocumentsAccordion(); + form.setInputValue('indexField.input', nonExistentDoc.index); + form.setInputValue('idField.input', nonExistentDoc.id); + await actions.clickAddDocumentButton(); + + // Verify error rendered + expect(exists('addDocumentError')).toBe(true); + expect(exists('addDocumentSuccess')).toBe(false); + expect(find('addDocumentError').text()).toContain(error.message); + }); + }); }); describe('Processors', () => { diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.container.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.container.tsx index b49eea5b59ab0..23dda55db41f8 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.container.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.container.tsx @@ -12,6 +12,7 @@ import { useTestPipelineContext } from '../../context'; import { serialize } from '../../serialize'; import { DeserializeResult } from '../../deserialize'; import { Document } from '../../types'; +import { useIsMounted } from '../../use_is_mounted'; import { TestPipelineFlyout as ViewComponent } from './test_pipeline_flyout'; import { TestPipelineFlyoutTab } from './test_pipeline_flyout_tabs'; @@ -34,6 +35,7 @@ export const TestPipelineFlyout: React.FunctionComponent = ({ processors, }) => { const { services } = useKibana(); + const isMounted = useIsMounted(); const { testPipelineData, @@ -74,6 +76,10 @@ export const TestPipelineFlyout: React.FunctionComponent = ({ pipeline: { ...serializedProcessors }, }); + if (!isMounted.current) { + return { isSuccessful: false }; + } + setIsRunningTest(false); if (error) { @@ -123,6 +129,7 @@ export const TestPipelineFlyout: React.FunctionComponent = ({ return { isSuccessful: true }; }, [ + isMounted, processors, services.api, services.notifications.toasts, diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.tsx index 46271a6bce51c..d4895f8805531 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout.tsx @@ -16,7 +16,7 @@ import { EuiCallOut, } from '@elastic/eui'; -import { Form, FormHook } from '../../../../../shared_imports'; +import { FormHook } from '../../../../../shared_imports'; import { Document } from '../../types'; import { Tabs, TestPipelineFlyoutTab, OutputTab, DocumentsTab } from './test_pipeline_flyout_tabs'; @@ -71,19 +71,11 @@ export const TestPipelineFlyout: React.FunctionComponent = ({ } else { // default to "Documents" tab tabContent = ( -
- - + validateAndTestPipeline={validateAndTestPipeline} + isRunningTest={isRunningTest} + /> ); } diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_document_form.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_document_form.tsx new file mode 100644 index 0000000000000..340cf1af92300 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_document_form.tsx @@ -0,0 +1,209 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useState, FunctionComponent } from 'react'; +import { i18n } from '@kbn/i18n'; +import { + EuiButton, + EuiFlexGroup, + EuiFlexItem, + EuiCallOut, + EuiSpacer, + EuiText, + EuiIcon, +} from '@elastic/eui'; + +import { + getUseField, + Field, + useKibana, + useForm, + Form, + TextField, + fieldValidators, + FieldConfig, +} from '../../../../../../shared_imports'; +import { useIsMounted } from '../../../use_is_mounted'; +import { Document } from '../../../types'; + +const UseField = getUseField({ component: Field }); + +const { emptyField } = fieldValidators; + +const i18nTexts = { + addDocumentButton: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.addDocuments.addDocumentButtonLabel', + { + defaultMessage: 'Add document', + } + ), + addDocumentErrorMessage: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.addDocuments.addDocumentErrorMessage', + { + defaultMessage: 'Error adding document', + } + ), + addDocumentSuccessMessage: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.addDocuments.addDocumentSuccessMessage', + { + defaultMessage: 'Document added', + } + ), + indexField: { + fieldLabel: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.addDocuments.indexFieldLabel', + { + defaultMessage: 'Index', + } + ), + validationMessage: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.addDocuments.indexRequiredErrorMessage', + { + defaultMessage: 'An index name is required.', + } + ), + }, + idField: { + fieldLabel: i18n.translate('xpack.ingestPipelines.pipelineEditor.addDocuments.idFieldLabel', { + defaultMessage: 'Document ID', + }), + validationMessage: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.addDocuments.idRequiredErrorMessage', + { + defaultMessage: 'A document ID is required.', + } + ), + }, +}; + +const fieldsConfig: Record = { + index: { + label: i18nTexts.indexField.fieldLabel, + validations: [ + { + validator: emptyField(i18nTexts.indexField.validationMessage), + }, + ], + }, + id: { + label: i18nTexts.idField.fieldLabel, + validations: [ + { + validator: emptyField(i18nTexts.idField.validationMessage), + }, + ], + }, +}; + +interface Props { + onAddDocuments: (document: Document) => void; +} + +export const AddDocumentForm: FunctionComponent = ({ onAddDocuments }) => { + const { services } = useKibana(); + const isMounted = useIsMounted(); + + const [isLoadingDocument, setIsLoadingDocument] = useState(false); + const [documentError, setDocumentError] = useState(undefined); + const [isDocumentAdded, setIsDocumentAdded] = useState(false); + + const { form } = useForm({ defaultValue: { index: '', id: '' } }); + + const submitForm = async (e: React.FormEvent) => { + const { isValid, data } = await form.submit(); + + const { id, index } = data; + + if (isValid) { + setIsLoadingDocument(true); + setDocumentError(undefined); + setIsDocumentAdded(false); + + const { error, data: document } = await services.api.loadDocument(index, id); + + if (!isMounted.current) { + return; + } + + setIsLoadingDocument(false); + + if (error) { + setDocumentError(error); + return; + } + + setIsDocumentAdded(true); + onAddDocuments(document); + } + }; + + return ( +
+ {documentError && ( + <> + +

{documentError.message}

+ + + + + )} + + + + + + + + + + + {i18nTexts.addDocumentButton} + + + + {isDocumentAdded && ( + + + + + + + + {i18nTexts.addDocumentSuccessMessage} + + + + + )} + + + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.scss b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.scss new file mode 100644 index 0000000000000..2bf234fab2ece --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.scss @@ -0,0 +1,4 @@ +.addDocumentsAccordion { + background-color: $euiColorLightestShade; + padding: $euiSizeM; +} diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.tsx new file mode 100644 index 0000000000000..88ced6e9e94dd --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/add_documents_accordion.tsx @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FunctionComponent, useState, useEffect } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { EuiAccordion, EuiText, EuiSpacer, EuiLink } from '@elastic/eui'; +import { UrlGeneratorsDefinition } from 'src/plugins/share/public'; + +import { useKibana } from '../../../../../../../shared_imports'; +import { useIsMounted } from '../../../../use_is_mounted'; +import { AddDocumentForm } from '../add_document_form'; + +import './add_documents_accordion.scss'; + +const DISCOVER_URL_GENERATOR_ID = 'DISCOVER_APP_URL_GENERATOR'; + +const i18nTexts = { + addDocumentsButton: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.addDocumentsAccordion.addDocumentsButtonLabel', + { + defaultMessage: 'Add documents from index', + } + ), +}; + +interface Props { + onAddDocuments: (document: any) => void; +} + +export const AddDocumentsAccordion: FunctionComponent = ({ onAddDocuments }) => { + const { services } = useKibana(); + const isMounted = useIsMounted(); + const [discoverLink, setDiscoverLink] = useState(undefined); + + useEffect(() => { + const getDiscoverUrl = async (): Promise => { + let isDeprecated: UrlGeneratorsDefinition['isDeprecated']; + let createUrl: UrlGeneratorsDefinition['createUrl']; + + // This try/catch may not be necessary once + // https://github.com/elastic/kibana/issues/78344 is addressed + try { + ({ isDeprecated, createUrl } = services.urlGenerators.getUrlGenerator( + DISCOVER_URL_GENERATOR_ID + )); + } catch (e) { + // Discover plugin is not enabled + setDiscoverLink(undefined); + return; + } + + if (isDeprecated) { + setDiscoverLink(undefined); + return; + } + + const discoverUrl = await createUrl({ indexPatternId: undefined }); + + if (isMounted.current) { + setDiscoverLink(discoverUrl); + } + }; + + getDiscoverUrl(); + }, [isMounted, services.urlGenerators]); + + return ( + +
+ +

+ + {discoverLink && ( + <> + {' '} + + Discover + + ), + }} + /> + + )} +

+ + + + + +
+ + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/index.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/index.ts new file mode 100644 index 0000000000000..cb00ec640b5a6 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/add_documents_accordion/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { AddDocumentsAccordion } from './add_documents_accordion'; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/documents_schema.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/documents_schema.tsx index e8ac223d56ed9..d0e0596375cb2 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/documents_schema.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/documents_schema.tsx @@ -82,6 +82,30 @@ export const documentsSchema: FormSchema = { } }, }, + { + validator: ({ value }: ValidationFuncArg) => { + const parsedJSON = JSON.parse(value); + + const isMissingSourceField = parsedJSON.find((document: { _source?: object }) => { + if (!document._source) { + return true; + } + + return false; + }); + + if (isMissingSourceField) { + return { + message: i18n.translate( + 'xpack.ingestPipelines.testPipelineFlyout.documentsForm.sourceFieldRequiredError', + { + defaultMessage: 'Documents require a _source field.', + } + ), + }; + } + }, + }, ], }, }; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/tab_documents.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/tab_documents.tsx index b2326644340a7..6fd340054d2a4 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/tab_documents.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/components/test_pipeline/test_pipeline_flyout_tabs/tab_documents.tsx @@ -4,98 +4,131 @@ * you may not use this file except in compliance with the Elastic License. */ -import React from 'react'; +import React, { FunctionComponent, useCallback } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; import { EuiSpacer, EuiText, EuiButton, EuiLink } from '@elastic/eui'; -import { getUseField, Field, JsonEditorField, useKibana } from '../../../../../../shared_imports'; +import { + getUseField, + Field, + JsonEditorField, + useKibana, + useFormData, + FormHook, + Form, +} from '../../../../../../shared_imports'; + +import { AddDocumentsAccordion } from './add_documents_accordion'; const UseField = getUseField({ component: Field }); interface Props { validateAndTestPipeline: () => Promise; isRunningTest: boolean; - isSubmitButtonDisabled: boolean; + form: FormHook; } -export const DocumentsTab: React.FunctionComponent = ({ +export const DocumentsTab: FunctionComponent = ({ validateAndTestPipeline, - isSubmitButtonDisabled, isRunningTest, + form, }) => { const { services } = useKibana(); + const [, formatData] = useFormData({ form }); + + const onAddDocumentHandler = useCallback( + (document) => { + const { documents: existingDocuments = [] } = formatData(); + + form.reset({ defaultValue: { documents: [...existingDocuments, document] } }); + }, + [form, formatData] + ); + return ( -
- -

- - {i18n.translate( - 'xpack.ingestPipelines.testPipelineFlyout.documentsTab.simulateDocumentionLink', - { - defaultMessage: 'Learn more.', - } - )} - +

+
+ +

+ + {i18n.translate( + 'xpack.ingestPipelines.testPipelineFlyout.documentsTab.simulateDocumentionLink', + { + defaultMessage: 'Learn more.', + } + )} + + ), + }} + /> +

+ + + + + + + + + {/* Documents editor */} + -

- - - - - {/* Documents editor */} - - - - - - {isRunningTest ? ( - - ) : ( - - )} - -
+ }, + }} + /> + + + + + {isRunningTest ? ( + + ) : ( + + )} + +
+ ); }; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/context/test_pipeline_context.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/context/test_pipeline_context.tsx index 9aafeafa10b27..314964f808e44 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/context/test_pipeline_context.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/context/test_pipeline_context.tsx @@ -15,6 +15,7 @@ import { } from '../deserialize'; import { serialize } from '../serialize'; import { Document } from '../types'; +import { useIsMounted } from '../use_is_mounted'; export interface TestPipelineData { config: { @@ -127,6 +128,7 @@ export const reducer: Reducer = (state, action) => { export const TestPipelineContextProvider = ({ children }: { children: React.ReactNode }) => { const [state, dispatch] = useReducer(reducer, DEFAULT_TEST_PIPELINE_CONTEXT.testPipelineData); const { services } = useKibana(); + const isMounted = useIsMounted(); const updateTestOutputPerProcessor = useCallback( async (documents: Document[] | undefined, processors: DeserializeResult) => { @@ -152,6 +154,10 @@ export const TestPipelineContextProvider = ({ children }: { children: React.Reac pipeline: { ...serializedProcessorsWithTag }, }); + if (!isMounted.current) { + return; + } + if (error) { dispatch({ type: 'updateOutputPerProcessor', @@ -180,7 +186,7 @@ export const TestPipelineContextProvider = ({ children }: { children: React.Reac }, }); }, - [services.api, services.notifications.toasts] + [isMounted, services.api, services.notifications.toasts] ); return ( diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/use_is_mounted.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/use_is_mounted.ts new file mode 100644 index 0000000000000..c0df15e8a7fb7 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_processors_editor/use_is_mounted.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useEffect, useRef } from 'react'; + +export const useIsMounted = () => { + const isMounted = useRef(false); + + useEffect(() => { + isMounted.current = true; + + return () => { + isMounted.current = false; + }; + }, []); + + return isMounted; +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/index.tsx b/x-pack/plugins/ingest_pipelines/public/application/index.tsx index 6ffebd1854b78..0a71babc53315 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/index.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/index.tsx @@ -9,6 +9,7 @@ import React, { ReactNode } from 'react'; import { render, unmountComponentAtNode } from 'react-dom'; import { NotificationsSetup, IUiSettingsClient } from 'kibana/public'; import { ManagementAppMountParams } from 'src/plugins/management/public'; +import { SharePluginStart } from 'src/plugins/share/public'; import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/public'; import { API_BASE_PATH } from '../../common/constants'; @@ -26,6 +27,7 @@ export interface AppServices { notifications: NotificationsSetup; history: ManagementAppMountParams['history']; uiSettings: IUiSettingsClient; + urlGenerators: SharePluginStart['urlGenerators']; } export interface CoreServices { diff --git a/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts b/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts index 16ba9f9cd7a12..f7094a71a7792 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts @@ -6,15 +6,16 @@ import { CoreSetup } from 'src/core/public'; import { ManagementAppMountParams } from 'src/plugins/management/public'; +import { StartDependencies } from '../types'; import { documentationService, uiMetricService, apiService, breadcrumbService } from './services'; import { renderApp } from '.'; export async function mountManagementSection( - { http, getStartServices, notifications }: CoreSetup, + { http, getStartServices, notifications }: CoreSetup, params: ManagementAppMountParams ) { const { element, setBreadcrumbs, history } = params; - const [coreStart] = await getStartServices(); + const [coreStart, depsStart] = await getStartServices(); const { docLinks, i18n: { Context: I18nContext }, @@ -31,6 +32,7 @@ export async function mountManagementSection( notifications, history, uiSettings: coreStart.uiSettings, + urlGenerators: depsStart.share.urlGenerators, }; return renderApp(element, I18nContext, services, { http }); diff --git a/x-pack/plugins/ingest_pipelines/public/application/services/api.ts b/x-pack/plugins/ingest_pipelines/public/application/services/api.ts index 552e0ed0c41b2..2d6ab0477a603 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/services/api.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/services/api.ts @@ -120,6 +120,15 @@ export class ApiService { return result; } + + public async loadDocument(index: string, id: string) { + const result = await this.sendRequest({ + path: `${API_BASE_PATH}/documents/${encodeURIComponent(index)}/${encodeURIComponent(id)}`, + method: 'get', + }); + + return result; + } } export const apiService = new ApiService(); diff --git a/x-pack/plugins/ingest_pipelines/public/plugin.ts b/x-pack/plugins/ingest_pipelines/public/plugin.ts index 6c2f4a0898327..8b60967702742 100644 --- a/x-pack/plugins/ingest_pipelines/public/plugin.ts +++ b/x-pack/plugins/ingest_pipelines/public/plugin.ts @@ -9,11 +9,12 @@ import { CoreSetup, Plugin } from 'src/core/public'; import { PLUGIN_ID } from '../common/constants'; import { uiMetricService, apiService } from './application/services'; -import { Dependencies } from './types'; +import { SetupDependencies, StartDependencies } from './types'; import { registerUrlGenerator } from './url_generator'; -export class IngestPipelinesPlugin implements Plugin { - public setup(coreSetup: CoreSetup, plugins: Dependencies): void { +export class IngestPipelinesPlugin + implements Plugin { + public setup(coreSetup: CoreSetup, plugins: SetupDependencies): void { const { management, usageCollection, share } = plugins; const { http, getStartServices } = coreSetup; diff --git a/x-pack/plugins/ingest_pipelines/public/shared_imports.ts b/x-pack/plugins/ingest_pipelines/public/shared_imports.ts index 703b7a90f9356..13de8a74225ab 100644 --- a/x-pack/plugins/ingest_pipelines/public/shared_imports.ts +++ b/x-pack/plugins/ingest_pipelines/public/shared_imports.ts @@ -47,6 +47,7 @@ export { getFieldValidityAndErrorMessage, ValidationFunc, ValidationConfig, + useFormData, } from '../../../../src/plugins/es_ui_shared/static/forms/hook_form_lib'; export { @@ -65,6 +66,7 @@ export { NumericField, SelectField, CheckBoxField, + TextField, } from '../../../../src/plugins/es_ui_shared/static/forms/components'; export { diff --git a/x-pack/plugins/ingest_pipelines/public/types.ts b/x-pack/plugins/ingest_pipelines/public/types.ts index e968c87226d07..1638e60e98505 100644 --- a/x-pack/plugins/ingest_pipelines/public/types.ts +++ b/x-pack/plugins/ingest_pipelines/public/types.ts @@ -6,10 +6,14 @@ import { ManagementSetup } from 'src/plugins/management/public'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/public'; -import { SharePluginSetup } from '../../../../src/plugins/share/public'; +import { SharePluginStart, SharePluginSetup } from 'src/plugins/share/public'; -export interface Dependencies { +export interface SetupDependencies { management: ManagementSetup; usageCollection: UsageCollectionSetup; share: SharePluginSetup; } + +export interface StartDependencies { + share: SharePluginStart; +} diff --git a/x-pack/plugins/ingest_pipelines/public/url_generator.ts b/x-pack/plugins/ingest_pipelines/public/url_generator.ts index 043d449a0440a..c53ff083ea098 100644 --- a/x-pack/plugins/ingest_pipelines/public/url_generator.ts +++ b/x-pack/plugins/ingest_pipelines/public/url_generator.ts @@ -13,7 +13,7 @@ import { getEditPath, getListPath, } from './application/services/navigation'; -import { Dependencies } from './types'; +import { SetupDependencies } from './types'; import { PLUGIN_ID } from '../common/constants'; export const INGEST_PIPELINES_APP_ULR_GENERATOR = 'INGEST_PIPELINES_APP_URL_GENERATOR'; @@ -83,8 +83,8 @@ export class IngestPipelinesUrlGenerator export const registerUrlGenerator = ( coreSetup: CoreSetup, - management: Dependencies['management'], - share: Dependencies['share'] + management: SetupDependencies['management'], + share: SetupDependencies['share'] ) => { const getAppBasePath = async (absolute = false) => { const [coreStart] = await coreSetup.getStartServices(); diff --git a/x-pack/plugins/ingest_pipelines/server/routes/api/documents.ts b/x-pack/plugins/ingest_pipelines/server/routes/api/documents.ts new file mode 100644 index 0000000000000..1f19112e069d5 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/server/routes/api/documents.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { schema } from '@kbn/config-schema'; + +import { API_BASE_PATH } from '../../../common/constants'; +import { RouteDependencies } from '../../types'; + +const paramsSchema = schema.object({ + index: schema.string(), + id: schema.string(), +}); + +export const registerDocumentsRoute = ({ + router, + license, + lib: { isEsError }, +}: RouteDependencies): void => { + router.get( + { + path: `${API_BASE_PATH}/documents/{index}/{id}`, + validate: { + params: paramsSchema, + }, + }, + license.guardApiRoute(async (ctx, req, res) => { + const { callAsCurrentUser } = ctx.core.elasticsearch.legacy.client; + const { index, id } = req.params; + + try { + const document = await callAsCurrentUser('get', { index, id }); + + const { _id, _index, _source } = document; + + return res.ok({ + body: { + _id, + _index, + _source, + }, + }); + } catch (error) { + if (isEsError(error)) { + return res.customError({ + statusCode: error.statusCode, + body: error, + }); + } + + return res.internalError({ body: error }); + } + }) + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts b/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts index 58a4bf5617659..7c0ab19917d1f 100644 --- a/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts +++ b/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts @@ -15,3 +15,5 @@ export { registerPrivilegesRoute } from './privileges'; export { registerDeleteRoute } from './delete'; export { registerSimulateRoute } from './simulate'; + +export { registerDocumentsRoute } from './documents'; diff --git a/x-pack/plugins/ingest_pipelines/server/routes/index.ts b/x-pack/plugins/ingest_pipelines/server/routes/index.ts index f703a460143f4..5e80be4388b25 100644 --- a/x-pack/plugins/ingest_pipelines/server/routes/index.ts +++ b/x-pack/plugins/ingest_pipelines/server/routes/index.ts @@ -13,6 +13,7 @@ import { registerPrivilegesRoute, registerDeleteRoute, registerSimulateRoute, + registerDocumentsRoute, } from './api'; export class ApiRoutes { @@ -23,5 +24,6 @@ export class ApiRoutes { registerPrivilegesRoute(dependencies); registerDeleteRoute(dependencies); registerSimulateRoute(dependencies); + registerDocumentsRoute(dependencies); } } diff --git a/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts b/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts index b3fab42a46114..b80306b0e6d38 100644 --- a/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts +++ b/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts @@ -14,7 +14,13 @@ const API_BASE_PATH = '/api/ingest_pipelines'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); - const { createPipeline, deletePipeline, cleanupPipelines } = registerEsHelpers(getService); + const { + createPipeline, + deletePipeline, + cleanupPipelines, + createIndex, + deleteIndex, + } = registerEsHelpers(getService); describe('Pipelines', function () { after(async () => { @@ -445,5 +451,59 @@ export default function ({ getService }: FtrProviderContext) { expect(body.docs?.length).to.eql(2); }); }); + + describe('Fetch documents', () => { + const INDEX = 'test_index'; + const DOCUMENT_ID = '1'; + const DOCUMENT = { + name: 'John Doe', + }; + + before(async () => { + // Create an index with a document that can be used to test GET request + try { + await createIndex({ id: DOCUMENT_ID, index: INDEX, body: DOCUMENT }); + } catch (err) { + // eslint-disable-next-line no-console + console.log('[Setup error] Error creating index'); + throw err; + } + }); + + after(async () => { + // Clean up index created + try { + await deleteIndex(INDEX); + } catch (err) { + // eslint-disable-next-line no-console + console.log('[Cleanup error] Error deleting index'); + throw err; + } + }); + + it('should return a document', async () => { + const uri = `${API_BASE_PATH}/documents/${INDEX}/${DOCUMENT_ID}`; + + const { body } = await supertest.get(uri).set('kbn-xsrf', 'xxx').expect(200); + + expect(body).to.eql({ + _index: INDEX, + _id: DOCUMENT_ID, + _source: DOCUMENT, + }); + }); + + it('should return an error if the document does not exist', async () => { + const uri = `${API_BASE_PATH}/documents/${INDEX}/2`; // Document 2 does not exist + + const { body } = await supertest.get(uri).set('kbn-xsrf', 'xxx').expect(404); + + expect(body).to.eql({ + error: 'Not Found', + message: 'Not Found', + statusCode: 404, + }); + }); + }); }); } diff --git a/x-pack/test/api_integration/apis/management/ingest_pipelines/lib/elasticsearch.ts b/x-pack/test/api_integration/apis/management/ingest_pipelines/lib/elasticsearch.ts index 6de91e1154a85..aeed61cb0bf92 100644 --- a/x-pack/test/api_integration/apis/management/ingest_pipelines/lib/elasticsearch.ts +++ b/x-pack/test/api_integration/apis/management/ingest_pipelines/lib/elasticsearch.ts @@ -50,9 +50,19 @@ export const registerEsHelpers = (getService: FtrProviderContext['getService']) console.log(`[Cleanup error] Error deleting ES resources: ${err.message}`); }); + const createIndex = (index: { index: string; id: string; body: object }) => { + return es.index(index); + }; + + const deleteIndex = (indexName: string) => { + return es.indices.delete({ index: indexName }); + }; + return { createPipeline, deletePipeline, cleanupPipelines, + createIndex, + deleteIndex, }; }; From 966f00ac5956ac32e8aa76c54706cf7901328ba9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cau=C3=AA=20Marcondes?= <55978943+cauemarcondes@users.noreply.github.com> Date: Mon, 28 Sep 2020 14:12:58 +0100 Subject: [PATCH 07/11] [APM] Alerting: Add global option to create all alert types (#78151) * adding alert to service page * sending on alert per service environment and transaction type * addressing PR comment * addressing PR comment Co-authored-by: Elastic Machine --- .../AlertingFlyout/index.tsx | 4 +- .../alerting/ServiceAlertTrigger/index.tsx | 16 +- .../TransactionDurationAlertTrigger/index.tsx | 10 +- .../index.tsx | 26 +- .../index.tsx | 12 +- .../components/alerting/fields.test.tsx | 61 ++++ .../apm/public/components/alerting/fields.tsx | 15 +- .../alerting/get_alert_capabilities.ts | 32 ++ .../Home/alerting_popover_flyout/index.tsx | 186 ++++++++++ .../apm/public/components/app/Home/index.tsx | 25 +- .../index.tsx | 6 +- .../components/app/ServiceDetails/index.tsx | 30 +- .../register_error_count_alert_type.test.ts | 197 +++++++++++ .../alerts/register_error_count_alert_type.ts | 81 ++++- ...action_duration_anomaly_alert_type.test.ts | 326 ++++++++++++++++++ ...transaction_duration_anomaly_alert_type.ts | 117 +++++-- ..._transaction_error_rate_alert_type.test.ts | 289 ++++++++++++++++ ...ister_transaction_error_rate_alert_type.ts | 93 ++++- .../lib/service_map/get_service_anomalies.ts | 12 +- 19 files changed, 1419 insertions(+), 119 deletions(-) rename x-pack/plugins/apm/public/components/{app/ServiceDetails/AlertIntegrations => alerting}/AlertingFlyout/index.tsx (86%) create mode 100644 x-pack/plugins/apm/public/components/alerting/fields.test.tsx create mode 100644 x-pack/plugins/apm/public/components/alerting/get_alert_capabilities.ts create mode 100644 x-pack/plugins/apm/public/components/app/Home/alerting_popover_flyout/index.tsx rename x-pack/plugins/apm/public/components/app/ServiceDetails/{AlertIntegrations => alerting_popover_flyout}/index.tsx (97%) create mode 100644 x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.test.ts create mode 100644 x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts create mode 100644 x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts diff --git a/x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/AlertingFlyout/index.tsx b/x-pack/plugins/apm/public/components/alerting/AlertingFlyout/index.tsx similarity index 86% rename from x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/AlertingFlyout/index.tsx rename to x-pack/plugins/apm/public/components/alerting/AlertingFlyout/index.tsx index ad3f1696ad5e3..3bee6b2388264 100644 --- a/x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/AlertingFlyout/index.tsx +++ b/x-pack/plugins/apm/public/components/alerting/AlertingFlyout/index.tsx @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ import React from 'react'; -import { AlertType } from '../../../../../../common/alert_types'; -import { AlertAdd } from '../../../../../../../triggers_actions_ui/public'; +import { AlertType } from '../../../../common/alert_types'; +import { AlertAdd } from '../../../../../triggers_actions_ui/public'; type AlertAddProps = React.ComponentProps; diff --git a/x-pack/plugins/apm/public/components/alerting/ServiceAlertTrigger/index.tsx b/x-pack/plugins/apm/public/components/alerting/ServiceAlertTrigger/index.tsx index 86dc7f5a90475..b4d3e8f3ad241 100644 --- a/x-pack/plugins/apm/public/components/alerting/ServiceAlertTrigger/index.tsx +++ b/x-pack/plugins/apm/public/components/alerting/ServiceAlertTrigger/index.tsx @@ -34,11 +34,17 @@ export function ServiceAlertTrigger(props: Props) { useEffect(() => { // we only want to run this on mount to set default values - setAlertProperty('name', `${alertTypeName} | ${params.serviceName}`); - setAlertProperty('tags', [ - 'apm', - `service.name:${params.serviceName}`.toLowerCase(), - ]); + + const alertName = params.serviceName + ? `${alertTypeName} | ${params.serviceName}` + : alertTypeName; + setAlertProperty('name', alertName); + + const tags = ['apm']; + if (params.serviceName) { + tags.push(`service.name:${params.serviceName}`.toLowerCase()); + } + setAlertProperty('tags', tags); Object.keys(params).forEach((key) => { setAlertParams(key, params[key]); }); diff --git a/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx b/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx index 3ddd623d9e848..ce98354c94c7e 100644 --- a/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx +++ b/x-pack/plugins/apm/public/components/alerting/TransactionDurationAlertTrigger/index.tsx @@ -90,16 +90,16 @@ export function TransactionDurationAlertTrigger(props: Props) { const fields = [ , - setAlertParams('environment', e.target.value)} - />, ({ text: key, value: key }))} onChange={(e) => setAlertParams('transactionType', e.target.value)} />, + setAlertParams('environment', e.target.value)} + />, (); - const { start, end } = urlParams; + const { start, end, transactionType } = urlParams; const { environmentOptions } = useEnvironments({ serviceName, start, end }); - const supportedTransactionTypes = transactionTypes.filter((transactionType) => - [TRANSACTION_PAGE_LOAD, TRANSACTION_REQUEST].includes(transactionType) - ); - if (!supportedTransactionTypes.length || !serviceName) { + if (serviceName && !transactionTypes.length) { return null; } - // 'page-load' for RUM, 'request' otherwise - const transactionType = supportedTransactionTypes[0]; - const defaults: Params = { windowSize: 15, windowUnit: 'm', - transactionType, + transactionType: transactionType || transactionTypes[0], serviceName, environment: urlParams.environment || ENVIRONMENT_ALL.value, anomalySeverityType: ANOMALY_SEVERITY.CRITICAL, @@ -82,7 +72,11 @@ export function TransactionDurationAnomalyAlertTrigger(props: Props) { const fields = [ , - , + ({ text: key, value: key }))} + onChange={(e) => setAlertParams('transactionType', e.target.value)} + />, , - setAlertParams('environment', e.target.value)} - />, ({ text: key, value: key }))} onChange={(e) => setAlertParams('transactionType', e.target.value)} />, + setAlertParams('environment', e.target.value)} + />, { + describe('Service Fiels', () => { + it('renders with value', () => { + const component = render(); + expectTextsInDocument(component, ['foo']); + }); + it('renders with All when value is not defined', () => { + const component = render(); + expectTextsInDocument(component, ['All']); + }); + }); + describe('Transaction Type Field', () => { + it('renders select field when multiple options available', () => { + const options = [ + { text: 'Foo', value: 'foo' }, + { text: 'Bar', value: 'bar' }, + ]; + const { getByText, getByTestId } = render( + + ); + + act(() => { + fireEvent.click(getByText('Foo')); + }); + + const selectBar = getByTestId('transactionTypeField'); + expect(selectBar instanceof HTMLSelectElement).toBeTruthy(); + const selectOptions = (selectBar as HTMLSelectElement).options; + expect(selectOptions.length).toEqual(2); + expect( + Object.values(selectOptions).map((option) => option.value) + ).toEqual(['foo', 'bar']); + }); + it('renders read-only field when single option available', () => { + const options = [{ text: 'Bar', value: 'bar' }]; + const component = render( + + ); + expectTextsInDocument(component, ['Bar']); + }); + it('renders read-only All option when no option available', () => { + const component = render(); + expectTextsInDocument(component, ['All']); + }); + + it('renders current value when available', () => { + const component = render(); + expectTextsInDocument(component, ['foo']); + }); + }); +}); diff --git a/x-pack/plugins/apm/public/components/alerting/fields.tsx b/x-pack/plugins/apm/public/components/alerting/fields.tsx index e145d03671a18..aac64649546cc 100644 --- a/x-pack/plugins/apm/public/components/alerting/fields.tsx +++ b/x-pack/plugins/apm/public/components/alerting/fields.tsx @@ -11,13 +11,17 @@ import { EuiSelectOption } from '@elastic/eui'; import { getEnvironmentLabel } from '../../../common/environment_filter_values'; import { PopoverExpression } from './ServiceAlertTrigger/PopoverExpression'; +const ALL_OPTION = i18n.translate('xpack.apm.alerting.fields.all_option', { + defaultMessage: 'All', +}); + export function ServiceField({ value }: { value?: string }) { return ( ); } @@ -53,7 +57,7 @@ export function TransactionTypeField({ options, onChange, }: { - currentValue: string; + currentValue?: string; options?: EuiSelectOption[]; onChange?: (event: React.ChangeEvent) => void; }) { @@ -61,13 +65,16 @@ export function TransactionTypeField({ defaultMessage: 'Type', }); - if (!options || options.length === 1) { - return ; + if (!options || options.length <= 1) { + return ( + + ); } return ( { + const canReadAlerts = !!capabilities.apm['alerting:show']; + const canSaveAlerts = !!capabilities.apm['alerting:save']; + const isAlertingPluginEnabled = 'alerts' in plugins; + const isAlertingAvailable = + isAlertingPluginEnabled && (canReadAlerts || canSaveAlerts); + const isMlPluginEnabled = 'ml' in plugins; + const canReadAnomalies = !!( + isMlPluginEnabled && + capabilities.ml.canAccessML && + capabilities.ml.canGetJobs + ); + + return { + isAlertingAvailable, + canReadAlerts, + canSaveAlerts, + canReadAnomalies, + }; +}; diff --git a/x-pack/plugins/apm/public/components/app/Home/alerting_popover_flyout/index.tsx b/x-pack/plugins/apm/public/components/app/Home/alerting_popover_flyout/index.tsx new file mode 100644 index 0000000000000..7e6331c1fa3a8 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/Home/alerting_popover_flyout/index.tsx @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + EuiButtonEmpty, + EuiContextMenu, + EuiContextMenuPanelDescriptor, + EuiPopover, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React, { useState } from 'react'; +import { AlertType } from '../../../../../common/alert_types'; +import { useApmPluginContext } from '../../../../hooks/useApmPluginContext'; +import { AlertingFlyout } from '../../../alerting/AlertingFlyout'; + +const alertLabel = i18n.translate('xpack.apm.home.alertsMenu.alerts', { + defaultMessage: 'Alerts', +}); +const transactionDurationLabel = i18n.translate( + 'xpack.apm.home.alertsMenu.transactionDuration', + { defaultMessage: 'Transaction duration' } +); +const transactionErrorRateLabel = i18n.translate( + 'xpack.apm.home.alertsMenu.transactionErrorRate', + { defaultMessage: 'Transaction error rate' } +); +const errorCountLabel = i18n.translate('xpack.apm.home.alertsMenu.errorCount', { + defaultMessage: 'Error count', +}); +const createThresholdAlertLabel = i18n.translate( + 'xpack.apm.home.alertsMenu.createThresholdAlert', + { defaultMessage: 'Create threshold alert' } +); +const createAnomalyAlertAlertLabel = i18n.translate( + 'xpack.apm.home.alertsMenu.createAnomalyAlert', + { defaultMessage: 'Create anomaly alert' } +); + +const CREATE_TRANSACTION_DURATION_ALERT_PANEL_ID = + 'create_transaction_duration_panel'; +const CREATE_TRANSACTION_ERROR_RATE_ALERT_PANEL_ID = + 'create_transaction_error_rate_panel'; +const CREATE_ERROR_COUNT_ALERT_PANEL_ID = 'create_error_count_panel'; + +interface Props { + canReadAlerts: boolean; + canSaveAlerts: boolean; + canReadAnomalies: boolean; +} + +export function AlertingPopoverAndFlyout(props: Props) { + const { canSaveAlerts, canReadAlerts, canReadAnomalies } = props; + + const plugin = useApmPluginContext(); + + const [popoverOpen, setPopoverOpen] = useState(false); + + const [alertType, setAlertType] = useState(null); + + const button = ( + setPopoverOpen(true)} + > + {alertLabel} + + ); + + const panels: EuiContextMenuPanelDescriptor[] = [ + { + id: 0, + title: alertLabel, + items: [ + ...(canSaveAlerts + ? [ + { + name: transactionDurationLabel, + panel: CREATE_TRANSACTION_DURATION_ALERT_PANEL_ID, + }, + { + name: transactionErrorRateLabel, + panel: CREATE_TRANSACTION_ERROR_RATE_ALERT_PANEL_ID, + }, + { + name: errorCountLabel, + panel: CREATE_ERROR_COUNT_ALERT_PANEL_ID, + }, + ] + : []), + ...(canReadAlerts + ? [ + { + name: i18n.translate( + 'xpack.apm.home.alertsMenu.viewActiveAlerts', + { defaultMessage: 'View active alerts' } + ), + href: plugin.core.http.basePath.prepend( + '/app/management/insightsAndAlerting/triggersActions/alerts' + ), + icon: 'tableOfContents', + }, + ] + : []), + ], + }, + + // transaction duration panel + { + id: CREATE_TRANSACTION_DURATION_ALERT_PANEL_ID, + title: transactionDurationLabel, + items: [ + // anomaly alerts + ...(canReadAnomalies + ? [ + { + name: createAnomalyAlertAlertLabel, + onClick: () => { + setAlertType(AlertType.TransactionDurationAnomaly); + setPopoverOpen(false); + }, + }, + ] + : []), + ], + }, + + // transaction error rate panel + { + id: CREATE_TRANSACTION_ERROR_RATE_ALERT_PANEL_ID, + title: transactionErrorRateLabel, + items: [ + // threshold alerts + { + name: createThresholdAlertLabel, + onClick: () => { + setAlertType(AlertType.TransactionErrorRate); + setPopoverOpen(false); + }, + }, + ], + }, + + // error alerts panel + { + id: CREATE_ERROR_COUNT_ALERT_PANEL_ID, + title: errorCountLabel, + items: [ + { + name: createThresholdAlertLabel, + onClick: () => { + setAlertType(AlertType.ErrorCount); + setPopoverOpen(false); + }, + }, + ], + }, + ]; + + return ( + <> + setPopoverOpen(false)} + panelPaddingSize="none" + anchorPosition="downRight" + > + + + { + if (!visible) { + setAlertType(null); + } + }} + /> + + ); +} diff --git a/x-pack/plugins/apm/public/components/app/Home/index.tsx b/x-pack/plugins/apm/public/components/app/Home/index.tsx index b2f15dbb11341..446f7b978a434 100644 --- a/x-pack/plugins/apm/public/components/app/Home/index.tsx +++ b/x-pack/plugins/apm/public/components/app/Home/index.tsx @@ -15,17 +15,19 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { $ElementType } from 'utility-types'; import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; +import { getAlertingCapabilities } from '../../alerting/get_alert_capabilities'; import { ApmHeader } from '../../shared/ApmHeader'; import { EuiTabLink } from '../../shared/EuiTabLink'; +import { AnomalyDetectionSetupLink } from '../../shared/Links/apm/AnomalyDetectionSetupLink'; import { ServiceMapLink } from '../../shared/Links/apm/ServiceMapLink'; import { ServiceOverviewLink } from '../../shared/Links/apm/ServiceOverviewLink'; import { SettingsLink } from '../../shared/Links/apm/SettingsLink'; -import { AnomalyDetectionSetupLink } from '../../shared/Links/apm/AnomalyDetectionSetupLink'; import { TraceOverviewLink } from '../../shared/Links/apm/TraceOverviewLink'; import { SetupInstructionsLink } from '../../shared/Links/SetupInstructionsLink'; import { ServiceMap } from '../ServiceMap'; import { ServiceOverview } from '../ServiceOverview'; import { TraceOverview } from '../TraceOverview'; +import { AlertingPopoverAndFlyout } from './alerting_popover_flyout'; function getHomeTabs({ serviceMapEnabled = true, @@ -83,13 +85,21 @@ interface Props { } export function Home({ tab }: Props) { - const { config, core } = useApmPluginContext(); - const canAccessML = !!core.application.capabilities.ml?.canAccessML; + const { config, core, plugins } = useApmPluginContext(); + const capabilities = core.application.capabilities; + const canAccessML = !!capabilities.ml?.canAccessML; const homeTabs = getHomeTabs(config); const selectedTab = homeTabs.find( (homeTab) => homeTab.name === tab ) as $ElementType; + const { + isAlertingAvailable, + canReadAlerts, + canSaveAlerts, + canReadAnomalies, + } = getAlertingCapabilities(plugins, core.application.capabilities); + return (
@@ -106,6 +116,15 @@ export function Home({ tab }: Props) { + {isAlertingAvailable && ( + + + + )} {canAccessML && ( diff --git a/x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/index.tsx b/x-pack/plugins/apm/public/components/app/ServiceDetails/alerting_popover_flyout/index.tsx similarity index 97% rename from x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/index.tsx rename to x-pack/plugins/apm/public/components/app/ServiceDetails/alerting_popover_flyout/index.tsx index c11bfdeae945b..3a8d24f0a8b02 100644 --- a/x-pack/plugins/apm/public/components/app/ServiceDetails/AlertIntegrations/index.tsx +++ b/x-pack/plugins/apm/public/components/app/ServiceDetails/alerting_popover_flyout/index.tsx @@ -7,14 +7,14 @@ import { EuiButtonEmpty, EuiContextMenu, - EuiPopover, EuiContextMenuPanelDescriptor, + EuiPopover, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import React, { useState } from 'react'; import { AlertType } from '../../../../../common/alert_types'; -import { AlertingFlyout } from './AlertingFlyout'; import { useApmPluginContext } from '../../../../hooks/useApmPluginContext'; +import { AlertingFlyout } from '../../../alerting/AlertingFlyout'; const alertLabel = i18n.translate( 'xpack.apm.serviceDetails.alertsMenu.alerts', @@ -53,7 +53,7 @@ interface Props { canReadAnomalies: boolean; } -export function AlertIntegrations(props: Props) { +export function AlertingPopoverAndFlyout(props: Props) { const { canSaveAlerts, canReadAlerts, canReadAnomalies } = props; const plugin = useApmPluginContext(); diff --git a/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx b/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx index 67c4a7c4cde1b..8825702cafd51 100644 --- a/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx +++ b/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx @@ -14,8 +14,9 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { RouteComponentProps } from 'react-router-dom'; import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; +import { getAlertingCapabilities } from '../../alerting/get_alert_capabilities'; import { ApmHeader } from '../../shared/ApmHeader'; -import { AlertIntegrations } from './AlertIntegrations'; +import { AlertingPopoverAndFlyout } from './alerting_popover_flyout'; import { ServiceDetailTabs } from './ServiceDetailTabs'; interface Props extends RouteComponentProps<{ serviceName: string }> { @@ -23,20 +24,15 @@ interface Props extends RouteComponentProps<{ serviceName: string }> { } export function ServiceDetails({ match, tab }: Props) { - const plugin = useApmPluginContext(); + const { core, plugins } = useApmPluginContext(); const { serviceName } = match.params; - const capabilities = plugin.core.application.capabilities; - const canReadAlerts = !!capabilities.apm['alerting:show']; - const canSaveAlerts = !!capabilities.apm['alerting:save']; - const isAlertingPluginEnabled = 'alerts' in plugin.plugins; - const isAlertingAvailable = - isAlertingPluginEnabled && (canReadAlerts || canSaveAlerts); - const isMlPluginEnabled = 'ml' in plugin.plugins; - const canReadAnomalies = !!( - isMlPluginEnabled && - capabilities.ml.canAccessML && - capabilities.ml.canGetJobs - ); + + const { + isAlertingAvailable, + canReadAlerts, + canSaveAlerts, + canReadAnomalies, + } = getAlertingCapabilities(plugins, core.application.capabilities); const ADD_DATA_LABEL = i18n.translate('xpack.apm.addDataButtonLabel', { defaultMessage: 'Add data', @@ -53,7 +49,7 @@ export function ServiceDetails({ match, tab }: Props) { {isAlertingAvailable && ( - = (source: Rx.Observable) => Rx.Observable; +const pipeClosure = (fn: Operator): Operator => { + return (source: Rx.Observable) => { + return Rx.defer(() => fn(source)); + }; +}; +const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( + pipeClosure((source$) => { + return source$.pipe(map((i) => i)); + }), + toArray() +) as unknown) as Observable; + +describe('Error count alert', () => { + it("doesn't send an alert when error count is less than threshold", async () => { + let alertExecutor: any; + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + registerErrorCountAlertType({ + alerts, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const services = { + callCluster: jest.fn(() => ({ + hits: { + total: { + value: 0, + }, + }, + })), + alertInstanceFactory: jest.fn(), + }; + const params = { threshold: 1 }; + + await alertExecutor!({ services, params }); + expect(services.alertInstanceFactory).not.toBeCalled(); + }); + + it('sends alerts with service name and environment', async () => { + let alertExecutor: any; + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + registerErrorCountAlertType({ + alerts, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const scheduleActions = jest.fn(); + const services = { + callCluster: jest.fn(() => ({ + hits: { + total: { + value: 2, + }, + }, + aggregations: { + services: { + buckets: [ + { + key: 'foo', + environments: { + buckets: [{ key: 'env-foo' }, { key: 'env-foo-2' }], + }, + }, + { + key: 'bar', + environments: { + buckets: [{ key: 'env-bar' }, { key: 'env-bar-2' }], + }, + }, + ], + }, + }, + })), + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + }; + const params = { threshold: 1 }; + + await alertExecutor!({ services, params }); + [ + 'apm.error_rate_foo_env-foo', + 'apm.error_rate_foo_env-foo-2', + 'apm.error_rate_bar_env-bar', + 'apm.error_rate_bar_env-bar-2', + ].forEach((instanceName) => + expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + ); + + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'foo', + environment: 'env-foo', + threshold: 1, + triggerValue: 2, + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'foo', + environment: 'env-foo-2', + threshold: 1, + triggerValue: 2, + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'bar', + environment: 'env-bar', + threshold: 1, + triggerValue: 2, + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'bar', + environment: 'env-bar-2', + threshold: 1, + triggerValue: 2, + }); + }); + it('sends alerts with service name', async () => { + let alertExecutor: any; + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + registerErrorCountAlertType({ + alerts, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const scheduleActions = jest.fn(); + const services = { + callCluster: jest.fn(() => ({ + hits: { + total: { + value: 2, + }, + }, + aggregations: { + services: { + buckets: [ + { + key: 'foo', + }, + { + key: 'bar', + }, + ], + }, + }, + })), + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + }; + const params = { threshold: 1 }; + + await alertExecutor!({ services, params }); + ['apm.error_rate_foo', 'apm.error_rate_bar'].forEach((instanceName) => + expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + ); + + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'foo', + environment: undefined, + threshold: 1, + triggerValue: 2, + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'bar', + environment: undefined, + threshold: 1, + triggerValue: 2, + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts index 5455cd9f6a495..26e4a5e84b995 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts @@ -5,22 +5,21 @@ */ import { schema } from '@kbn/config-schema'; +import { isEmpty } from 'lodash'; import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; -import { ProcessorEvent } from '../../../common/processor_event'; -import { getEnvironmentUiFilterES } from '../helpers/convert_ui_filters/get_environment_ui_filter_es'; +import { APMConfig } from '../..'; +import { AlertingPlugin } from '../../../../alerts/server'; import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; -import { - ESSearchResponse, - ESSearchRequest, -} from '../../../typings/elasticsearch'; import { PROCESSOR_EVENT, + SERVICE_ENVIRONMENT, SERVICE_NAME, } from '../../../common/elasticsearch_fieldnames'; -import { AlertingPlugin } from '../../../../alerts/server'; +import { ProcessorEvent } from '../../../common/processor_event'; +import { ESSearchResponse } from '../../../typings/elasticsearch'; +import { getEnvironmentUiFilterES } from '../helpers/convert_ui_filters/get_environment_ui_filter_es'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; -import { APMConfig } from '../..'; import { apmActionVariables } from './action_variables'; interface RegisterAlertParams { @@ -32,7 +31,7 @@ const paramsSchema = schema.object({ windowSize: schema.number(), windowUnit: schema.string(), threshold: schema.number(), - serviceName: schema.string(), + serviceName: schema.maybe(schema.string()), environment: schema.string(), }); @@ -83,30 +82,74 @@ export function registerErrorCountAlertType({ }, }, { term: { [PROCESSOR_EVENT]: ProcessorEvent.error } }, - { term: { [SERVICE_NAME]: alertParams.serviceName } }, + ...(alertParams.serviceName + ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] + : []), ...getEnvironmentUiFilterES(alertParams.environment), ], }, }, + aggs: { + services: { + terms: { + field: SERVICE_NAME, + size: 50, + }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + }, + }, + }, + }, + }, }, }; const response: ESSearchResponse< unknown, - ESSearchRequest + typeof searchParams > = await services.callCluster('search', searchParams); const errorCount = response.hits.total.value; if (errorCount > alertParams.threshold) { - const alertInstance = services.alertInstanceFactory( - AlertType.ErrorCount - ); - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { - serviceName: alertParams.serviceName, - environment: alertParams.environment, - threshold: alertParams.threshold, - triggerValue: errorCount, + function scheduleAction({ + serviceName, + environment, + }: { + serviceName: string; + environment?: string; + }) { + const alertInstanceName = [ + AlertType.ErrorCount, + serviceName, + environment, + ] + .filter((name) => name) + .join('_'); + + const alertInstance = services.alertInstanceFactory( + alertInstanceName + ); + alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + environment, + threshold: alertParams.threshold, + triggerValue: errorCount, + }); + } + response.aggregations?.services.buckets.forEach((serviceBucket) => { + const serviceName = serviceBucket.key as string; + if (isEmpty(serviceBucket.environments?.buckets)) { + scheduleAction({ serviceName }); + } else { + serviceBucket.environments.buckets.forEach((envBucket) => { + const environment = envBucket.key as string; + scheduleAction({ serviceName, environment }); + }); + } }); } }, diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts new file mode 100644 index 0000000000000..6e97262dd77bb --- /dev/null +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.test.ts @@ -0,0 +1,326 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Observable } from 'rxjs'; +import * as Rx from 'rxjs'; +import { toArray, map } from 'rxjs/operators'; +import { AlertingPlugin } from '../../../../alerts/server'; +import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type'; +import { APMConfig } from '../..'; +import { ANOMALY_SEVERITY } from '../../../../ml/common'; +import { Job, MlPluginSetup } from '../../../../ml/server'; +import * as GetServiceAnomalies from '../service_map/get_service_anomalies'; + +type Operator = (source: Rx.Observable) => Rx.Observable; +const pipeClosure = (fn: Operator): Operator => { + return (source: Rx.Observable) => { + return Rx.defer(() => fn(source)); + }; +}; +const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( + pipeClosure((source$) => { + return source$.pipe(map((i) => i)); + }), + toArray() +) as unknown) as Observable; + +describe('Transaction duration anomaly alert', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + describe("doesn't send alert", () => { + it('ml is not defined', async () => { + let alertExecutor: any; + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + registerTransactionDurationAnomalyAlertType({ + alerts, + ml: undefined, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const services = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(), + }; + const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; + + await alertExecutor!({ services, params }); + expect(services.callCluster).not.toHaveBeenCalled(); + expect(services.alertInstanceFactory).not.toHaveBeenCalled(); + }); + + it('ml jobs are not available', async () => { + jest + .spyOn(GetServiceAnomalies, 'getMLJobs') + .mockReturnValue(Promise.resolve([])); + + let alertExecutor: any; + + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + const ml = ({ + mlSystemProvider: () => ({ mlAnomalySearch: jest.fn() }), + anomalyDetectorsProvider: jest.fn(), + } as unknown) as MlPluginSetup; + + registerTransactionDurationAnomalyAlertType({ + alerts, + ml, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const services = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(), + }; + const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; + + await alertExecutor!({ services, params }); + expect(services.callCluster).not.toHaveBeenCalled(); + expect(services.alertInstanceFactory).not.toHaveBeenCalled(); + }); + + it('anomaly is less than threshold', async () => { + jest + .spyOn(GetServiceAnomalies, 'getMLJobs') + .mockReturnValue( + Promise.resolve([{ job_id: '1' }, { job_id: '2' }] as Job[]) + ); + + let alertExecutor: any; + + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + const ml = ({ + mlSystemProvider: () => ({ + mlAnomalySearch: () => ({ + hits: { total: { value: 0 } }, + }), + }), + anomalyDetectorsProvider: jest.fn(), + } as unknown) as MlPluginSetup; + + registerTransactionDurationAnomalyAlertType({ + alerts, + ml, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const services = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(), + }; + const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; + + await alertExecutor!({ services, params }); + expect(services.callCluster).not.toHaveBeenCalled(); + expect(services.alertInstanceFactory).not.toHaveBeenCalled(); + }); + }); + + describe('sends alert', () => { + it('with service name, environment and transaction type', async () => { + jest.spyOn(GetServiceAnomalies, 'getMLJobs').mockReturnValue( + Promise.resolve([ + { + job_id: '1', + custom_settings: { + job_tags: { + environment: 'production', + }, + }, + } as unknown, + { + job_id: '2', + custom_settings: { + job_tags: { + environment: 'production', + }, + }, + } as unknown, + ] as Job[]) + ); + + let alertExecutor: any; + + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + const ml = ({ + mlSystemProvider: () => ({ + mlAnomalySearch: () => ({ + hits: { total: { value: 2 } }, + aggregations: { + services: { + buckets: [ + { + key: 'foo', + transaction_types: { + buckets: [{ key: 'type-foo' }], + }, + }, + { + key: 'bar', + transaction_types: { + buckets: [{ key: 'type-bar' }], + }, + }, + ], + }, + }, + }), + }), + anomalyDetectorsProvider: jest.fn(), + } as unknown) as MlPluginSetup; + + registerTransactionDurationAnomalyAlertType({ + alerts, + ml, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const scheduleActions = jest.fn(); + const services = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + }; + const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; + + await alertExecutor!({ services, params }); + + await alertExecutor!({ services, params }); + [ + 'apm.transaction_duration_anomaly_foo_production_type-foo', + 'apm.transaction_duration_anomaly_bar_production_type-bar', + ].forEach((instanceName) => + expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + ); + + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'foo', + transactionType: 'type-foo', + environment: 'production', + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'bar', + transactionType: 'type-bar', + environment: 'production', + }); + }); + + it('with service name', async () => { + jest.spyOn(GetServiceAnomalies, 'getMLJobs').mockReturnValue( + Promise.resolve([ + { + job_id: '1', + custom_settings: { + job_tags: { + environment: 'production', + }, + }, + } as unknown, + { + job_id: '2', + custom_settings: { + job_tags: { + environment: 'testing', + }, + }, + } as unknown, + ] as Job[]) + ); + + let alertExecutor: any; + + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + const ml = ({ + mlSystemProvider: () => ({ + mlAnomalySearch: () => ({ + hits: { total: { value: 2 } }, + aggregations: { + services: { + buckets: [{ key: 'foo' }, { key: 'bar' }], + }, + }, + }), + }), + anomalyDetectorsProvider: jest.fn(), + } as unknown) as MlPluginSetup; + + registerTransactionDurationAnomalyAlertType({ + alerts, + ml, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const scheduleActions = jest.fn(); + const services = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + }; + const params = { anomalySeverityType: ANOMALY_SEVERITY.MINOR }; + + await alertExecutor!({ services, params }); + + await alertExecutor!({ services, params }); + [ + 'apm.transaction_duration_anomaly_foo_production', + 'apm.transaction_duration_anomaly_foo_testing', + 'apm.transaction_duration_anomaly_bar_production', + 'apm.transaction_duration_anomaly_bar_testing', + ].forEach((instanceName) => + expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + ); + + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'foo', + transactionType: undefined, + environment: 'production', + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'bar', + transactionType: undefined, + environment: 'production', + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'foo', + transactionType: undefined, + environment: 'testing', + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'bar', + transactionType: undefined, + environment: 'testing', + }); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts index 61cd79b672735..36b7964e8128d 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts @@ -6,6 +6,7 @@ import { schema } from '@kbn/config-schema'; import { Observable } from 'rxjs'; +import { isEmpty } from 'lodash'; import { ANOMALY_SEVERITY } from '../../../../ml/common'; import { KibanaRequest } from '../../../../../../src/core/server'; import { @@ -16,7 +17,7 @@ import { import { AlertingPlugin } from '../../../../alerts/server'; import { APMConfig } from '../..'; import { MlPluginSetup } from '../../../../ml/server'; -import { getMLJobIds } from '../service_map/get_service_anomalies'; +import { getMLJobs } from '../service_map/get_service_anomalies'; import { apmActionVariables } from './action_variables'; interface RegisterAlertParams { @@ -26,8 +27,8 @@ interface RegisterAlertParams { } const paramsSchema = schema.object({ - serviceName: schema.string(), - transactionType: schema.string(), + serviceName: schema.maybe(schema.string()), + transactionType: schema.maybe(schema.string()), windowSize: schema.number(), windowUnit: schema.string(), environment: schema.string(), @@ -72,10 +73,7 @@ export function registerTransactionDurationAnomalyAlertType({ const { mlAnomalySearch } = ml.mlSystemProvider(request); const anomalyDetectors = ml.anomalyDetectorsProvider(request); - const mlJobIds = await getMLJobIds( - anomalyDetectors, - alertParams.environment - ); + const mlJobs = await getMLJobs(anomalyDetectors, alertParams.environment); const selectedOption = ANOMALY_ALERT_SEVERITY_TYPES.find( (option) => option.type === alertParams.anomalySeverityType @@ -89,19 +87,19 @@ export function registerTransactionDurationAnomalyAlertType({ const threshold = selectedOption.threshold; - if (mlJobIds.length === 0) { + if (mlJobs.length === 0) { return {}; } const anomalySearchParams = { + terminateAfter: 1, body: { - terminateAfter: 1, size: 0, query: { bool: { filter: [ { term: { result_type: 'record' } }, - { terms: { job_id: mlJobIds } }, + { terms: { job_id: mlJobs.map((job) => job.job_id) } }, { range: { timestamp: { @@ -110,11 +108,24 @@ export function registerTransactionDurationAnomalyAlertType({ }, }, }, - { - term: { - partition_field_value: alertParams.serviceName, - }, - }, + ...(alertParams.serviceName + ? [ + { + term: { + partition_field_value: alertParams.serviceName, + }, + }, + ] + : []), + ...(alertParams.transactionType + ? [ + { + term: { + by_field_value: alertParams.transactionType, + }, + }, + ] + : []), { range: { record_score: { @@ -125,22 +136,82 @@ export function registerTransactionDurationAnomalyAlertType({ ], }, }, + aggs: { + services: { + terms: { + field: 'partition_field_value', + size: 50, + }, + aggs: { + transaction_types: { + terms: { + field: 'by_field_value', + }, + }, + }, + }, + }, }, }; const response = ((await mlAnomalySearch( anomalySearchParams - )) as unknown) as { hits: { total: { value: number } } }; + )) as unknown) as { + hits: { total: { value: number } }; + aggregations?: { + services: { + buckets: Array<{ + key: string; + transaction_types: { buckets: Array<{ key: string }> }; + }>; + }; + }; + }; + const hitCount = response.hits.total.value; if (hitCount > 0) { - const alertInstance = services.alertInstanceFactory( - AlertType.TransactionDurationAnomaly - ); - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { - serviceName: alertParams.serviceName, - transactionType: alertParams.transactionType, - environment: alertParams.environment, + function scheduleAction({ + serviceName, + environment, + transactionType, + }: { + serviceName: string; + environment?: string; + transactionType?: string; + }) { + const alertInstanceName = [ + AlertType.TransactionDurationAnomaly, + serviceName, + environment, + transactionType, + ] + .filter((name) => name) + .join('_'); + + const alertInstance = services.alertInstanceFactory( + alertInstanceName + ); + alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + environment, + transactionType, + }); + } + + mlJobs.map((job) => { + const environment = job.custom_settings?.job_tags?.environment; + response.aggregations?.services.buckets.forEach((serviceBucket) => { + const serviceName = serviceBucket.key as string; + if (isEmpty(serviceBucket.transaction_types?.buckets)) { + scheduleAction({ serviceName, environment }); + } else { + serviceBucket.transaction_types?.buckets.forEach((typeBucket) => { + const transactionType = typeBucket.key as string; + scheduleAction({ serviceName, environment, transactionType }); + }); + } + }); }); } }, diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts new file mode 100644 index 0000000000000..90db48f84b5d9 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.test.ts @@ -0,0 +1,289 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Observable } from 'rxjs'; +import * as Rx from 'rxjs'; +import { toArray, map } from 'rxjs/operators'; +import { AlertingPlugin } from '../../../../alerts/server'; +import { APMConfig } from '../..'; +import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type'; + +type Operator = (source: Rx.Observable) => Rx.Observable; +const pipeClosure = (fn: Operator): Operator => { + return (source: Rx.Observable) => { + return Rx.defer(() => fn(source)); + }; +}; +const mockedConfig$ = (Rx.of('apm_oss.errorIndices').pipe( + pipeClosure((source$) => { + return source$.pipe(map((i) => i)); + }), + toArray() +) as unknown) as Observable; + +describe('Transaction error rate alert', () => { + it("doesn't send an alert when rate is less than threshold", async () => { + let alertExecutor: any; + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + registerTransactionErrorRateAlertType({ + alerts, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const services = { + callCluster: jest.fn(() => ({ + hits: { + total: { + value: 0, + }, + }, + })), + alertInstanceFactory: jest.fn(), + }; + const params = { threshold: 1 }; + + await alertExecutor!({ services, params }); + expect(services.alertInstanceFactory).not.toBeCalled(); + }); + + it('sends alerts with service name, transaction type and environment', async () => { + let alertExecutor: any; + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + registerTransactionErrorRateAlertType({ + alerts, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const scheduleActions = jest.fn(); + const services = { + callCluster: jest.fn(() => ({ + hits: { + total: { + value: 4, + }, + }, + aggregations: { + erroneous_transactions: { + doc_count: 2, + }, + services: { + buckets: [ + { + key: 'foo', + transaction_types: { + buckets: [ + { + key: 'type-foo', + environments: { + buckets: [{ key: 'env-foo' }, { key: 'env-foo-2' }], + }, + }, + ], + }, + }, + { + key: 'bar', + transaction_types: { + buckets: [ + { + key: 'type-bar', + environments: { + buckets: [{ key: 'env-bar' }, { key: 'env-bar-2' }], + }, + }, + ], + }, + }, + ], + }, + }, + })), + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + }; + const params = { threshold: 10 }; + + await alertExecutor!({ services, params }); + [ + 'apm.transaction_error_rate_foo_type-foo_env-foo', + 'apm.transaction_error_rate_foo_type-foo_env-foo-2', + 'apm.transaction_error_rate_bar_type-bar_env-bar', + 'apm.transaction_error_rate_bar_type-bar_env-bar-2', + ].forEach((instanceName) => + expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + ); + + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'foo', + transactionType: 'type-foo', + environment: 'env-foo', + threshold: 10, + triggerValue: 50, + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'foo', + transactionType: 'type-foo', + environment: 'env-foo-2', + threshold: 10, + triggerValue: 50, + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'bar', + transactionType: 'type-bar', + environment: 'env-bar', + threshold: 10, + triggerValue: 50, + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'bar', + transactionType: 'type-bar', + environment: 'env-bar-2', + threshold: 10, + triggerValue: 50, + }); + }); + it('sends alerts with service name and transaction type', async () => { + let alertExecutor: any; + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + registerTransactionErrorRateAlertType({ + alerts, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const scheduleActions = jest.fn(); + const services = { + callCluster: jest.fn(() => ({ + hits: { + total: { + value: 4, + }, + }, + aggregations: { + erroneous_transactions: { + doc_count: 2, + }, + services: { + buckets: [ + { + key: 'foo', + transaction_types: { + buckets: [{ key: 'type-foo' }], + }, + }, + { + key: 'bar', + transaction_types: { + buckets: [{ key: 'type-bar' }], + }, + }, + ], + }, + }, + })), + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + }; + const params = { threshold: 10 }; + + await alertExecutor!({ services, params }); + [ + 'apm.transaction_error_rate_foo_type-foo', + 'apm.transaction_error_rate_bar_type-bar', + ].forEach((instanceName) => + expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + ); + + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'foo', + transactionType: 'type-foo', + environment: undefined, + threshold: 10, + triggerValue: 50, + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'bar', + transactionType: 'type-bar', + environment: undefined, + threshold: 10, + triggerValue: 50, + }); + }); + + it('sends alerts with service name', async () => { + let alertExecutor: any; + const alerts = { + registerType: ({ executor }) => { + alertExecutor = executor; + }, + } as AlertingPlugin['setup']; + + registerTransactionErrorRateAlertType({ + alerts, + config$: mockedConfig$, + }); + expect(alertExecutor).toBeDefined(); + + const scheduleActions = jest.fn(); + const services = { + callCluster: jest.fn(() => ({ + hits: { + total: { + value: 4, + }, + }, + aggregations: { + erroneous_transactions: { + doc_count: 2, + }, + services: { + buckets: [{ key: 'foo' }, { key: 'bar' }], + }, + }, + })), + alertInstanceFactory: jest.fn(() => ({ scheduleActions })), + }; + const params = { threshold: 10 }; + + await alertExecutor!({ services, params }); + [ + 'apm.transaction_error_rate_foo', + 'apm.transaction_error_rate_bar', + ].forEach((instanceName) => + expect(services.alertInstanceFactory).toHaveBeenCalledWith(instanceName) + ); + + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'foo', + transactionType: undefined, + environment: undefined, + threshold: 10, + triggerValue: 50, + }); + expect(scheduleActions).toHaveBeenCalledWith('threshold_met', { + serviceName: 'bar', + transactionType: undefined, + environment: undefined, + threshold: 10, + triggerValue: 50, + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts index a6ed40fc15ec6..e14360029e5dd 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts @@ -7,6 +7,7 @@ import { schema } from '@kbn/config-schema'; import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; +import { isEmpty } from 'lodash'; import { ProcessorEvent } from '../../../common/processor_event'; import { EventOutcome } from '../../../common/event_outcome'; import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; @@ -16,6 +17,7 @@ import { SERVICE_NAME, TRANSACTION_TYPE, EVENT_OUTCOME, + SERVICE_ENVIRONMENT, } from '../../../common/elasticsearch_fieldnames'; import { AlertingPlugin } from '../../../../alerts/server'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; @@ -32,8 +34,8 @@ const paramsSchema = schema.object({ windowSize: schema.number(), windowUnit: schema.string(), threshold: schema.number(), - transactionType: schema.string(), - serviceName: schema.string(), + transactionType: schema.maybe(schema.string()), + serviceName: schema.maybe(schema.string()), environment: schema.string(), }); @@ -84,8 +86,18 @@ export function registerTransactionErrorRateAlertType({ }, }, { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - { term: { [SERVICE_NAME]: alertParams.serviceName } }, - { term: { [TRANSACTION_TYPE]: alertParams.transactionType } }, + ...(alertParams.serviceName + ? [{ term: { [SERVICE_NAME]: alertParams.serviceName } }] + : []), + ...(alertParams.transactionType + ? [ + { + term: { + [TRANSACTION_TYPE]: alertParams.transactionType, + }, + }, + ] + : []), ...getEnvironmentUiFilterES(alertParams.environment), ], }, @@ -94,6 +106,24 @@ export function registerTransactionErrorRateAlertType({ erroneous_transactions: { filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } }, }, + services: { + terms: { + field: SERVICE_NAME, + size: 50, + }, + aggs: { + transaction_types: { + terms: { field: TRANSACTION_TYPE }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + }, + }, + }, + }, + }, + }, }, }, }; @@ -114,16 +144,53 @@ export function registerTransactionErrorRateAlertType({ (errornousTransactionsCount / totalTransactionCount) * 100; if (transactionErrorRate > alertParams.threshold) { - const alertInstance = services.alertInstanceFactory( - AlertType.TransactionErrorRate - ); + function scheduleAction({ + serviceName, + environment, + transactionType, + }: { + serviceName: string; + environment?: string; + transactionType?: string; + }) { + const alertInstanceName = [ + AlertType.TransactionErrorRate, + serviceName, + transactionType, + environment, + ] + .filter((name) => name) + .join('_'); + + const alertInstance = services.alertInstanceFactory( + alertInstanceName + ); + alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { + serviceName, + transactionType, + environment, + threshold: alertParams.threshold, + triggerValue: transactionErrorRate, + }); + } - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, { - serviceName: alertParams.serviceName, - transactionType: alertParams.transactionType, - environment: alertParams.environment, - threshold: alertParams.threshold, - triggerValue: transactionErrorRate, + response.aggregations?.services.buckets.forEach((serviceBucket) => { + const serviceName = serviceBucket.key as string; + if (isEmpty(serviceBucket.transaction_types?.buckets)) { + scheduleAction({ serviceName }); + } else { + serviceBucket.transaction_types.buckets.forEach((typeBucket) => { + const transactionType = typeBucket.key as string; + if (isEmpty(typeBucket.environments?.buckets)) { + scheduleAction({ serviceName, transactionType }); + } else { + typeBucket.environments.buckets.forEach((envBucket) => { + const environment = envBucket.key as string; + scheduleAction({ serviceName, transactionType, environment }); + }); + } + }); + } }); } }, diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts index 44c0c96142096..895fc70d76af1 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts @@ -180,7 +180,7 @@ function transformResponseToServiceAnomalies( return serviceAnomaliesMap; } -export async function getMLJobIds( +export async function getMLJobs( anomalyDetectors: ReturnType, environment?: string ) { @@ -198,7 +198,15 @@ export async function getMLJobIds( if (!matchingMLJob) { return []; } - return [matchingMLJob.job_id]; + return [matchingMLJob]; } + return mlJobs; +} + +export async function getMLJobIds( + anomalyDetectors: ReturnType, + environment?: string +) { + const mlJobs = await getMLJobs(anomalyDetectors, environment); return mlJobs.map((job) => job.job_id); } From 53d49381c84df5f7eeae5d912917324d3c4333b4 Mon Sep 17 00:00:00 2001 From: Daniil Suleiman <31325372+sulemanof@users.noreply.github.com> Date: Mon, 28 Sep 2020 16:14:30 +0300 Subject: [PATCH 08/11] Implement tagcloud renderer (#77910) * Implement toExpressionAst for tagcloud * Implement tagcloud vis renderer * Use resize observer * Use common no data message * Update build_pipeline.test * Update tag cloud tests * Revert "Use common no data message" This reverts commit fddf019575f4e22aced1ef1f262d8b499d0e8da7. * Update interpreter functional tests * Add tests for toExpressionAst fn * Use throttled chart update * Update renderer Co-authored-by: Elastic Machine --- .../__snapshots__/tag_cloud_fn.test.ts.snap | 31 ++- .../public/__snapshots__/to_ast.test.ts.snap | 171 ++++++++++++++ .../vis_type_tagcloud/public/_tag_cloud.scss | 14 -- .../public/components/label.js | 2 +- .../public/components/tag_cloud.scss | 26 +++ .../public/components/tag_cloud_chart.tsx | 84 +++++++ .../components/tag_cloud_visualization.js | 216 +++++++++--------- .../tag_cloud_visualization.test.js | 77 ++----- .../vis_type_tagcloud/public/index.scss | 8 - .../vis_type_tagcloud/public/plugin.ts | 10 +- .../vis_type_tagcloud/public/tag_cloud_fn.ts | 26 +-- .../public/tag_cloud_type.ts | 14 +- .../public/tag_cloud_vis_renderer.tsx | 54 +++++ .../vis_type_tagcloud/public/to_ast.test.ts | 84 +++++++ .../vis_type_tagcloud/public/to_ast.ts | 60 +++++ src/plugins/visualizations/public/index.ts | 2 +- .../__snapshots__/build_pipeline.test.ts.snap | 6 - .../public/legacy/build_pipeline.test.ts | 22 -- .../public/legacy/build_pipeline.ts | 60 +---- src/plugins/visualizations/public/vis.ts | 4 +- .../snapshots/baseline/partial_test_1.json | 2 +- .../snapshots/baseline/tagcloud_all_data.json | 2 +- .../snapshots/baseline/tagcloud_fontsize.json | 2 +- .../baseline/tagcloud_metric_data.json | 2 +- .../snapshots/baseline/tagcloud_options.json | 2 +- .../snapshots/session/partial_test_1.json | 2 +- .../snapshots/session/tagcloud_all_data.json | 2 +- .../snapshots/session/tagcloud_fontsize.json | 2 +- .../session/tagcloud_metric_data.json | 2 +- .../snapshots/session/tagcloud_options.json | 2 +- 30 files changed, 674 insertions(+), 317 deletions(-) create mode 100644 src/plugins/vis_type_tagcloud/public/__snapshots__/to_ast.test.ts.snap delete mode 100644 src/plugins/vis_type_tagcloud/public/_tag_cloud.scss create mode 100644 src/plugins/vis_type_tagcloud/public/components/tag_cloud.scss create mode 100644 src/plugins/vis_type_tagcloud/public/components/tag_cloud_chart.tsx delete mode 100644 src/plugins/vis_type_tagcloud/public/index.scss create mode 100644 src/plugins/vis_type_tagcloud/public/tag_cloud_vis_renderer.tsx create mode 100644 src/plugins/vis_type_tagcloud/public/to_ast.test.ts create mode 100644 src/plugins/vis_type_tagcloud/public/to_ast.ts diff --git a/src/plugins/vis_type_tagcloud/public/__snapshots__/tag_cloud_fn.test.ts.snap b/src/plugins/vis_type_tagcloud/public/__snapshots__/tag_cloud_fn.test.ts.snap index 8e28be33515f7..debc7ab27c632 100644 --- a/src/plugins/vis_type_tagcloud/public/__snapshots__/tag_cloud_fn.test.ts.snap +++ b/src/plugins/vis_type_tagcloud/public/__snapshots__/tag_cloud_fn.test.ts.snap @@ -2,25 +2,9 @@ exports[`interpreter/functions#tagcloud returns an object with the correct structure 1`] = ` Object { - "as": "visualization", + "as": "tagloud_vis", "type": "render", "value": Object { - "params": Object { - "listenOnChange": true, - }, - "visConfig": Object { - "maxFontSize": 72, - "metric": Object { - "accessor": 0, - "format": Object { - "id": "number", - }, - }, - "minFontSize": 18, - "orientation": "single", - "scale": "linear", - "showLabel": true, - }, "visData": Object { "columns": Array [ Object { @@ -35,6 +19,19 @@ Object { ], "type": "kibana_datatable", }, + "visParams": Object { + "maxFontSize": 72, + "metric": Object { + "accessor": 0, + "format": Object { + "id": "number", + }, + }, + "minFontSize": 18, + "orientation": "single", + "scale": "linear", + "showLabel": true, + }, "visType": "tagcloud", }, } diff --git a/src/plugins/vis_type_tagcloud/public/__snapshots__/to_ast.test.ts.snap b/src/plugins/vis_type_tagcloud/public/__snapshots__/to_ast.test.ts.snap new file mode 100644 index 0000000000000..d64bdfb1f46f9 --- /dev/null +++ b/src/plugins/vis_type_tagcloud/public/__snapshots__/to_ast.test.ts.snap @@ -0,0 +1,171 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`tagcloud vis toExpressionAst function should match snapshot params fulfilled 1`] = ` +Object { + "chain": Array [ + Object { + "arguments": Object { + "aggConfigs": Array [ + "[]", + ], + "includeFormatHints": Array [ + false, + ], + "index": Array [ + "123", + ], + "metricsAtAllLevels": Array [ + false, + ], + "partialRows": Array [ + false, + ], + }, + "function": "esaggs", + "type": "function", + }, + Object { + "arguments": Object { + "bucket": Array [ + Object { + "chain": Array [ + Object { + "arguments": Object { + "accessor": Array [ + 0, + ], + "format": Array [ + "terms", + ], + "formatParams": Array [ + "{\\"id\\":\\"string\\",\\"otherBucketLabel\\":\\"Other\\",\\"missingBucketLabel\\":\\"Missing\\"}", + ], + }, + "function": "visdimension", + "type": "function", + }, + ], + "type": "expression", + }, + ], + "maxFontSize": Array [ + 15, + ], + "metric": Array [ + Object { + "chain": Array [ + Object { + "arguments": Object { + "accessor": Array [ + 1, + ], + "format": Array [ + "number", + ], + }, + "function": "visdimension", + "type": "function", + }, + ], + "type": "expression", + }, + ], + "minFontSize": Array [ + 5, + ], + "orientation": Array [ + "single", + ], + "scale": Array [ + "linear", + ], + "showLabel": Array [ + true, + ], + }, + "function": "tagcloud", + "type": "function", + }, + ], + "type": "expression", +} +`; + +exports[`tagcloud vis toExpressionAst function should match snapshot without params 1`] = ` +Object { + "chain": Array [ + Object { + "arguments": Object { + "aggConfigs": Array [ + "[]", + ], + "includeFormatHints": Array [ + false, + ], + "index": Array [ + "123", + ], + "metricsAtAllLevels": Array [ + false, + ], + "partialRows": Array [ + false, + ], + }, + "function": "esaggs", + "type": "function", + }, + Object { + "arguments": Object { + "bucket": Array [ + Object { + "chain": Array [ + Object { + "arguments": Object { + "accessor": Array [ + 0, + ], + "format": Array [ + "terms", + ], + "formatParams": Array [ + "{\\"id\\":\\"string\\",\\"otherBucketLabel\\":\\"Other\\",\\"missingBucketLabel\\":\\"Missing\\"}", + ], + }, + "function": "visdimension", + "type": "function", + }, + ], + "type": "expression", + }, + ], + "metric": Array [ + Object { + "chain": Array [ + Object { + "arguments": Object { + "accessor": Array [ + 1, + ], + "format": Array [ + "number", + ], + }, + "function": "visdimension", + "type": "function", + }, + ], + "type": "expression", + }, + ], + "showLabel": Array [ + false, + ], + }, + "function": "tagcloud", + "type": "function", + }, + ], + "type": "expression", +} +`; diff --git a/src/plugins/vis_type_tagcloud/public/_tag_cloud.scss b/src/plugins/vis_type_tagcloud/public/_tag_cloud.scss deleted file mode 100644 index 08901bebc0349..0000000000000 --- a/src/plugins/vis_type_tagcloud/public/_tag_cloud.scss +++ /dev/null @@ -1,14 +0,0 @@ -.tgcVis { - position: absolute; - top: 0; - left: 0; - width: 100%; - height: 100%; - overflow: hidden; -} - -.tgcVisLabel { - width: 100%; - text-align: center; - font-weight: $euiFontWeightBold; -} diff --git a/src/plugins/vis_type_tagcloud/public/components/label.js b/src/plugins/vis_type_tagcloud/public/components/label.js index 168ec4b270fde..88b3c2f851138 100644 --- a/src/plugins/vis_type_tagcloud/public/components/label.js +++ b/src/plugins/vis_type_tagcloud/public/components/label.js @@ -28,7 +28,7 @@ export class Label extends Component { render() { return (
{this.state.label} diff --git a/src/plugins/vis_type_tagcloud/public/components/tag_cloud.scss b/src/plugins/vis_type_tagcloud/public/components/tag_cloud.scss new file mode 100644 index 0000000000000..37867f1ed1c17 --- /dev/null +++ b/src/plugins/vis_type_tagcloud/public/components/tag_cloud.scss @@ -0,0 +1,26 @@ +// Prefix all styles with "tgc" to avoid conflicts. +// Examples +// tgcChart +// tgcChart__legend +// tgcChart__legend--small +// tgcChart__legend-isLoading + +.tgcChart__container, .tgcChart__wrapper { + flex: 1 1 0; + display: flex; +} + +.tgcChart { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + overflow: hidden; +} + +.tgcChart__label { + width: 100%; + text-align: center; + font-weight: $euiFontWeightBold; +} diff --git a/src/plugins/vis_type_tagcloud/public/components/tag_cloud_chart.tsx b/src/plugins/vis_type_tagcloud/public/components/tag_cloud_chart.tsx new file mode 100644 index 0000000000000..18a09ec9f4969 --- /dev/null +++ b/src/plugins/vis_type_tagcloud/public/components/tag_cloud_chart.tsx @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React, { useEffect, useMemo, useRef } from 'react'; +import { EuiResizeObserver } from '@elastic/eui'; +import { throttle } from 'lodash'; + +import { TagCloudVisDependencies } from '../plugin'; +import { TagCloudVisRenderValue } from '../tag_cloud_fn'; +// @ts-ignore +import { TagCloudVisualization } from './tag_cloud_visualization'; + +import './tag_cloud.scss'; + +type TagCloudChartProps = TagCloudVisDependencies & + TagCloudVisRenderValue & { + fireEvent: (event: any) => void; + renderComplete: () => void; + }; + +export const TagCloudChart = ({ + colors, + visData, + visParams, + fireEvent, + renderComplete, +}: TagCloudChartProps) => { + const chartDiv = useRef(null); + const visController = useRef(null); + + useEffect(() => { + visController.current = new TagCloudVisualization(chartDiv.current, colors, fireEvent); + return () => { + visController.current.destroy(); + visController.current = null; + }; + }, [colors, fireEvent]); + + useEffect(() => { + if (visController.current) { + visController.current.render(visData, visParams).then(renderComplete); + } + }, [visData, visParams, renderComplete]); + + const updateChartSize = useMemo( + () => + throttle(() => { + if (visController.current) { + visController.current.render().then(renderComplete); + } + }, 300), + [renderComplete] + ); + + return ( + + {(resizeRef) => ( +
+
+
+ )} + + ); +}; + +// default export required for React.Lazy +// eslint-disable-next-line import/no-default-export +export { TagCloudChart as default }; diff --git a/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.js b/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.js index e43b3bdc747ab..5ec22d2c6a4d9 100644 --- a/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.js +++ b/src/plugins/vis_type_tagcloud/public/components/tag_cloud_visualization.js @@ -32,126 +32,138 @@ import d3 from 'd3'; const MAX_TAG_COUNT = 200; -export function createTagCloudVisualization({ colors }) { - const colorScale = d3.scale.ordinal().range(colors.seedColors); - return class TagCloudVisualization { - constructor(node, vis) { - this._containerNode = node; - - const cloudRelativeContainer = document.createElement('div'); - cloudRelativeContainer.classList.add('tgcVis'); - cloudRelativeContainer.setAttribute('style', 'position: relative'); - const cloudContainer = document.createElement('div'); - cloudContainer.classList.add('tgcVis'); - cloudContainer.setAttribute('data-test-subj', 'tagCloudVisualization'); - this._containerNode.classList.add('visChart--vertical'); - cloudRelativeContainer.appendChild(cloudContainer); - this._containerNode.appendChild(cloudRelativeContainer); - - this._vis = vis; - this._truncated = false; - this._tagCloud = new TagCloud(cloudContainer, colorScale); - this._tagCloud.on('select', (event) => { - if (!this._visParams.bucket) { - return; - } - this._vis.API.events.filter({ - table: event.meta.data, - column: 0, - row: event.meta.rowIndex, - }); - }); - this._renderComplete$ = Rx.fromEvent(this._tagCloud, 'renderComplete'); - - this._feedbackNode = document.createElement('div'); - this._containerNode.appendChild(this._feedbackNode); - this._feedbackMessage = React.createRef(); - render( - - - , - this._feedbackNode - ); - - this._labelNode = document.createElement('div'); - this._containerNode.appendChild(this._labelNode); - this._label = React.createRef(); - render(