From 3153dff96b8d213858e9456282866771165b3657 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 9 Jul 2020 14:58:11 -0700 Subject: [PATCH 1/5] fix visual regression job (#70999) --- test/scripts/jenkins_xpack_visual_regression.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scripts/jenkins_xpack_visual_regression.sh b/test/scripts/jenkins_xpack_visual_regression.sh index 930d4a74345d90..726af43f28e31a 100755 --- a/test/scripts/jenkins_xpack_visual_regression.sh +++ b/test/scripts/jenkins_xpack_visual_regression.sh @@ -5,7 +5,7 @@ source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh" echo " -> building and extracting default Kibana distributable" cd "$KIBANA_DIR" -node scripts/build --debug +node scripts/build --debug --no-oss linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" installDir="$PARENT_DIR/install/kibana" mkdir -p "$installDir" From 8e2277a667883b1aedc2c4ba46ec672a154c6c90 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 9 Jul 2020 15:04:51 -0700 Subject: [PATCH 2/5] send slack notifications on visual baseline failures --- .ci/Jenkinsfile_visual_baseline | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/Jenkinsfile_visual_baseline b/.ci/Jenkinsfile_visual_baseline index 2a16c499fa168c..7c7cc8d98c306e 100644 --- a/.ci/Jenkinsfile_visual_baseline +++ b/.ci/Jenkinsfile_visual_baseline @@ -21,5 +21,6 @@ kibanaPipeline(timeoutMinutes: 120) { } kibanaPipeline.sendMail() + slackNotifications.onFailure() } } From d570ab1eb27b1d876a8eb099bea84f234fb0199c Mon Sep 17 00:00:00 2001 From: Dmitry Lemeshko Date: Fri, 10 Jul 2020 00:50:27 +0200 Subject: [PATCH 3/5] [services/remote/webdriver] fix eslint error (#71346) --- test/functional/services/remote/webdriver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functional/services/remote/webdriver.ts b/test/functional/services/remote/webdriver.ts index 867e78dfad8dcb..0611c80f59b92f 100644 --- a/test/functional/services/remote/webdriver.ts +++ b/test/functional/services/remote/webdriver.ts @@ -28,7 +28,7 @@ import { delay } from 'bluebird'; import chromeDriver from 'chromedriver'; // @ts-ignore types not available import geckoDriver from 'geckodriver'; -import { Builder, Capabilities, logging } from 'selenium-webdriver'; +import { Builder, logging } from 'selenium-webdriver'; import chrome from 'selenium-webdriver/chrome'; import firefox from 'selenium-webdriver/firefox'; import edge from 'selenium-webdriver/edge'; From c9e8650a216635aec08399174cfe72f991e1fb3c Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Thu, 9 Jul 2020 18:57:46 -0400 Subject: [PATCH 4/5] Deduplication of entries and items before sending to endpoint (#71297) * Deduplication of entries and items before sending to endpoint * Renaming --- .../endpoint/lib/artifacts/lists.test.ts | 139 +++++++++++++++++- .../server/endpoint/lib/artifacts/lists.ts | 19 ++- 2 files changed, 153 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index acde455f77cb4b..1a19306b2fd606 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -139,6 +139,139 @@ describe('buildEventTypeSignal', () => { }); }); + test('it should deduplicate exception entries', async () => { + const testEntries: EntriesArray = [ + { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' }, + { + field: 'host.hostname.text', + operator: 'included', + type: 'match_any', + value: ['estc', 'kibana'], + }, + ]; + + const expectedEndpointExceptions = { + type: 'simple', + entries: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_caseless', + value: 'DOMAIN', + }, + { + field: 'server.ip', + operator: 'included', + type: 'exact_cased', + value: '192.168.1.1', + }, + { + field: 'host.hostname', + operator: 'included', + type: 'exact_caseless_any', + value: ['estc', 'kibana'], + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); + }); + + test('it should not deduplicate exception entries across nested boundaries', async () => { + const testEntries: EntriesArray = [ + { + entries: [ + { field: 'nested.field', operator: 'included', type: 'match', value: 'some value' }, + ], + field: 'some.parentField', + type: 'nested', + }, + // Same as above but not inside the nest + { field: 'nested.field', operator: 'included', type: 'match', value: 'some value' }, + ]; + + const expectedEndpointExceptions = { + type: 'simple', + entries: [ + { + entries: [ + { + field: 'nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.parentField', + type: 'nested', + }, + { + field: 'nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); + }); + + test('it should deduplicate exception items', async () => { + const testEntries: EntriesArray = [ + { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' }, + ]; + + const expectedEndpointExceptions = { + type: 'simple', + entries: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_caseless', + value: 'DOMAIN', + }, + { + field: 'server.ip', + operator: 'included', + type: 'exact_cased', + value: '192.168.1.1', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + + // Create a second exception item with the same entries + first.data[1] = getExceptionListItemSchemaMock(); + first.data[1].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); + }); + test('it should ignore unsupported entries', async () => { // Lists and exists are not supported by the Endpoint const testEntries: EntriesArray = [ @@ -178,8 +311,9 @@ describe('buildEventTypeSignal', () => { }); test('it should convert the exception lists response to the proper endpoint format while paging', async () => { - // The first call returns one exception + // The first call returns two exceptions const first = getFoundExceptionListItemSchemaMock(); + first.data.push(getExceptionListItemSchemaMock()); // The second call returns two exceptions const second = getFoundExceptionListItemSchemaMock(); @@ -194,7 +328,8 @@ describe('buildEventTypeSignal', () => { .mockReturnValueOnce(second) .mockReturnValueOnce(third); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); - expect(resp.entries.length).toEqual(3); + // Expect 2 exceptions, the first two calls returned the same exception list items + expect(resp.entries.length).toEqual(2); }); test('it should handle no exceptions', async () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 556405adff62f9..b756c4e3d14c33 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -97,10 +97,18 @@ export function translateToEndpointExceptions( exc: FoundExceptionListItemSchema, schemaVersion: string ): TranslatedExceptionListItem[] { + const entrySet = new Set(); + const entriesFiltered: TranslatedExceptionListItem[] = []; if (schemaVersion === 'v1') { - return exc.data.map((item) => { - return translateItem(schemaVersion, item); + exc.data.forEach((entry) => { + const translatedItem = translateItem(schemaVersion, entry); + const entryHash = createHash('sha256').update(JSON.stringify(translatedItem)).digest('hex'); + if (!entrySet.has(entryHash)) { + entriesFiltered.push(translatedItem); + entrySet.add(entryHash); + } }); + return entriesFiltered; } else { throw new Error('unsupported schemaVersion'); } @@ -124,12 +132,17 @@ function translateItem( schemaVersion: string, item: ExceptionListItemSchema ): TranslatedExceptionListItem { + const itemSet = new Set(); return { type: item.type, entries: item.entries.reduce((translatedEntries: TranslatedEntry[], entry) => { const translatedEntry = translateEntry(schemaVersion, entry); if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) { - translatedEntries.push(translatedEntry); + const itemHash = createHash('sha256').update(JSON.stringify(translatedEntry)).digest('hex'); + if (!itemSet.has(itemHash)) { + translatedEntries.push(translatedEntry); + itemSet.add(itemHash); + } } return translatedEntries; }, []), From 589a891bb3829c1b8b00ead2f1ce18ee74916341 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 9 Jul 2020 15:58:39 -0700 Subject: [PATCH 5/5] jenkins_xpack_saved_objects_field_metrics.sh expects to be run from the KIBANA_DIR in CI --- test/scripts/jenkins_xpack_visual_regression.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/scripts/jenkins_xpack_visual_regression.sh b/test/scripts/jenkins_xpack_visual_regression.sh index 726af43f28e31a..ac567a188a6d40 100755 --- a/test/scripts/jenkins_xpack_visual_regression.sh +++ b/test/scripts/jenkins_xpack_visual_regression.sh @@ -22,5 +22,5 @@ yarn percy exec -t 10000 -- -- \ # cd "$KIBANA_DIR" # source "test/scripts/jenkins_xpack_page_load_metrics.sh" -cd "$XPACK_DIR" -source "$KIBANA_DIR/test/scripts/jenkins_xpack_saved_objects_field_metrics.sh" +cd "$KIBANA_DIR" +source "test/scripts/jenkins_xpack_saved_objects_field_metrics.sh"