Merge branch 'master' into fix/form-lib-memoize

.ci/Jenkinsfile_visual_baseline

@@ -21,5 +21,6 @@ kibanaPipeline(timeoutMinutes: 120) {
  }
 
  kibanaPipeline.sendMail()
+ slackNotifications.onFailure()
  }
 }

test/functional/services/remote/webdriver.ts

@@ -28,7 +28,7 @@ import { delay } from 'bluebird';
 import chromeDriver from 'chromedriver';
 // @ts-ignore types not available
 import geckoDriver from 'geckodriver';
-import { Builder, Capabilities, logging } from 'selenium-webdriver';
+import { Builder, logging } from 'selenium-webdriver';
 import chrome from 'selenium-webdriver/chrome';
 import firefox from 'selenium-webdriver/firefox';
 import edge from 'selenium-webdriver/edge';

test/scripts/jenkins_xpack_visual_regression.sh

@@ -5,7 +5,7 @@ source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh"
 
 echo " -> building and extracting default Kibana distributable"
 cd "$KIBANA_DIR"
-node scripts/build --debug
+node scripts/build --debug --no-oss
 linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
 installDir="$PARENT_DIR/install/kibana"
 mkdir -p "$installDir"
@@ -22,5 +22,5 @@ yarn percy exec -t 10000 -- -- \
 # cd "$KIBANA_DIR"
 # source "test/scripts/jenkins_xpack_page_load_metrics.sh"
 
-cd "$XPACK_DIR"
-source "$KIBANA_DIR/test/scripts/jenkins_xpack_saved_objects_field_metrics.sh"
+cd "$KIBANA_DIR"
+source "test/scripts/jenkins_xpack_saved_objects_field_metrics.sh"

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts

@@ -139,6 +139,139 @@ describe('buildEventTypeSignal', () => {
  });
  });
 
+ test('it should deduplicate exception entries', async () => {
+ const testEntries: EntriesArray = [
+ { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
+ { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
+ { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
+ { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' },
+ {
+ field: 'host.hostname.text',
+ operator: 'included',
+ type: 'match_any',
+ value: ['estc', 'kibana'],
+ },
+ ];
+
+ const expectedEndpointExceptions = {
+ type: 'simple',
+ entries: [
+ {
+ field: 'server.domain',
+ operator: 'included',
+ type: 'exact_caseless',
+ value: 'DOMAIN',
+ },
+ {
+ field: 'server.ip',
+ operator: 'included',
+ type: 'exact_cased',
+ value: '192.168.1.1',
+ },
+ {
+ field: 'host.hostname',
+ operator: 'included',
+ type: 'exact_caseless_any',
+ value: ['estc', 'kibana'],
+ },
+ ],
+ };
+
+ const first = getFoundExceptionListItemSchemaMock();
+ first.data[0].entries = testEntries;
+ mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
+
+ const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
+ expect(resp).toEqual({
+ entries: [expectedEndpointExceptions],
+ });
+ });
+
+ test('it should not deduplicate exception entries across nested boundaries', async () => {
+ const testEntries: EntriesArray = [
+ {
+ entries: [
+ { field: 'nested.field', operator: 'included', type: 'match', value: 'some value' },
+ ],
+ field: 'some.parentField',
+ type: 'nested',
+ },
+ // Same as above but not inside the nest
+ { field: 'nested.field', operator: 'included', type: 'match', value: 'some value' },
+ ];
+
+ const expectedEndpointExceptions = {
+ type: 'simple',
+ entries: [
+ {
+ entries: [
+ {
+ field: 'nested.field',
+ operator: 'included',
+ type: 'exact_cased',
+ value: 'some value',
+ },
+ ],
+ field: 'some.parentField',
+ type: 'nested',
+ },
+ {
+ field: 'nested.field',
+ operator: 'included',
+ type: 'exact_cased',
+ value: 'some value',
+ },
+ ],
+ };
+
+ const first = getFoundExceptionListItemSchemaMock();
+ first.data[0].entries = testEntries;
+ mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
+
+ const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
+ expect(resp).toEqual({
+ entries: [expectedEndpointExceptions],
+ });
+ });
+
+ test('it should deduplicate exception items', async () => {
+ const testEntries: EntriesArray = [
+ { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' },
+ { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' },
+ ];
+
+ const expectedEndpointExceptions = {
+ type: 'simple',
+ entries: [
+ {
+ field: 'server.domain',
+ operator: 'included',
+ type: 'exact_caseless',
+ value: 'DOMAIN',
+ },
+ {
+ field: 'server.ip',
+ operator: 'included',
+ type: 'exact_cased',
+ value: '192.168.1.1',
+ },
+ ],
+ };
+
+ const first = getFoundExceptionListItemSchemaMock();
+ first.data[0].entries = testEntries;
+
+ // Create a second exception item with the same entries
+ first.data[1] = getExceptionListItemSchemaMock();
+ first.data[1].entries = testEntries;
+ mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
+
+ const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
+ expect(resp).toEqual({
+ entries: [expectedEndpointExceptions],
+ });
+ });
+
  test('it should ignore unsupported entries', async () => {
  // Lists and exists are not supported by the Endpoint
  const testEntries: EntriesArray = [
@@ -178,8 +311,9 @@ describe('buildEventTypeSignal', () => {
  });
 
  test('it should convert the exception lists response to the proper endpoint format while paging', async () => {
- // The first call returns one exception
+ // The first call returns two exceptions
  const first = getFoundExceptionListItemSchemaMock();
+ first.data.push(getExceptionListItemSchemaMock());
 
  // The second call returns two exceptions
  const second = getFoundExceptionListItemSchemaMock();
@@ -194,7 +328,8 @@ describe('buildEventTypeSignal', () => {
  .mockReturnValueOnce(second)
  .mockReturnValueOnce(third);
  const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
- expect(resp.entries.length).toEqual(3);
+ // Expect 2 exceptions, the first two calls returned the same exception list items
+ expect(resp.entries.length).toEqual(2);
  });
 
  test('it should handle no exceptions', async () => {

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts

@@ -97,10 +97,18 @@ export function translateToEndpointExceptions(
  exc: FoundExceptionListItemSchema,
  schemaVersion: string
 ): TranslatedExceptionListItem[] {
+ const entrySet = new Set();
+ const entriesFiltered: TranslatedExceptionListItem[] = [];
  if (schemaVersion === 'v1') {
- return exc.data.map((item) => {
- return translateItem(schemaVersion, item);
+ exc.data.forEach((entry) => {
+ const translatedItem = translateItem(schemaVersion, entry);
+ const entryHash = createHash('sha256').update(JSON.stringify(translatedItem)).digest('hex');
+ if (!entrySet.has(entryHash)) {
+ entriesFiltered.push(translatedItem);
+ entrySet.add(entryHash);
+ }
  });
+ return entriesFiltered;
  } else {
  throw new Error('unsupported schemaVersion');
  }
@@ -124,12 +132,17 @@ function translateItem(
  schemaVersion: string,
  item: ExceptionListItemSchema
 ): TranslatedExceptionListItem {
+ const itemSet = new Set();
  return {
  type: item.type,
  entries: item.entries.reduce((translatedEntries: TranslatedEntry[], entry) => {
  const translatedEntry = translateEntry(schemaVersion, entry);
  if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) {
- translatedEntries.push(translatedEntry);
+ const itemHash = createHash('sha256').update(JSON.stringify(translatedEntry)).digest('hex');
+ if (!itemSet.has(itemHash)) {
+ translatedEntries.push(translatedEntry);
+ itemSet.add(itemHash);
+ }
  }
  return translatedEntries;
  }, []),