[Security Solution][Detecttions] Indicator enrichment tweaks (#92989)

* Update copy of rule config

* Encode threat index as part of our named query

* Add index to named query, and enrich both id and index

We still need mappings and to fix integration tests, but this generates
the correct data.

* Update integration tests with new enrichment fields

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx

@@ -198,14 +198,14 @@ export const schema: FormSchema<AboutStepRule> = {
     label: i18n.translate(
       'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathLabel',
       {
-        defaultMessage: 'Threat Indicator Path',
+        defaultMessage: 'Indicator prefix override',
       }
     ),
     helpText: i18n.translate(
       'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathHelpText',
       {
         defaultMessage:
-          'Specify the document path containing your threat indicator fields. Used for enrichment of indicator match alerts.',
+          'Specify the document prefix containing your indicator fields. Used for enrichment of indicator match alerts.',
       }
     ),
     labelAppend: OptionalFieldLabel,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts

@@ -84,6 +84,7 @@ export const createInnerAndClauses = ({
                   query: value[0],
                   _name: encodeThreatMatchNamedQuery({
                     id: threatListItem._id,
+                    index: threatListItem._index,
                     field: threatMappingEntry.field,
                     value: threatMappingEntry.value,
                   }),

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts

@@ -12,6 +12,7 @@ export const getNamedQueryMock = (
   overrides: Partial<ThreatMatchNamedQuery> = {}
 ): ThreatMatchNamedQuery => ({
   id: 'id',
+  index: 'index',
   field: 'field',
   value: 'value',
   ...overrides,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts

@@ -88,7 +88,12 @@ describe('buildMatchedIndicator', () => {
       }),
     ];
     queries = [
-      getNamedQueryMock({ id: '123', field: 'event.field', value: 'threat.indicator.domain' }),
+      getNamedQueryMock({
+        id: '123',
+        index: 'threat-index',
+        field: 'event.field',
+        value: 'threat.indicator.domain',
+      }),
     ];
   });
 
@@ -112,6 +117,26 @@ describe('buildMatchedIndicator', () => {
     expect(get(indicator, 'matched.atomic')).toEqual('domain_1');
   });
 
+  it('returns the _id of the matched indicator as matched.id', () => {
+    const [indicator] = buildMatchedIndicator({
+      queries,
+      threats,
+      indicatorPath,
+    });
+
+    expect(get(indicator, 'matched.id')).toEqual('123');
+  });
+
+  it('returns the _index of the matched indicator as matched.index', () => {
+    const [indicator] = buildMatchedIndicator({
+      queries,
+      threats,
+      indicatorPath,
+    });
+
+    expect(get(indicator, 'matched.index')).toEqual('threat-index');
+  });
+
   it('returns the field of the matched indicator as matched.field', () => {
     const [indicator] = buildMatchedIndicator({
       queries,
@@ -173,6 +198,8 @@ describe('buildMatchedIndicator', () => {
         domain: 'domain_1',
         matched: {
           atomic: 'domain_1',
+          id: '123',
+          index: 'threat-index',
           field: 'event.field',
           type: 'type_1',
         },
@@ -211,6 +238,8 @@ describe('buildMatchedIndicator', () => {
         indicator_field: 'indicator_field_1',
         matched: {
           atomic: 'domain_1',
+          id: '123',
+          index: 'threat-index',
           field: 'event.field',
           type: 'indicator_type',
         },
@@ -237,6 +266,8 @@ describe('buildMatchedIndicator', () => {
       {
         matched: {
           atomic: undefined,
+          id: '123',
+          index: 'threat-index',
           field: 'event.field',
           type: undefined,
         },
@@ -262,6 +293,8 @@ describe('buildMatchedIndicator', () => {
       {
         matched: {
           atomic: undefined,
+          id: '123',
+          index: 'threat-index',
           field: 'event.field',
           type: undefined,
         },
@@ -295,6 +328,8 @@ describe('buildMatchedIndicator', () => {
         domain: 'foo',
         matched: {
           atomic: undefined,
+          id: '123',
+          index: 'threat-index',
           field: 'event.field',
           type: 'first',
         },
@@ -362,7 +397,12 @@ describe('enrichSignalThreatMatches', () => {
       }),
     ];
     matchedQuery = encodeThreatMatchNamedQuery(
-      getNamedQueryMock({ id: '123', field: 'event.field', value: 'threat.indicator.domain' })
+      getNamedQueryMock({
+        id: '123',
+        index: 'indicator_index',
+        field: 'event.field',
+        value: 'threat.indicator.domain',
+      })
     );
   });
 
@@ -395,7 +435,13 @@ describe('enrichSignalThreatMatches', () => {
       { existing: 'indicator' },
       {
         domain: 'domain_1',
-        matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' },
+        matched: {
+          atomic: 'domain_1',
+          id: '123',
+          index: 'indicator_index',
+          field: 'event.field',
+          type: 'type_1',
+        },
         other: 'other_1',
         type: 'type_1',
       },
@@ -418,7 +464,13 @@ describe('enrichSignalThreatMatches', () => {
 
     expect(indicators).toEqual([
       {
-        matched: { atomic: undefined, field: 'event.field', type: undefined },
+        matched: {
+          atomic: undefined,
+          id: '123',
+          index: 'indicator_index',
+          field: 'event.field',
+          type: undefined,
+        },
       },
     ]);
   });
@@ -441,7 +493,13 @@ describe('enrichSignalThreatMatches', () => {
       { existing: 'indicator' },
       {
         domain: 'domain_1',
-        matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' },
+        matched: {
+          atomic: 'domain_1',
+          id: '123',
+          index: 'indicator_index',
+          field: 'event.field',
+          type: 'type_1',
+        },
         other: 'other_1',
         type: 'type_1',
       },
@@ -477,6 +535,7 @@ describe('enrichSignalThreatMatches', () => {
     matchedQuery = encodeThreatMatchNamedQuery(
       getNamedQueryMock({
         id: '123',
+        index: 'custom_index',
         field: 'event.field',
         value: 'custom_threat.custom_indicator.domain',
       })
@@ -496,7 +555,13 @@ describe('enrichSignalThreatMatches', () => {
     expect(indicators).toEqual([
       {
         domain: 'custom_domain',
-        matched: { atomic: 'custom_domain', field: 'event.field', type: 'custom_type' },
+        matched: {
+          atomic: 'custom_domain',
+          id: '123',
+          index: 'custom_index',
+          field: 'event.field',
+          type: 'custom_type',
+        },
         other: 'custom_other',
         type: 'custom_type',
       },
@@ -526,7 +591,12 @@ describe('enrichSignalThreatMatches', () => {
       _id: 'signal123',
       matched_queries: [
         encodeThreatMatchNamedQuery(
-          getNamedQueryMock({ id: '456', field: 'event.other', value: 'threat.indicator.domain' })
+          getNamedQueryMock({
+            id: '456',
+            index: 'other_custom_index',
+            field: 'event.other',
+            value: 'threat.indicator.domain',
+          })
         ),
       ],
     });
@@ -545,14 +615,22 @@ describe('enrichSignalThreatMatches', () => {
     expect(indicators).toEqual([
       {
         domain: 'domain_1',
-        matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' },
+        matched: {
+          atomic: 'domain_1',
+          id: '123',
+          index: 'indicator_index',
+          field: 'event.field',
+          type: 'type_1',
+        },
         other: 'other_1',
         type: 'type_1',
       },
       {
         domain: 'domain_2',
         matched: {
           atomic: 'domain_2',
+          id: '456',
+          index: 'other_custom_index',
           field: 'event.other',
           type: 'type_2',
         },

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts

@@ -60,7 +60,7 @@ export const buildMatchedIndicator = ({
 
     return {
       ...indicator,
-      matched: { atomic, field: query.field, type },
+      matched: { atomic, field: query.field, id: query.id, index: query.index, type },
     };
   });
 

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts

@@ -199,6 +199,7 @@ export interface SortWithTieBreaker {
 
 export interface ThreatMatchNamedQuery {
   id: string;
+  index: string;
   field: string;
   value: string;
 }

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts

@@ -589,6 +589,7 @@ describe('utils', () => {
       it('generates a string that can be later decoded', () => {
         const encoded = encodeThreatMatchNamedQuery({
           id: 'id',
+          index: 'index',
           field: 'field',
           value: 'value',
         });
@@ -601,6 +602,7 @@ describe('utils', () => {
       it('can decode an encoded query', () => {
         const query: ThreatMatchNamedQuery = {
           id: 'my_id',
+          index: 'index',
           field: 'threat.indicator.domain',
           value: 'host.name',
         };
@@ -623,14 +625,15 @@ describe('utils', () => {
       it('raises an error if the query is missing a value', () => {
         const badQuery: ThreatMatchNamedQuery = {
           id: 'my_id',
+          index: 'index',
           // @ts-expect-error field intentionally undefined
           field: undefined,
           value: 'host.name',
         };
         const badInput = encodeThreatMatchNamedQuery(badQuery);
 
         expect(() => decodeThreatMatchNamedQuery(badInput)).toThrowError(
-          'Decoded query is invalid. Decoded value: {"id":"my_id","field":"","value":"host.name"}'
+          'Decoded query is invalid. Decoded value: {"id":"my_id","index":"index","field":"","value":"host.name"}'
         );
       });
     });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts

@@ -115,21 +115,22 @@ export const combineConcurrentResults = (
   return combineResults(currentResult, maxedNewResult);
 };
 
-const separator = '___SEPARATOR___';
+const separator = '__SEP__';
 export const encodeThreatMatchNamedQuery = ({
   id,
+  index,
   field,
   value,
 }: ThreatMatchNamedQuery): string => {
-  return [id, field, value].join(separator);
+  return [id, index, field, value].join(separator);
 };
 
 export const decodeThreatMatchNamedQuery = (encoded: string): ThreatMatchNamedQuery => {
   const queryValues = encoded.split(separator);
-  const [id, field, value] = queryValues;
-  const query = { id, field, value };
+  const [id, index, field, value] = queryValues;
+  const query = { id, index, field, value };
 
-  if (queryValues.length !== 3 || !queryValues.every(Boolean)) {
+  if (queryValues.length !== 4 || !queryValues.every(Boolean)) {
     const queryString = JSON.stringify(query);
     throw new Error(`Decoded query is invalid. Decoded value: ${queryString}`);
   }