[Security Solution] Split rule executor by rule type and validate typ…

…e specific rule params (#94857)

* Split rule executors into different files

* Pass type-specific rule SOs to rule executor functions

* Genericize function to narrow ruleSO type

* Remove undefined return type from getExceptions

* Remove unintentional change to SIGNALS_TEMPLATE_VERSION

* Remove extra validation now covered by schemas

* Remove extra validation from ML rule executor

* Fix types

* syncs schemas

* Revert "syncs schemas"

This reverts commit b1dd59e.

* Fix api test and move threshold executor test

* kinda adds eql test

* Refactor and fix unit tests

* fixes marshalls mistake

Co-authored-by: Davis Plumlee <davis.plumlee@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts

@@ -22,7 +22,7 @@ import { Query, Language, Index, TimestampOverrideOrUndefined } from './schemas/
 export const getQueryFilter = (
   query: Query,
   language: Language,
-  filters: Array<Partial<Filter>>,
+  filters: unknown,
   index: Index,
   lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>,
   excludeExceptions: boolean = true
@@ -48,7 +48,7 @@ export const getQueryFilter = (
     chunkSize: 1024,
   });
   const initialQuery = { query, language };
-  const allFilters = getAllFilters((filters as unknown) as Filter[], exceptionFilter);
+  const allFilters = getAllFilters(filters as Filter[], exceptionFilter);
 
   return buildEsQuery(indexPattern, initialQuery, allFilters, config);
 };

x-pack/plugins/security_solution/common/validate.ts

@@ -26,6 +26,19 @@ export const validate = <T extends t.Mixed>(
   return pipe(checked, fold(left, right));
 };
 
+export const validateNonExact = <T extends t.Mixed>(
+  obj: object,
+  schema: T
+): [t.TypeOf<T> | null, string | null] => {
+  const decoded = schema.decode(obj);
+  const left = (errors: t.Errors): [T | null, string | null] => [
+    null,
+    formatErrors(errors).join(','),
+  ];
+  const right = (output: T): [T | null, string | null] => [output, null];
+  return pipe(decoded, fold(left, right));
+};
+
 export const validateEither = <T extends t.Mixed, A extends unknown>(
   schema: T,
   obj: A

x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.mock.ts

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock';
+import {
+  BaseRuleParams,
+  EqlRuleParams,
+  MachineLearningRuleParams,
+  ThresholdRuleParams,
+} from './rule_schemas';
+
+const getBaseRuleParams = (): BaseRuleParams => {
+  return {
+    author: ['Elastic'],
+    buildingBlockType: 'default',
+    ruleId: 'rule-1',
+    description: 'Detecting root and admin users',
+    falsePositives: [],
+    immutable: false,
+    from: 'now-6m',
+    to: 'now',
+    severity: 'high',
+    severityMapping: [],
+    license: 'Elastic License',
+    outputIndex: '.siem-signals',
+    references: ['http://google.com'],
+    riskScore: 50,
+    riskScoreMapping: [],
+    ruleNameOverride: undefined,
+    maxSignals: 10000,
+    note: '',
+    timelineId: undefined,
+    timelineTitle: undefined,
+    timestampOverride: undefined,
+    meta: undefined,
+    threat: [],
+    version: 1,
+    exceptionsList: getListArrayMock(),
+  };
+};
+
+export const getThresholdRuleParams = (): ThresholdRuleParams => {
+  return {
+    ...getBaseRuleParams(),
+    type: 'threshold',
+    language: 'kuery',
+    index: ['some-index'],
+    query: 'host.name: *',
+    filters: undefined,
+    savedId: undefined,
+    threshold: {
+      field: 'host.id',
+      value: 5,
+    },
+  };
+};
+
+export const getEqlRuleParams = (): EqlRuleParams => {
+  return {
+    ...getBaseRuleParams(),
+    type: 'eql',
+    language: 'eql',
+    index: ['some-index'],
+    query: 'any where true',
+    filters: undefined,
+    eventCategoryOverride: undefined,
+  };
+};
+
+export const getMlRuleParams = (): MachineLearningRuleParams => {
+  return {
+    ...getBaseRuleParams(),
+    type: 'machine_learning',
+    anomalyThreshold: 42,
+    machineLearningJobId: 'my-job',
+  };
+};

x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_schemas.ts

@@ -104,6 +104,8 @@ const eqlSpecificRuleParams = t.type({
   filters: filtersOrUndefined,
   eventCategoryOverride: eventCategoryOverrideOrUndefined,
 });
+export const eqlRuleParams = t.intersection([baseRuleParams, eqlSpecificRuleParams]);
+export type EqlRuleParams = t.TypeOf<typeof eqlRuleParams>;
 
 const threatSpecificRuleParams = t.type({
   type: t.literal('threat_match'),
@@ -121,6 +123,8 @@ const threatSpecificRuleParams = t.type({
   concurrentSearches: concurrentSearchesOrUndefined,
   itemsPerSearch: itemsPerSearchOrUndefined,
 });
+export const threatRuleParams = t.intersection([baseRuleParams, threatSpecificRuleParams]);
+export type ThreatRuleParams = t.TypeOf<typeof threatRuleParams>;
 
 const querySpecificRuleParams = t.exact(
   t.type({
@@ -132,6 +136,8 @@ const querySpecificRuleParams = t.exact(
     savedId: savedIdOrUndefined,
   })
 );
+export const queryRuleParams = t.intersection([baseRuleParams, querySpecificRuleParams]);
+export type QueryRuleParams = t.TypeOf<typeof queryRuleParams>;
 
 const savedQuerySpecificRuleParams = t.type({
   type: t.literal('saved_query'),
@@ -143,6 +149,8 @@ const savedQuerySpecificRuleParams = t.type({
   filters: filtersOrUndefined,
   savedId: saved_id,
 });
+export const savedQueryRuleParams = t.intersection([baseRuleParams, savedQuerySpecificRuleParams]);
+export type SavedQueryRuleParams = t.TypeOf<typeof savedQueryRuleParams>;
 
 const thresholdSpecificRuleParams = t.type({
   type: t.literal('threshold'),
@@ -153,12 +161,19 @@ const thresholdSpecificRuleParams = t.type({
   savedId: savedIdOrUndefined,
   threshold,
 });
+export const thresholdRuleParams = t.intersection([baseRuleParams, thresholdSpecificRuleParams]);
+export type ThresholdRuleParams = t.TypeOf<typeof thresholdRuleParams>;
 
 const machineLearningSpecificRuleParams = t.type({
   type: t.literal('machine_learning'),
   anomalyThreshold: anomaly_threshold,
   machineLearningJobId: machine_learning_job_id,
 });
+export const machineLearningRuleParams = t.intersection([
+  baseRuleParams,
+  machineLearningSpecificRuleParams,
+]);
+export type MachineLearningRuleParams = t.TypeOf<typeof machineLearningRuleParams>;
 
 export const typeSpecificRuleParams = t.union([
   eqlSpecificRuleParams,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { loggingSystemMock } from 'src/core/server/mocks';
+import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks';
+import { RuleStatusService } from '../rule_status_service';
+import { eqlExecutor } from './eql';
+import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
+import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock';
+import { getEqlRuleParams } from '../../schemas/rule_schemas.mock';
+import { getIndexVersion } from '../../routes/index/get_index_version';
+import { SIGNALS_TEMPLATE_VERSION } from '../../routes/index/get_signals_template';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks';
+
+jest.mock('../../routes/index/get_index_version');
+
+describe('eql_executor', () => {
+  const version = '8.0.0';
+  let logger: ReturnType<typeof loggingSystemMock.createLogger>;
+  let alertServices: AlertServicesMock;
+  let ruleStatusService: Record<string, jest.Mock>;
+  (getIndexVersion as jest.Mock).mockReturnValue(SIGNALS_TEMPLATE_VERSION);
+  const eqlSO = {
+    id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
+    type: 'alert',
+    version: '1',
+    updated_at: '2020-03-27T22:55:59.577Z',
+    attributes: {
+      actions: [],
+      enabled: true,
+      name: 'rule-name',
+      tags: ['some fake tag 1', 'some fake tag 2'],
+      createdBy: 'sample user',
+      createdAt: '2020-03-27T22:55:59.577Z',
+      updatedBy: 'sample user',
+      schedule: {
+        interval: '5m',
+      },
+      throttle: 'no_actions',
+      params: getEqlRuleParams(),
+    },
+    references: [],
+  };
+  const searchAfterSize = 7;
+
+  beforeEach(() => {
+    alertServices = alertsMock.createAlertServices();
+    logger = loggingSystemMock.createLogger();
+    ruleStatusService = {
+      success: jest.fn(),
+      find: jest.fn(),
+      goingToRun: jest.fn(),
+      error: jest.fn(),
+      partialFailure: jest.fn(),
+    };
+    alertServices.scopedClusterClient.asCurrentUser.transport.request.mockResolvedValue(
+      elasticsearchClientMock.createSuccessTransportRequestPromise({
+        hits: {
+          total: { value: 10 },
+        },
+      })
+    );
+  });
+
+  describe('eqlExecutor', () => {
+    it('should set a warning when exception list for eql rule contains value list exceptions', async () => {
+      const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })];
+      try {
+        await eqlExecutor({
+          rule: eqlSO,
+          exceptionItems,
+          ruleStatusService: (ruleStatusService as unknown) as RuleStatusService,
+          services: alertServices,
+          version,
+          logger,
+          refresh: false,
+          searchAfterSize,
+        });
+      } catch (err) {
+        // eqlExecutor will throw until we have an EQL response mock that conforms to the
+        // expected EQL response format, so just catch the error and check the status service
+      }
+      expect(ruleStatusService.partialFailure).toHaveBeenCalled();
+      expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain(
+        'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules'
+      );
+    });
+  });
+});