[SIEM][Detection Engine] Follow up issues from PR 68127 (#68612) (#68774

) ## Summary * Smaller follow ups and bug fixes from: #68127 * Added unknown to `findDifferencesRecursive` * Added linter rule to catch NodeJS code in the common folders for both `lists` and `security_solution` * Removed the Hapi server type from the common folder of lists ### Checklist * Added unknown to the correct locations - [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios
elastic · Jun 10, 2020 · 36379ed · 36379ed
1 parent d95b5ea
commit 36379ed
Show file tree

Hide file tree

Showing 65 changed files with 867 additions and 797 deletions.
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -594,8 +594,11 @@ module.exports = {
  * Security Solution overrides
  */
  {
- // front end typescript and javascript files only
- files: ['x-pack/plugins/security_solution/public/**/*.{js,ts,tsx}'],
+ // front end and common typescript and javascript files only
+ files: [
+ 'x-pack/plugins/security_solution/public/**/*.{js,ts,tsx}',
+ 'x-pack/plugins/security_solution/common/**/*.{js,ts,tsx}',
+ ],
  rules: {
  'import/no-nodejs-modules': 'error',
  'no-restricted-imports': [
@@ -771,6 +774,23 @@ module.exports = {
  /**
  * Lists overrides
  */
+ {
+ // front end and common typescript and javascript files only
+ files: [
+ 'x-pack/plugins/lists/public/**/*.{js,ts,tsx}',
+ 'x-pack/plugins/lists/common/**/*.{js,ts,tsx}',
+ ],
+ rules: {
+ 'import/no-nodejs-modules': 'error',
+ 'no-restricted-imports': [
+ 'error',
+ {
+ // prevents UI code from importing server side code and then webpack including it when doing builds
+ patterns: ['**/server/*'],
+ },
+ ],
+ },
+ },
  {
  // typescript and javascript for front and back end
  files: ['x-pack/plugins/lists/**/*.{js,ts,tsx}'],

diff --git a/x-pack/plugins/lists/common/schemas/request/import_list_item_schema.ts b/x-pack/plugins/lists/common/schemas/request/import_list_item_schema.ts
@@ -6,9 +6,6 @@
 
 /* eslint-disable @typescript-eslint/camelcase */
 
-// TODO: You cannot import a stream from common into the front end code! CHANGE THIS
-import { Readable } from 'stream';
-
 import * as t from 'io-ts';
 
 import { file } from '../common/schemas';
@@ -20,17 +17,3 @@ export const importListItemSchema = t.exact(
 );
 
 export type ImportListItemSchema = t.TypeOf<typeof importListItemSchema>;
-
-// TODO: You cannot import a stream from common into the front end code! CHANGE THIS
-export interface HapiReadableStream extends Readable {
- hapi: {
- filename: string;
- };
-}
-
-/**
- * Special interface since we are streaming in a file through a reader
- */
-export interface ImportListItemHapiFileSchema {
- file: HapiReadableStream;
-}
diff --git a/x-pack/plugins/lists/server/routes/import_list_item_route.ts b/x-pack/plugins/lists/server/routes/import_list_item_route.ts
@@ -4,6 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { Readable } from 'stream';
+
 import { IRouter } from 'kibana/server';
 
 import { LIST_ITEM_URL } from '../../common/constants';
@@ -13,15 +15,23 @@ import {
  transformError,
  validate,
 } from '../siem_server_deps';
-import {
- ImportListItemHapiFileSchema,
- importListItemQuerySchema,
- importListItemSchema,
- listSchema,
-} from '../../common/schemas';
+import { importListItemQuerySchema, importListItemSchema, listSchema } from '../../common/schemas';
 
 import { getListClient } from '.';
 
+export interface HapiReadableStream extends Readable {
+ hapi: {
+ filename: string;
+ };
+}
+
+/**
+ * Special interface since we are streaming in a file through a reader
+ */
+export interface ImportListItemHapiFileSchema {
+ file: HapiReadableStream;
+}
+
 export const importListItemRoute = (router: IRouter): void => {
  router.post(
  {

diff --git a/x-pack/plugins/lists/server/services/items/buffer_lines.test.ts b/x-pack/plugins/lists/server/services/items/buffer_lines.test.ts
@@ -4,9 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { TestReadable } from '../../../common/test_readable.mock';
-
 import { BufferLines } from './buffer_lines';
+import { TestReadable } from './test_readable.mock';
 
 describe('buffer_lines', () => {
  test('it can read a single line', (done) => {

diff --git a/...lugins/lists/common/test_readable.mock.ts → ...rver/services/items/test_readable.mock.ts b/...lugins/lists/common/test_readable.mock.ts → ...rver/services/items/test_readable.mock.ts
diff --git a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.mock.ts b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.mock.ts
@@ -3,11 +3,12 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { TestReadable } from '../../../common/test_readable.mock';
 import { getCallClusterMock } from '../../../common/get_call_cluster.mock';
 import { ImportListItemsToStreamOptions, WriteBufferToItemsOptions } from '../items';
 import { LIST_ID, LIST_ITEM_INDEX, META, TYPE, USER } from '../../../common/constants.mock';
 
+import { TestReadable } from './test_readable.mock';
+
 export const getImportListItemsToStreamOptionsMock = (): ImportListItemsToStreamOptions => ({
  callCluster: getCallClusterMock(),
  listId: LIST_ID,

diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts
@@ -143,6 +143,9 @@ export type Throttle = t.TypeOf<typeof throttle>;
 export const throttleOrNull = t.union([throttle, t.null]);
 export type ThrottleOrNull = t.TypeOf<typeof throttleOrNull>;
 
+export const throttleOrNullOrUndefined = t.union([throttle, t.null, t.undefined]);
+export type ThrottleOrUndefinedOrNull = t.TypeOf<typeof throttleOrNullOrUndefined>;
+
 export const anomaly_threshold = PositiveInteger;
 export type AnomalyThreshold = t.TypeOf<typeof PositiveInteger>;
 
@@ -156,11 +159,8 @@ export const machineLearningJobIdOrUndefined = t.union([machine_learning_job_id,
 export type MachineLearningJobIdOrUndefined = t.TypeOf<typeof machineLearningJobIdOrUndefined>;
 
 /**
- * Note that this is a plain unknown object because we allow the UI
- * to send us extra additional information as "meta" which can be anything.
- *
- * TODO: Strip away extra information and possibly even "freeze" this object
- * so we have tighter control over 3rd party data structures.
+ * Note that this is a non-exact io-ts type as we allow extra meta information
+ * to be added to the meta object
  */
 export const meta = t.object;
 export type Meta = t.TypeOf<typeof meta>;
@@ -192,8 +192,10 @@ export const severityOrUndefined = t.union([severity, t.undefined]);
 export type SeverityOrUndefined = t.TypeOf<typeof severityOrUndefined>;
 
 export const status = t.keyof({ open: null, closed: null });
+export type Status = t.TypeOf<typeof status>;
 
 export const job_status = t.keyof({ succeeded: null, failed: null, 'going to run': null });
+export type JobStatus = t.TypeOf<typeof job_status>;
 
 // TODO: Create a regular expression type or custom date math part type here
 export const to = t.string;
@@ -307,10 +309,20 @@ export const versionOrUndefined = t.union([version, t.undefined]);
 export type VersionOrUndefined = t.TypeOf<typeof versionOrUndefined>;
 
 export const last_success_at = IsoDateString;
+export type LastSuccessAt = t.TypeOf<typeof IsoDateString>;
+
 export const last_success_message = t.string;
+export type LastSuccessMessage = t.TypeOf<typeof last_success_message>;
+
 export const last_failure_at = IsoDateString;
+export type LastFailureAt = t.TypeOf<typeof last_failure_at>;
+
 export const last_failure_message = t.string;
+export type LastFailureMessage = t.TypeOf<typeof last_failure_message>;
+
 export const status_date = IsoDateString;
+export type StatusDate = t.TypeOf<typeof status_date>;
+
 export const rules_installed = PositiveInteger;
 export const rules_updated = PositiveInteger;
 export const status_code = PositiveInteger;

diff --git a/...ins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts b/...ins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts
@@ -7,19 +7,30 @@
 import { CreateRulesSchema, CreateRulesSchemaDecoded } from './create_rules_schema';
 import { DEFAULT_MAX_SIGNALS } from '../../../constants';
 
-export const getCreateRulesSchemaMock = (): CreateRulesSchema => ({
- description: 'some description',
+export const getCreateRulesSchemaMock = (ruleId = 'rule-1'): CreateRulesSchema => ({
+ description: 'Detecting root and admin users',
  name: 'Query with a rule id',
  query: 'user.name: root or user.name: admin',
  severity: 'high',
  type: 'query',
  risk_score: 55,
  language: 'kuery',
- rule_id: 'rule-1',
+ rule_id: ruleId,
 });
 
+export const getCreateMlRulesSchemaMock = (ruleId = 'rule-1') => {
+ const { query, language, index, ...mlParams } = getCreateRulesSchemaMock(ruleId);
+
+ return {
+ ...mlParams,
+ type: 'machine_learning',
+ anomaly_threshold: 58,
+ machine_learning_job_id: 'typical-ml-job-id',
+ };
+};
+
 export const getCreateRulesSchemaDecodedMock = (): CreateRulesSchemaDecoded => ({
- description: 'some description',
+ description: 'Detecting root and admin users',
  name: 'Query with a rule id',
  query: 'user.name: root or user.name: admin',
  severity: 'high',

diff --git a/...ins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts b/...ins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts
@@ -7,15 +7,27 @@
 import { ImportRulesSchema, ImportRulesSchemaDecoded } from './import_rules_schema';
 import { DEFAULT_MAX_SIGNALS } from '../../../constants';
 
-export const getImportRulesSchemaMock = (): ImportRulesSchema => ({
+export const getImportRulesSchemaMock = (ruleId = 'rule-1'): ImportRulesSchema => ({
  description: 'some description',
  name: 'Query with a rule id',
  query: 'user.name: root or user.name: admin',
  severity: 'high',
  type: 'query',
  risk_score: 55,
  language: 'kuery',
- rule_id: 'rule-1',
+ rule_id: ruleId,
+});
+
+export const getImportRulesWithIdSchemaMock = (ruleId = 'rule-1'): ImportRulesSchema => ({
+ id: '6afb8ce1-ea94-4790-8653-fd0b021d2113',
+ description: 'some description',
+ name: 'Query with a rule id',
+ query: 'user.name: root or user.name: admin',
+ severity: 'high',
+ type: 'query',
+ risk_score: 55,
+ language: 'kuery',
+ rule_id: ruleId,
 });
 
 export const getImportRulesSchemaDecodedMock = (): ImportRulesSchemaDecoded => ({
@@ -42,3 +54,22 @@ export const getImportRulesSchemaDecodedMock = (): ImportRulesSchemaDecoded => (
  rule_id: 'rule-1',
  immutable: false,
 });
+
+/**
+ * Given an array of rules, builds an NDJSON string of rules
+ * as we might import/export
+ * @param rules Array of rule objects with which to generate rule JSON
+ */
+export const rulesToNdJsonString = (rules: ImportRulesSchema[]) => {
+ return rules.map((rule) => JSON.stringify(rule)).join('\r\n');
+};
+
+/**
+ * Given an array of rule IDs, builds an NDJSON string of rules
+ * as we might import
+ * @param ruleIds Array of ruleIds with which to generate rule JSON
+ */
+export const ruleIdsToNdJsonString = (ruleIds: string[]) => {
+ const rules = ruleIds.map((ruleId) => getImportRulesSchemaMock(ruleId));
+ return rulesToNdJsonString(rules);
+};
diff --git a/.../plugins/security_solution/common/detection_engine/schemas/response/error_schema.mocks.ts b/.../plugins/security_solution/common/detection_engine/schemas/response/error_schema.mocks.ts
@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ErrorSchema } from './error_schema';
+
+export const getErrorSchemaMock = (
+ id: string = '819eded6-e9c8-445b-a647-519aea39e063'
+): ErrorSchema => ({
+ id,
+ error: {
+ status_code: 404,
+ message: 'id: "819eded6-e9c8-445b-a647-519aea39e063" not found',
+ },
+});
diff --git a/...k/plugins/security_solution/common/detection_engine/schemas/response/error_schema.test.ts b/...k/plugins/security_solution/common/detection_engine/schemas/response/error_schema.test.ts
@@ -7,15 +7,15 @@
 import { left } from 'fp-ts/lib/Either';
 import { pipe } from 'fp-ts/lib/pipeable';
 
-import { getErrorPayload } from './__mocks__/utils';
 import { errorSchema, ErrorSchema } from './error_schema';
 import { exactCheck } from '../../../exact_check';
 import { foldLeftRight, getPaths } from '../../../test_utils';
+import { getErrorSchemaMock } from './error_schema.mocks';
 
 describe('error_schema', () => {
  test('it should validate an error with a UUID given for id', () => {
- const error = getErrorPayload();
- const decoded = errorSchema.decode(getErrorPayload());
+ const error = getErrorSchemaMock();
+ const decoded = errorSchema.decode(getErrorSchemaMock());
  const checked = exactCheck(error, decoded);
  const message = pipe(checked, foldLeftRight);
 
@@ -24,7 +24,7 @@ describe('error_schema', () => {
  });
 
  test('it should validate an error with a plain string given for id since sometimes we echo the user id which might not be a UUID back out to them', () => {
- const error = getErrorPayload('fake id');
+ const error = getErrorSchemaMock('fake id');
  const decoded = errorSchema.decode(error);
  const checked = exactCheck(error, decoded);
  const message = pipe(checked, foldLeftRight);
@@ -35,7 +35,7 @@ describe('error_schema', () => {
 
  test('it should NOT validate an error when it has extra data next to a valid payload element', () => {
  type InvalidError = ErrorSchema & { invalid_extra_data?: string };
- const error: InvalidError = getErrorPayload();
+ const error: InvalidError = getErrorSchemaMock();
  error.invalid_extra_data = 'invalid_extra_data';
  const decoded = errorSchema.decode(error);
  const checked = exactCheck(error, decoded);
@@ -46,7 +46,7 @@ describe('error_schema', () => {
  });
 
  test('it should NOT validate an error when it has required elements deleted from it', () => {
- const error = getErrorPayload();
+ const error = getErrorSchemaMock();
  delete error.error;
  const decoded = errorSchema.decode(error);
  const checked = exactCheck(error, decoded);

diff --git a/...ins/security_solution/common/detection_engine/schemas/response/find_rules_schema.mocks.ts b/...ins/security_solution/common/detection_engine/schemas/response/find_rules_schema.mocks.ts
@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { FindRulesSchema } from './find_rules_schema';
+import { getRulesSchemaMock } from './rules_schema.mocks';
+
+export const getFindRulesSchemaMock = (): FindRulesSchema => ({
+ page: 1,
+ perPage: 1,
+ total: 1,
+ data: [getRulesSchemaMock()],
+});