[SIEM][Detection Engine] Follow up issues from PR 68127 (elastic#68612)

## Summary

* Smaller follow ups and bug fixes from: elastic#68127
* Added unknown to `findDifferencesRecursive`
* Added linter rule to catch NodeJS code in the common folders for both `lists` and `security_solution`
* Removed the Hapi server type from the common folder of lists

### Checklist

* Added unknown to the correct locations

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

.eslintrc.js

@@ -594,8 +594,11 @@ module.exports = {
      * Security Solution overrides
      */
     {
-      // front end typescript and javascript files only
-      files: ['x-pack/plugins/security_solution/public/**/*.{js,ts,tsx}'],
+      // front end and common typescript and javascript files only
+      files: [
+        'x-pack/plugins/security_solution/public/**/*.{js,ts,tsx}',
+        'x-pack/plugins/security_solution/common/**/*.{js,ts,tsx}',
+      ],
       rules: {
         'import/no-nodejs-modules': 'error',
         'no-restricted-imports': [
@@ -771,6 +774,23 @@ module.exports = {
     /**
      * Lists overrides
      */
+    {
+      // front end and common typescript and javascript files only
+      files: [
+        'x-pack/plugins/lists/public/**/*.{js,ts,tsx}',
+        'x-pack/plugins/lists/common/**/*.{js,ts,tsx}',
+      ],
+      rules: {
+        'import/no-nodejs-modules': 'error',
+        'no-restricted-imports': [
+          'error',
+          {
+            // prevents UI code from importing server side code and then webpack including it when doing builds
+            patterns: ['**/server/*'],
+          },
+        ],
+      },
+    },
     {
       // typescript and javascript for front and back end
       files: ['x-pack/plugins/lists/**/*.{js,ts,tsx}'],

x-pack/plugins/lists/common/schemas/request/import_list_item_schema.ts

@@ -6,9 +6,6 @@
 
 /* eslint-disable @typescript-eslint/camelcase */
 
-// TODO: You cannot import a stream from common into the front end code! CHANGE THIS
-import { Readable } from 'stream';
-
 import * as t from 'io-ts';
 
 import { file } from '../common/schemas';
@@ -20,17 +17,3 @@ export const importListItemSchema = t.exact(
 );
 
 export type ImportListItemSchema = t.TypeOf<typeof importListItemSchema>;
-
-// TODO: You cannot import a stream from common into the front end code! CHANGE THIS
-export interface HapiReadableStream extends Readable {
-  hapi: {
-    filename: string;
-  };
-}
-
-/**
- * Special interface since we are streaming in a file through a reader
- */
-export interface ImportListItemHapiFileSchema {
-  file: HapiReadableStream;
-}

x-pack/plugins/lists/server/routes/import_list_item_route.ts

@@ -4,6 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { Readable } from 'stream';
+
 import { IRouter } from 'kibana/server';
 
 import { LIST_ITEM_URL } from '../../common/constants';
@@ -13,15 +15,23 @@ import {
   transformError,
   validate,
 } from '../siem_server_deps';
-import {
-  ImportListItemHapiFileSchema,
-  importListItemQuerySchema,
-  importListItemSchema,
-  listSchema,
-} from '../../common/schemas';
+import { importListItemQuerySchema, importListItemSchema, listSchema } from '../../common/schemas';
 
 import { getListClient } from '.';
 
+export interface HapiReadableStream extends Readable {
+  hapi: {
+    filename: string;
+  };
+}
+
+/**
+ * Special interface since we are streaming in a file through a reader
+ */
+export interface ImportListItemHapiFileSchema {
+  file: HapiReadableStream;
+}
+
 export const importListItemRoute = (router: IRouter): void => {
   router.post(
     {

x-pack/plugins/lists/server/services/items/buffer_lines.test.ts

@@ -4,9 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { TestReadable } from '../../../common/test_readable.mock';
-
 import { BufferLines } from './buffer_lines';
+import { TestReadable } from './test_readable.mock';
 
 describe('buffer_lines', () => {
   test('it can read a single line', (done) => {

x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.mock.ts

@@ -3,11 +3,12 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { TestReadable } from '../../../common/test_readable.mock';
 import { getCallClusterMock } from '../../../common/get_call_cluster.mock';
 import { ImportListItemsToStreamOptions, WriteBufferToItemsOptions } from '../items';
 import { LIST_ID, LIST_ITEM_INDEX, META, TYPE, USER } from '../../../common/constants.mock';
 
+import { TestReadable } from './test_readable.mock';
+
 export const getImportListItemsToStreamOptionsMock = (): ImportListItemsToStreamOptions => ({
   callCluster: getCallClusterMock(),
   listId: LIST_ID,

x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts

@@ -143,6 +143,9 @@ export type Throttle = t.TypeOf<typeof throttle>;
 export const throttleOrNull = t.union([throttle, t.null]);
 export type ThrottleOrNull = t.TypeOf<typeof throttleOrNull>;
 
+export const throttleOrNullOrUndefined = t.union([throttle, t.null, t.undefined]);
+export type ThrottleOrUndefinedOrNull = t.TypeOf<typeof throttleOrNullOrUndefined>;
+
 export const anomaly_threshold = PositiveInteger;
 export type AnomalyThreshold = t.TypeOf<typeof PositiveInteger>;
 
@@ -156,11 +159,8 @@ export const machineLearningJobIdOrUndefined = t.union([machine_learning_job_id,
 export type MachineLearningJobIdOrUndefined = t.TypeOf<typeof machineLearningJobIdOrUndefined>;
 
 /**
- * Note that this is a plain unknown object because we allow the UI
- * to send us extra additional information as "meta" which can be anything.
- *
- * TODO: Strip away extra information and possibly even "freeze" this object
- * so we have tighter control over 3rd party data structures.
+ * Note that this is a non-exact io-ts type as we allow extra meta information
+ * to be added to the meta object
  */
 export const meta = t.object;
 export type Meta = t.TypeOf<typeof meta>;
@@ -192,8 +192,10 @@ export const severityOrUndefined = t.union([severity, t.undefined]);
 export type SeverityOrUndefined = t.TypeOf<typeof severityOrUndefined>;
 
 export const status = t.keyof({ open: null, closed: null });
+export type Status = t.TypeOf<typeof status>;
 
 export const job_status = t.keyof({ succeeded: null, failed: null, 'going to run': null });
+export type JobStatus = t.TypeOf<typeof job_status>;
 
 // TODO: Create a regular expression type or custom date math part type here
 export const to = t.string;
@@ -307,10 +309,20 @@ export const versionOrUndefined = t.union([version, t.undefined]);
 export type VersionOrUndefined = t.TypeOf<typeof versionOrUndefined>;
 
 export const last_success_at = IsoDateString;
+export type LastSuccessAt = t.TypeOf<typeof IsoDateString>;
+
 export const last_success_message = t.string;
+export type LastSuccessMessage = t.TypeOf<typeof last_success_message>;
+
 export const last_failure_at = IsoDateString;
+export type LastFailureAt = t.TypeOf<typeof last_failure_at>;
+
 export const last_failure_message = t.string;
+export type LastFailureMessage = t.TypeOf<typeof last_failure_message>;
+
 export const status_date = IsoDateString;
+export type StatusDate = t.TypeOf<typeof status_date>;
+
 export const rules_installed = PositiveInteger;
 export const rules_updated = PositiveInteger;
 export const status_code = PositiveInteger;

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.mock.ts

@@ -7,19 +7,30 @@
 import { CreateRulesSchema, CreateRulesSchemaDecoded } from './create_rules_schema';
 import { DEFAULT_MAX_SIGNALS } from '../../../constants';
 
-export const getCreateRulesSchemaMock = (): CreateRulesSchema => ({
-  description: 'some description',
+export const getCreateRulesSchemaMock = (ruleId = 'rule-1'): CreateRulesSchema => ({
+  description: 'Detecting root and admin users',
   name: 'Query with a rule id',
   query: 'user.name: root or user.name: admin',
   severity: 'high',
   type: 'query',
   risk_score: 55,
   language: 'kuery',
-  rule_id: 'rule-1',
+  rule_id: ruleId,
 });
 
+export const getCreateMlRulesSchemaMock = (ruleId = 'rule-1') => {
+  const { query, language, index, ...mlParams } = getCreateRulesSchemaMock(ruleId);
+
+  return {
+    ...mlParams,
+    type: 'machine_learning',
+    anomaly_threshold: 58,
+    machine_learning_job_id: 'typical-ml-job-id',
+  };
+};
+
 export const getCreateRulesSchemaDecodedMock = (): CreateRulesSchemaDecoded => ({
-  description: 'some description',
+  description: 'Detecting root and admin users',
   name: 'Query with a rule id',
   query: 'user.name: root or user.name: admin',
   severity: 'high',

x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.mock.ts

@@ -7,15 +7,27 @@
 import { ImportRulesSchema, ImportRulesSchemaDecoded } from './import_rules_schema';
 import { DEFAULT_MAX_SIGNALS } from '../../../constants';
 
-export const getImportRulesSchemaMock = (): ImportRulesSchema => ({
+export const getImportRulesSchemaMock = (ruleId = 'rule-1'): ImportRulesSchema => ({
   description: 'some description',
   name: 'Query with a rule id',
   query: 'user.name: root or user.name: admin',
   severity: 'high',
   type: 'query',
   risk_score: 55,
   language: 'kuery',
-  rule_id: 'rule-1',
+  rule_id: ruleId,
+});
+
+export const getImportRulesWithIdSchemaMock = (ruleId = 'rule-1'): ImportRulesSchema => ({
+  id: '6afb8ce1-ea94-4790-8653-fd0b021d2113',
+  description: 'some description',
+  name: 'Query with a rule id',
+  query: 'user.name: root or user.name: admin',
+  severity: 'high',
+  type: 'query',
+  risk_score: 55,
+  language: 'kuery',
+  rule_id: ruleId,
 });
 
 export const getImportRulesSchemaDecodedMock = (): ImportRulesSchemaDecoded => ({
@@ -42,3 +54,22 @@ export const getImportRulesSchemaDecodedMock = (): ImportRulesSchemaDecoded => (
   rule_id: 'rule-1',
   immutable: false,
 });
+
+/**
+ * Given an array of rules, builds an NDJSON string of rules
+ * as we might import/export
+ * @param rules Array of rule objects with which to generate rule JSON
+ */
+export const rulesToNdJsonString = (rules: ImportRulesSchema[]) => {
+  return rules.map((rule) => JSON.stringify(rule)).join('\r\n');
+};
+
+/**
+ * Given an array of rule IDs, builds an NDJSON string of rules
+ * as we might import
+ * @param ruleIds Array of ruleIds with which to generate rule JSON
+ */
+export const ruleIdsToNdJsonString = (ruleIds: string[]) => {
+  const rules = ruleIds.map((ruleId) => getImportRulesSchemaMock(ruleId));
+  return rulesToNdJsonString(rules);
+};

x-pack/plugins/security_solution/common/detection_engine/schemas/response/error_schema.mocks.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ErrorSchema } from './error_schema';
+
+export const getErrorSchemaMock = (
+  id: string = '819eded6-e9c8-445b-a647-519aea39e063'
+): ErrorSchema => ({
+  id,
+  error: {
+    status_code: 404,
+    message: 'id: "819eded6-e9c8-445b-a647-519aea39e063" not found',
+  },
+});

x-pack/plugins/security_solution/common/detection_engine/schemas/response/error_schema.test.ts

@@ -7,15 +7,15 @@
 import { left } from 'fp-ts/lib/Either';
 import { pipe } from 'fp-ts/lib/pipeable';
 
-import { getErrorPayload } from './__mocks__/utils';
 import { errorSchema, ErrorSchema } from './error_schema';
 import { exactCheck } from '../../../exact_check';
 import { foldLeftRight, getPaths } from '../../../test_utils';
+import { getErrorSchemaMock } from './error_schema.mocks';
 
 describe('error_schema', () => {
   test('it should validate an error with a UUID given for id', () => {
-    const error = getErrorPayload();
-    const decoded = errorSchema.decode(getErrorPayload());
+    const error = getErrorSchemaMock();
+    const decoded = errorSchema.decode(getErrorSchemaMock());
     const checked = exactCheck(error, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -24,7 +24,7 @@ describe('error_schema', () => {
   });
 
   test('it should validate an error with a plain string given for id since sometimes we echo the user id which might not be a UUID back out to them', () => {
-    const error = getErrorPayload('fake id');
+    const error = getErrorSchemaMock('fake id');
     const decoded = errorSchema.decode(error);
     const checked = exactCheck(error, decoded);
     const message = pipe(checked, foldLeftRight);
@@ -35,7 +35,7 @@ describe('error_schema', () => {
 
   test('it should NOT validate an error when it has extra data next to a valid payload element', () => {
     type InvalidError = ErrorSchema & { invalid_extra_data?: string };
-    const error: InvalidError = getErrorPayload();
+    const error: InvalidError = getErrorSchemaMock();
     error.invalid_extra_data = 'invalid_extra_data';
     const decoded = errorSchema.decode(error);
     const checked = exactCheck(error, decoded);
@@ -46,7 +46,7 @@ describe('error_schema', () => {
   });
 
   test('it should NOT validate an error when it has required elements deleted from it', () => {
-    const error = getErrorPayload();
+    const error = getErrorSchemaMock();
     delete error.error;
     const decoded = errorSchema.decode(error);
     const checked = exactCheck(error, decoded);

x-pack/plugins/security_solution/common/detection_engine/schemas/response/find_rules_schema.mocks.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { FindRulesSchema } from './find_rules_schema';
+import { getRulesSchemaMock } from './rules_schema.mocks';
+
+export const getFindRulesSchemaMock = (): FindRulesSchema => ({
+  page: 1,
+  perPage: 1,
+  total: 1,
+  data: [getRulesSchemaMock()],
+});