[SIEM][Lists] Adds 90% of the REST API and client API for exception l…

…ists and exception items (#66811)

## Summary

See for more details:
#65938

Adds pieces of the `exception list` and `exception list item` and refactors/cleans the code up where I had parts incorrect with little things such as the javascript library io-ts. Some unit tests were added but I am holding off until more of the operations solidify before adding the unit tests. Everything is still behind a feature flag that must be enabled and not advised still at this point to use so I feel ok pushing these parts forward.

Adds to the API:
- Create exception list
- Read exception list
- Update exception list
- Delete exception list (and exception list items that are associated with it)
- Create exception list item
- Find exception list (/_find)
- Read exception list item
- Update exception list item
- Delete exception list items individually
- Find exception list item (/_find)

What is still missing from the REST and client API?
- Patch exception list
- Patch exception list item
- Bulk versions of everything
- Import/Export options for these exception lists and list items

### Manual testing and REST API endpoints

Go here:
```sh
/projects/kibana/x-pack/plugins/lists/server/scripts
```

See the files:

```sh
delete_all_exception_lists.sh
delete_exception_list.sh
delete_exception_list_by_id.sh
delete_exception_list_item.sh
delete_exception_list_item_by_id.sh
exception_lists
find_exception_list_items.sh
find_exception_lists.sh
get_exception_list.sh
get_exception_list_by_id.sh
get_exception_list_item.sh
get_exception_list_item_by_id.sh
post_exception_list.sh
post_exception_list_item.sh
update_exception_list.sh
update_exception_list_item.sh
```

Ensure you first run:

```sh
./hard_reset
```

and ensure you have setup your kibana.dev.yml to have:

```yml
# Enable lists feature
xpack.lists.enabled: true
xpack.lists.listIndex: '.lists-frank'
xpack.lists.listItemIndex: '.items-frank'
```

Then you can use the above scripts to create, read, update, and delete exception list and exception list items as well as perform find commands against them all.

### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

Note: Some but limited unit tests at this point.

x-pack/plugins/lists/common/constants.ts

@@ -5,8 +5,14 @@
  */
 
 /**
- * Lists routes
+ * Value list routes
  */
-export const LIST_URL = `/api/lists`;
+export const LIST_URL = '/api/lists';
 export const LIST_INDEX = `${LIST_URL}/index`;
 export const LIST_ITEM_URL = `${LIST_URL}/items`;
+
+/**
+ * Exception list routes
+ */
+export const EXCEPTION_LIST_URL = '/api/exception_lists';
+export const EXCEPTION_LIST_ITEM_URL = '/api/exception_lists/items';

x-pack/plugins/lists/common/schemas/common/schemas.ts

@@ -8,7 +8,7 @@
 
 import * as t from 'io-ts';
 
-import { NonEmptyString } from '../types/non_empty_string';
+import { DefaultStringArray, NonEmptyString } from '../types';
 
 export const name = t.string;
 export type Name = t.TypeOf<typeof name>;
@@ -21,8 +21,9 @@ export const descriptionOrUndefined = t.union([description, t.undefined]);
 export type DescriptionOrUndefined = t.TypeOf<typeof descriptionOrUndefined>;
 
 export const list_id = NonEmptyString;
+export type ListId = t.TypeOf<typeof list_id>;
 export const list_idOrUndefined = t.union([list_id, t.undefined]);
-export type List_idOrUndefined = t.TypeOf<typeof list_idOrUndefined>;
+export type ListIdOrUndefined = t.TypeOf<typeof list_idOrUndefined>;
 
 export const item = t.string;
 export const created_at = t.string; // TODO: Make this into an ISO Date string check
@@ -60,3 +61,44 @@ export type MetaOrUndefined = t.TypeOf<typeof metaOrUndefined>;
 
 export const esDataTypeUnion = t.union([t.type({ ip }), t.type({ keyword })]);
 export type EsDataTypeUnion = t.TypeOf<typeof esDataTypeUnion>;
+
+export const tags = DefaultStringArray;
+export type Tags = t.TypeOf<typeof tags>;
+export const tagsOrUndefined = t.union([tags, t.undefined]);
+export type TagsOrUndefined = t.TypeOf<typeof tagsOrUndefined>;
+
+export const _tags = DefaultStringArray;
+export type _Tags = t.TypeOf<typeof _tags>;
+export const _tagsOrUndefined = t.union([_tags, t.undefined]);
+export type _TagsOrUndefined = t.TypeOf<typeof _tagsOrUndefined>;
+
+// TODO: Change this into a t.keyof enumeration when we know what types of lists we going to have.
+export const exceptionListType = t.string;
+export const exceptionListTypeOrUndefined = t.union([exceptionListType, t.undefined]);
+export type ExceptionListType = t.TypeOf<typeof exceptionListType>;
+export type ExceptionListTypeOrUndefined = t.TypeOf<typeof exceptionListTypeOrUndefined>;
+
+// TODO: Change this into a t.keyof enumeration when we know what types of lists we going to have.
+export const exceptionListItemType = t.string;
+export type ExceptionListItemType = t.TypeOf<typeof exceptionListItemType>;
+
+export const list_type = t.keyof({ item: null, list: null });
+export type ListType = t.TypeOf<typeof list_type>;
+
+// TODO: Investigate what the deep structure of a comment is really going to be and then change this to use that deep structure with a default array
+export const comment = DefaultStringArray;
+export type Comment = t.TypeOf<typeof comment>;
+export const commentOrUndefined = t.union([comment, t.undefined]);
+export type CommentOrUndefined = t.TypeOf<typeof commentOrUndefined>;
+
+export const item_id = NonEmptyString;
+export type ItemId = t.TypeOf<typeof item_id>;
+export const itemIdOrUndefined = t.union([item_id, t.undefined]);
+export type ItemIdOrUndefined = t.TypeOf<typeof itemIdOrUndefined>;
+
+export const per_page = t.number; // TODO: Change this out for PositiveNumber from siem
+export const total = t.number; // TODO: Change this out for PositiveNumber from siem
+export const page = t.number; // TODO: Change this out for PositiveNumber from siem
+export const sort_field = t.string;
+export const sort_order = t.keyof({ asc: null, desc: null });
+export const filter = t.string;

x-pack/plugins/lists/common/schemas/elastic_query/update_es_list_schema.ts

@@ -16,6 +16,7 @@ import {
   updated_by,
 } from '../common/schemas';
 
+// TODO: Should we use partial here and everywhere these are instead of this OrUndefined?
 export const updateEsListSchema = t.exact(
   t.type({
     description: descriptionOrUndefined,

x-pack/plugins/lists/common/schemas/index.ts

@@ -5,7 +5,9 @@
  */
 
 export * from './common';
-export * from './request';
-export * from './response';
 export * from './elastic_query';
 export * from './elastic_response';
+export * from './request';
+export * from './response';
+export * from './saved_objects';
+export * from './types';

x-pack/plugins/lists/common/schemas/request/create_exception_list_item_schema.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+  ItemId,
+  Tags,
+  _Tags,
+  _tags,
+  comment,
+  description,
+  exceptionListItemType,
+  list_id,
+  meta,
+  name,
+  tags,
+} from '../common/schemas';
+import { Identity, RequiredKeepUndefined } from '../../types';
+import { DefaultEntryArray, DefaultUuid } from '../types';
+import { EntriesArray } from '../types/entries';
+
+export const createExceptionListItemSchema = t.intersection([
+  t.exact(
+    t.type({
+      description,
+      list_id,
+      name,
+      type: exceptionListItemType,
+    })
+  ),
+  t.exact(
+    t.partial({
+      _tags, // defaults to empty array if not set during decode
+      comment, // defaults to empty array if not set during decode
+      entries: DefaultEntryArray, // defaults to empty array if not set during decode
+      item_id: DefaultUuid, // defaults to GUID (uuid v4) if not set during decode
+      meta, // defaults to undefined if not set during decode
+      tags, // defaults to empty array if not set during decode
+    })
+  ),
+]);
+
+export type CreateExceptionListItemSchemaPartial = Identity<
+  t.TypeOf<typeof createExceptionListItemSchema>
+>;
+export type CreateExceptionListItemSchema = RequiredKeepUndefined<
+  t.TypeOf<typeof createExceptionListItemSchema>
+>;
+
+// This type is used after a decode since the arrays turn into defaults of empty arrays
+// and if a item_id is not specified it turns into a default GUID
+export type CreateExceptionListItemSchemaDecoded = Identity<
+  Omit<CreateExceptionListItemSchema, '_tags' | 'tags' | 'item_id' | 'entries'> & {
+    _tags: _Tags;
+    tags: Tags;
+    item_id: ItemId;
+    entries: EntriesArray;
+  }
+>;

x-pack/plugins/lists/common/schemas/request/create_exception_list_schema.mock.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { DESCRIPTION, LIST_ID, META, NAME, TYPE } from '../../constants.mock';
+
+import { CreateExceptionListSchema } from './create_exception_list_schema';
+
+export const getCreateExceptionListSchemaMock = (): CreateExceptionListSchema => ({
+  _tags: [],
+  description: DESCRIPTION,
+  list_id: LIST_ID,
+  meta: META,
+  name: NAME,
+  tags: [],
+  type: TYPE,
+});

x-pack/plugins/lists/common/schemas/request/create_exception_list_schema.test.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { left } from 'fp-ts/lib/Either';
+import { pipe } from 'fp-ts/lib/pipeable';
+
+import { exactCheck, foldLeftRight, getPaths } from '../../siem_common_deps';
+
+import {
+  CreateExceptionListSchema,
+  createExceptionListSchema,
+} from './create_exception_list_schema';
+import { getCreateExceptionListSchemaMock } from './create_exception_list_schema.mock';
+
+describe('create_exception_list_schema', () => {
+  test('it should validate a typical exception lists request', () => {
+    const payload = getCreateExceptionListSchemaMock();
+    const decoded = createExceptionListSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it should accept an undefined for meta', () => {
+    const payload = getCreateExceptionListSchemaMock();
+    delete payload.meta;
+    const decoded = createExceptionListSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(payload);
+  });
+
+  test('it should accept an undefined for tags but return an array', () => {
+    const inputPayload = getCreateExceptionListSchemaMock();
+    const outputPayload = getCreateExceptionListSchemaMock();
+    delete inputPayload.tags;
+    outputPayload.tags = [];
+    const decoded = createExceptionListSchema.decode(inputPayload);
+    const checked = exactCheck(inputPayload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(outputPayload);
+  });
+
+  test('it should accept an undefined for _tags but return an array', () => {
+    const inputPayload = getCreateExceptionListSchemaMock();
+    const outputPayload = getCreateExceptionListSchemaMock();
+    delete inputPayload._tags;
+    outputPayload._tags = [];
+    const decoded = createExceptionListSchema.decode(inputPayload);
+    const checked = exactCheck(inputPayload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect(message.schema).toEqual(outputPayload);
+  });
+
+  test('it should accept an undefined for list_id and auto generate a uuid', () => {
+    const inputPayload = getCreateExceptionListSchemaMock();
+    delete inputPayload.list_id;
+    const decoded = createExceptionListSchema.decode(inputPayload);
+    const checked = exactCheck(inputPayload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual([]);
+    expect((message.schema as CreateExceptionListSchema).list_id).toMatch(
+      /^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i
+    );
+  });
+
+  test('it should accept an undefined for list_id and generate a correct body not counting the uuid', () => {
+    const inputPayload = getCreateExceptionListSchemaMock();
+    delete inputPayload.list_id;
+    const decoded = createExceptionListSchema.decode(inputPayload);
+    const checked = exactCheck(inputPayload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    delete (message.schema as CreateExceptionListSchema).list_id;
+    expect(message.schema).toEqual(inputPayload);
+  });
+
+  test('it should not allow an extra key to be sent in', () => {
+    const payload: CreateExceptionListSchema & {
+      extraKey?: string;
+    } = getCreateExceptionListSchemaMock();
+    payload.extraKey = 'some new value';
+    const decoded = createExceptionListSchema.decode(payload);
+    const checked = exactCheck(payload, decoded);
+    const message = pipe(checked, foldLeftRight);
+    expect(getPaths(left(message.errors))).toEqual(['invalid keys "extraKey"']);
+    expect(message.schema).toEqual({});
+  });
+});

x-pack/plugins/lists/common/schemas/request/create_exception_list_schema.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import {
+  ListId,
+  Tags,
+  _Tags,
+  _tags,
+  description,
+  exceptionListType,
+  meta,
+  name,
+  tags,
+} from '../common/schemas';
+import { Identity, RequiredKeepUndefined } from '../../types';
+import { DefaultUuid } from '../types/default_uuid';
+
+export const createExceptionListSchema = t.intersection([
+  t.exact(
+    t.type({
+      description,
+      name,
+      type: exceptionListType,
+    })
+  ),
+  t.exact(
+    t.partial({
+      _tags, // defaults to empty array if not set during decode
+      list_id: DefaultUuid, // defaults to a GUID (UUID v4) string if not set during decode
+      meta, // defaults to undefined if not set during decode
+      tags, // defaults to empty array if not set during decode
+    })
+  ),
+]);
+
+export type CreateExceptionListSchemaPartial = Identity<t.TypeOf<typeof createExceptionListSchema>>;
+export type CreateExceptionListSchema = RequiredKeepUndefined<
+  t.TypeOf<typeof createExceptionListSchema>
+>;
+
+// This type is used after a decode since the arrays turn into defaults of empty arrays.
+export type CreateExceptionListSchemaDecoded = Identity<
+  CreateExceptionListSchema & {
+    _tags: _Tags;
+    tags: Tags;
+    list_id: ListId;
+  }
+>;

x-pack/plugins/lists/common/schemas/request/create_list_item_schema.ts

@@ -8,7 +8,7 @@
 
 import * as t from 'io-ts';
 
-import { idOrUndefined, list_id, metaOrUndefined, value } from '../common/schemas';
+import { id, list_id, meta, value } from '../common/schemas';
 import { Identity, RequiredKeepUndefined } from '../../types';
 
 export const createListItemSchema = t.intersection([
@@ -18,7 +18,7 @@ export const createListItemSchema = t.intersection([
       value,
     })
   ),
-  t.exact(t.partial({ id: idOrUndefined, meta: metaOrUndefined })),
+  t.exact(t.partial({ id, meta })),
 ]);
 
 export type CreateListItemSchemaPartial = Identity<t.TypeOf<typeof createListItemSchema>>;

x-pack/plugins/lists/common/schemas/request/create_list_schema.ts

@@ -6,7 +6,7 @@
 
 import * as t from 'io-ts';
 
-import { description, idOrUndefined, metaOrUndefined, name, type } from '../common/schemas';
+import { description, id, meta, name, type } from '../common/schemas';
 import { Identity, RequiredKeepUndefined } from '../../types';
 
 export const createListSchema = t.intersection([
@@ -17,7 +17,7 @@ export const createListSchema = t.intersection([
       type,
     })
   ),
-  t.exact(t.partial({ id: idOrUndefined, meta: metaOrUndefined })),
+  t.exact(t.partial({ id, meta })),
 ]);
 
 export type CreateListSchemaPartial = Identity<t.TypeOf<typeof createListSchema>>;

x-pack/plugins/lists/common/schemas/request/delete_exception_list_item_schema.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+/* eslint-disable @typescript-eslint/camelcase */
+
+import * as t from 'io-ts';
+
+import { id, item_id } from '../common/schemas';
+
+export const deleteExceptionListItemSchema = t.exact(
+  t.partial({
+    id,
+    item_id,
+  })
+);
+
+export type DeleteExceptionListItemSchema = t.TypeOf<typeof deleteExceptionListItemSchema>;