[Security Solutions] (Phase 1) Adds an application cache called metrics entities and integrates it within Security Solutions behind a feature flag

.eslintrc.js

@@ -1114,6 +1114,118 @@ module.exports = {
         'prefer-destructuring': 'error',
       },
     },
+    /**
+     * Metrics entities overrides
+     */
+    {
+      // front end and common typescript and javascript files only
+      files: [
+        'x-pack/plugins/metrics_entities/public/**/*.{js,mjs,ts,tsx}',
+        'x-pack/plugins/metrics_entities/common/**/*.{js,mjs,ts,tsx}',
+      ],
+      rules: {
+        'import/no-nodejs-modules': 'error',
+        'no-restricted-imports': [
+          'error',
+          {
+            // prevents UI code from importing server side code and then webpack including it when doing builds
+            patterns: ['**/server/*'],
+          },
+        ],
+      },
+    },
+    {
+      // typescript and javascript for front and back end
+      files: ['x-pack/plugins/metrics_entities/**/*.{js,mjs,ts,tsx}'],
+      plugins: ['eslint-plugin-node'],
+      env: {
+        jest: true,
+      },
+      rules: {
+        'accessor-pairs': 'error',
+        'array-callback-return': 'error',
+        'no-array-constructor': 'error',
+        complexity: 'error',
+        'consistent-return': 'error',
+        'func-style': ['error', 'expression'],
+        'import/order': [
+          'error',
+          {
+            groups: ['builtin', 'external', 'internal', 'parent', 'sibling', 'index'],
+            'newlines-between': 'always',
+          },
+        ],
+        'sort-imports': [
+          'error',
+          {
+            ignoreDeclarationSort: true,
+          },
+        ],
+        'node/no-deprecated-api': 'error',
+        'no-bitwise': 'error',
+        'no-continue': 'error',
+        'no-dupe-keys': 'error',
+        'no-duplicate-case': 'error',
+        'no-duplicate-imports': 'error',
+        'no-empty-character-class': 'error',
+        'no-empty-pattern': 'error',
+        'no-ex-assign': 'error',
+        'no-extend-native': 'error',
+        'no-extra-bind': 'error',
+        'no-extra-boolean-cast': 'error',
+        'no-extra-label': 'error',
+        'no-func-assign': 'error',
+        'no-implicit-globals': 'error',
+        'no-implied-eval': 'error',
+        'no-invalid-regexp': 'error',
+        'no-inner-declarations': 'error',
+        'no-lone-blocks': 'error',
+        'no-multi-assign': 'error',
+        'no-misleading-character-class': 'error',
+        'no-new-symbol': 'error',
+        'no-obj-calls': 'error',
+        'no-param-reassign': ['error', { props: true }],
+        'no-process-exit': 'error',
+        'no-prototype-builtins': 'error',
+        'no-return-await': 'error',
+        'no-self-compare': 'error',
+        'no-shadow-restricted-names': 'error',
+        'no-sparse-arrays': 'error',
+        'no-this-before-super': 'error',
+        // rely on typescript
+        'no-undef': 'off',
+        'no-unreachable': 'error',
+        'no-unsafe-finally': 'error',
+        'no-useless-call': 'error',
+        'no-useless-catch': 'error',
+        'no-useless-concat': 'error',
+        'no-useless-computed-key': 'error',
+        'no-useless-escape': 'error',
+        'no-useless-rename': 'error',
+        'no-useless-return': 'error',
+        'no-void': 'error',
+        'one-var-declaration-per-line': 'error',
+        'prefer-object-spread': 'error',
+        'prefer-promise-reject-errors': 'error',
+        'prefer-rest-params': 'error',
+        'prefer-spread': 'error',
+        'prefer-template': 'error',
+        'require-atomic-updates': 'error',
+        'symbol-description': 'error',
+        'vars-on-top': 'error',
+        '@typescript-eslint/explicit-member-accessibility': 'error',
+        '@typescript-eslint/no-this-alias': 'error',
+        '@typescript-eslint/no-explicit-any': 'error',
+        '@typescript-eslint/no-useless-constructor': 'error',
+        '@typescript-eslint/unified-signatures': 'error',
+        '@typescript-eslint/explicit-function-return-type': 'error',
+        '@typescript-eslint/no-non-null-assertion': 'error',
+        '@typescript-eslint/no-unused-vars': 'error',
+        'no-template-curly-in-string': 'error',
+        'sort-keys': 'error',
+        'prefer-destructuring': 'error',
+      },
+    },
     /**
      * Alerting Services overrides
      */

api_docs/metrics_entities.json

@@ -0,0 +1,151 @@
+{
+  "id": "metricsEntities",
+  "client": {
+    "classes": [],
+    "functions": [],
+    "interfaces": [],
+    "enums": [],
+    "misc": [],
+    "objects": []
+  },
+  "server": {
+    "classes": [],
+    "functions": [],
+    "interfaces": [],
+    "enums": [],
+    "misc": [],
+    "objects": [],
+    "setup": {
+      "id": "def-server.MetricsEntitiesPluginSetup",
+      "type": "Interface",
+      "label": "MetricsEntitiesPluginSetup",
+      "description": [],
+      "tags": [],
+      "children": [
+        {
+          "tags": [],
+          "id": "def-server.MetricsEntitiesPluginSetup.getMetricsEntitiesClient",
+          "type": "Function",
+          "label": "getMetricsEntitiesClient",
+          "description": [],
+          "source": {
+            "path": "x-pack/plugins/metrics_entities/server/types.ts",
+            "lineNumber": 15
+          },
+          "signature": [
+            "GetMetricsEntitiesClientType"
+          ]
+        }
+      ],
+      "source": {
+        "path": "x-pack/plugins/metrics_entities/server/types.ts",
+        "lineNumber": 14
+      },
+      "lifecycle": "setup",
+      "initialIsOpen": true
+    },
+    "start": {
+      "id": "def-server.MetricsEntitiesPluginStart",
+      "type": "Type",
+      "label": "MetricsEntitiesPluginStart",
+      "tags": [],
+      "description": [],
+      "source": {
+        "path": "x-pack/plugins/metrics_entities/server/types.ts",
+        "lineNumber": 18
+      },
+      "signature": [
+        "void"
+      ],
+      "lifecycle": "start",
+      "initialIsOpen": true
+    }
+  },
+  "common": {
+    "classes": [],
+    "functions": [],
+    "interfaces": [],
+    "enums": [],
+    "misc": [
+      {
+        "tags": [],
+        "id": "def-common.ELASTIC_NAME",
+        "type": "string",
+        "label": "ELASTIC_NAME",
+        "description": [
+          "\nGlobal prefix for all the transform jobs"
+        ],
+        "source": {
+          "path": "x-pack/plugins/metrics_entities/common/constants.ts",
+          "lineNumber": 21
+        },
+        "signature": [
+          "\"estc\""
+        ],
+        "initialIsOpen": false
+      },
+      {
+        "tags": [],
+        "id": "def-common.METRICS_ENTITIES_TRANSFORMS",
+        "type": "string",
+        "label": "METRICS_ENTITIES_TRANSFORMS",
+        "description": [
+          "\nTransforms route"
+        ],
+        "source": {
+          "path": "x-pack/plugins/metrics_entities/common/constants.ts",
+          "lineNumber": 16
+        },
+        "initialIsOpen": false
+      },
+      {
+        "tags": [],
+        "id": "def-common.METRICS_ENTITIES_URL",
+        "type": "string",
+        "label": "METRICS_ENTITIES_URL",
+        "description": [
+          "\nBase route"
+        ],
+        "source": {
+          "path": "x-pack/plugins/metrics_entities/common/constants.ts",
+          "lineNumber": 11
+        },
+        "signature": [
+          "\"/api/metrics_entities\""
+        ],
+        "initialIsOpen": false
+      },
+      {
+        "tags": [],
+        "id": "def-common.PLUGIN_ID",
+        "type": "string",
+        "label": "PLUGIN_ID",
+        "description": [],
+        "source": {
+          "path": "x-pack/plugins/metrics_entities/common/index.ts",
+          "lineNumber": 8
+        },
+        "signature": [
+          "\"metricsEntities\""
+        ],
+        "initialIsOpen": false
+      },
+      {
+        "tags": [],
+        "id": "def-common.PLUGIN_NAME",
+        "type": "string",
+        "label": "PLUGIN_NAME",
+        "description": [],
+        "source": {
+          "path": "x-pack/plugins/metrics_entities/common/index.ts",
+          "lineNumber": 9
+        },
+        "signature": [
+          "\"metrics_entities\""
+        ],
+        "initialIsOpen": false
+      }
+    ],
+    "objects": []
+  }
+}

api_docs/metrics_entities.mdx

@@ -0,0 +1,26 @@
+---
+id: kibMetricsEntitiesPluginApi
+slug: /kibana-dev-docs/metricsEntitiesPluginApi
+title: metricsEntities
+image: https://source.unsplash.com/400x175/?github
+summary: API docs for the metricsEntities plugin
+date: 2020-11-16
+tags: ['contributor', 'dev', 'apidocs', 'kibana', 'metricsEntities']
+warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
+---
+
+import metricsEntitiesObj from './metrics_entities.json';
+
+## Server
+
+### Setup
+<DocDefinitionList data={[metricsEntitiesObj.server.setup]}/>
+
+### Start
+<DocDefinitionList data={[metricsEntitiesObj.server.start]}/>
+
+## Common
+
+### Consts, variables and types
+<DocDefinitionList data={metricsEntitiesObj.common.misc}/>
+

docs/developer/plugin-list.asciidoc

@@ -474,6 +474,10 @@ using the CURL scripts in the scripts folder.
 |Visualize geo data from Elasticsearch or 3rd party geo-services.
 
 
+|{kib-repo}blob/{branch}/x-pack/plugins/metrics_entities/README.md[metricsEntities]
+|Metrics entities where you can add transforms for metrics and entities.
+
+
 |{kib-repo}blob/{branch}/x-pack/plugins/ml/readme.md[ml]
 |This plugin provides access to the machine learning features provided by
 Elastic.

packages/kbn-cli-dev-mode/src/get_server_watch_paths.test.ts

@@ -75,6 +75,7 @@ it('produces the right watch and ignore list', () => {
       <absolute path>/x-pack/plugins/lists/server/scripts,
       <absolute path>/x-pack/plugins/security_solution/scripts,
       <absolute path>/x-pack/plugins/security_solution/server/lib/detection_engine/scripts,
+      <absolute path>/x-pack/plugins/metrics_entities/server/scripts,
     ]
   `);
 });

packages/kbn-cli-dev-mode/src/get_server_watch_paths.ts

@@ -66,6 +66,7 @@ export function getServerWatchPaths({ pluginPaths, pluginScanDirs }: Options) {
     fromRoot('x-pack/plugins/lists/server/scripts'),
     fromRoot('x-pack/plugins/security_solution/scripts'),
     fromRoot('x-pack/plugins/security_solution/server/lib/detection_engine/scripts'),
+    fromRoot('x-pack/plugins/metrics_entities/server/scripts'),
   ];
 
   return {

tsconfig.json

@@ -17,7 +17,7 @@
     "x-pack/typings/**/*",
     "x-pack/tasks/**/*",
     "x-pack/plugins/lists/**/*",
-    "x-pack/plugins/security_solution/**/*",
+    "x-pack/plugins/security_solution/**/*"
   ],
   "exclude": [
     "x-pack/plugins/security_solution/cypress/**/*"
@@ -110,6 +110,7 @@
     { "path": "./x-pack/plugins/licensing/tsconfig.json" },
     { "path": "./x-pack/plugins/logstash/tsconfig.json" },
     { "path": "./x-pack/plugins/maps/tsconfig.json" },
+    { "path": "./x-pack/plugins/metrics_entities/tsconfig.json" },
     { "path": "./x-pack/plugins/ml/tsconfig.json" },
     { "path": "./x-pack/plugins/monitoring/tsconfig.json" },
     { "path": "./x-pack/plugins/observability/tsconfig.json" },

tsconfig.refs.json

@@ -87,6 +87,7 @@
     { "path": "./x-pack/plugins/licensing/tsconfig.json" },
     { "path": "./x-pack/plugins/logstash/tsconfig.json" },
     { "path": "./x-pack/plugins/maps/tsconfig.json" },
+    { "path": "./x-pack/plugins/metrics_entities/tsconfig.json" },
     { "path": "./x-pack/plugins/ml/tsconfig.json" },
     { "path": "./x-pack/plugins/monitoring/tsconfig.json" },
     { "path": "./x-pack/plugins/observability/tsconfig.json" },

x-pack/.i18nrc.json

@@ -39,6 +39,7 @@
     "xpack.logstash": ["plugins/logstash"],
     "xpack.main": "legacy/plugins/xpack_main",
     "xpack.maps": ["plugins/maps"],
+    "xpack.metricsEntities": "plugins/metrics_entities",
     "xpack.ml": ["plugins/ml"],
     "xpack.monitoring": ["plugins/monitoring"],
     "xpack.osquery": ["plugins/osquery"],

x-pack/plugins/metrics_entities/README.md

@@ -0,0 +1,34 @@
+# metrics_entities
+
+Metrics entities where you can add transforms for metrics and entities.
+
+The transform jobs are on the file system as:
+ - modules (The different logical pieces)
+
+When you create a "group" of transforms you can create them using an optional prefix to separate things based on queries, security concerns, or different Kibana spaces.
+---
+
+## Scripts
+
+TODO: Add a note about scripts, setup, and running
+
+## Development
+
+TODO: Add how to develop here
+
+## TODO List
+- Add these properties to the route which are:
+  - disable_transforms,
+  - pipeline,
+   - Change the REST routes on post to change the indexes for whichever indexes you want
+ - Unit tests to ensure the data of the mapping.json includes the correct fields such as
+   _meta, at least one alias, a mapping section, etc... 
+ - Add text/keyword and other things to the mappings (not just keyword maybe?) ... At least review them one more time
+ - Module (server/modules) templating with variables and then push those variables down? (Maybe we don't need them)
+ - Add feature flag and have it turned off
+ - Add lib/modules folder with the transforms inside of it
+ - Add the REST Kibana security based tags if needed and push those to any plugins using this plugin. Something like: tags: ['access:metricsEntities-read'] and ['access:metricsEntities-all'],
+ - Add schema validation choosing some schema library (io-ts or Kibana Schema or ... )
+ - Add unit tests
+ - Add e2e tests
+ - Move ui code into this plugin from security solutions? (maybe?)