[ML] Fixing module datafeed overrides

x-pack/plugins/ml/common/types/modules.ts

@@ -11,7 +11,7 @@ export interface ModuleJob {
   config: Omit<Job, 'job_id'>;
 }
 
-export interface ModuleDataFeed {
+export interface ModuleDatafeed {
   id: string;
   config: Omit<Datafeed, 'datafeed_id'>;
 }
@@ -49,7 +49,7 @@ export interface Module {
   defaultIndexPattern: string;
   query: any;
   jobs: ModuleJob[];
-  datafeeds: ModuleDataFeed[];
+  datafeeds: ModuleDatafeed[];
   kibana: KibanaObjects;
 }
 

x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts

@@ -17,7 +17,7 @@ import { MlInfoResponse } from '../../../common/types/ml_server_info';
 import {
   KibanaObjects,
   KibanaObjectConfig,
-  ModuleDataFeed,
+  ModuleDatafeed,
   ModuleJob,
   Module,
   JobOverride,
@@ -283,7 +283,7 @@ export class DataRecognizer {
     }
 
     const jobs: ModuleJob[] = [];
-    const datafeeds: ModuleDataFeed[] = [];
+    const datafeeds: ModuleDatafeed[] = [];
     const kibana: KibanaObjects = {};
     // load all of the job configs
     await Promise.all(
@@ -710,7 +710,7 @@ export class DataRecognizer {
   // save the datafeeds.
   // if any fail (e.g. it already exists), catch the error and mark the result
   // as success: false
-  async saveDatafeeds(datafeeds: ModuleDataFeed[]) {
+  async saveDatafeeds(datafeeds: ModuleDatafeed[]) {
     return await Promise.all(
       datafeeds.map(async (datafeed) => {
         try {
@@ -723,7 +723,7 @@ export class DataRecognizer {
     );
   }
 
-  async saveDatafeed(datafeed: ModuleDataFeed) {
+  async saveDatafeed(datafeed: ModuleDatafeed) {
     return this._asInternalUser.ml.putDatafeed(
       {
         datafeed_id: datafeed.id,
@@ -734,7 +734,7 @@ export class DataRecognizer {
   }
 
   async startDatafeeds(
-    datafeeds: ModuleDataFeed[],
+    datafeeds: ModuleDatafeed[],
     start?: number,
     end?: number
   ): Promise<{ [key: string]: DatafeedResponse }> {
@@ -746,7 +746,7 @@ export class DataRecognizer {
   }
 
   async startDatafeed(
-    datafeed: ModuleDataFeed,
+    datafeed: ModuleDatafeed,
     start: number | undefined,
     end: number | undefined
   ): Promise<DatafeedResponse> {
@@ -1229,6 +1229,25 @@ export class DataRecognizer {
       const overrides = Array.isArray(datafeedOverrides) ? datafeedOverrides : [datafeedOverrides];
       const { datafeeds } = moduleConfig;
 
+      // for some items in the datafeed, we should not merge.
+      // we should instead use the whole override object
+      function overwriteObjects(source: ModuleDatafeed['config'], update: DatafeedOverride) {
+        Object.entries(update).forEach(([key, val]) => {
+          if (typeof val === 'object') {
+            switch (key) {
+              case 'query':
+              case 'aggregations':
+              case 'aggs':
+              case 'script_fields':
+                source[key] = val as any;
+                break;
+              default:
+                break;
+            }
+          }
+        });
+      }
+
       // separate all the overrides.
       // the overrides which don't contain a datafeed id or a job id will be applied to all jobs in the module
       const generalOverrides: GeneralDatafeedsOverride[] = [];
@@ -1244,6 +1263,7 @@ export class DataRecognizer {
       generalOverrides.forEach((o) => {
         datafeeds.forEach(({ config }) => {
           merge(config, o);
+          overwriteObjects(config, o);
         });
       });
 
@@ -1259,6 +1279,7 @@ export class DataRecognizer {
           delete o.job_id;
           delete o.datafeed_id;
           merge(datafeed.config, o);
+          overwriteObjects(datafeed.config, o);
         }
       });
     }

x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json

@@ -1,7 +1,9 @@
 {
   "job_type": "anomaly_detector",
   "description": "Logs UI: Detects anomalies in count of log entries by category",
-  "groups": ["logs-ui"],
+  "groups": [
+    "logs-ui"
+  ],
   "analysis_config": {
     "bucket_span": "15m",
     "categorization_field_name": "message",
@@ -14,7 +16,10 @@
         "use_null": true
       }
     ],
-    "influencers": ["event.dataset", "mlcategory"],
+    "influencers": [
+      "event.dataset",
+      "mlcategory"
+    ],
     "per_partition_categorization": {
       "enabled": true,
       "stop_on_warn": false

x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/logo.json

@@ -1,3 +1,3 @@
 {
-	"icon": "metricbeatApp"
+  "icon": "metricbeatApp"
 }

x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/manifest.json

@@ -8,7 +8,12 @@
   "query": {
     "bool": {
       "filter": {
-        "terms" : { "event.dataset" : ["system.cpu", "system.filesystem"]}
+        "terms": {
+          "event.dataset": [
+            "system.cpu",
+            "system.filesystem"
+          ]
+        }
       }
     }
   },

x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_high_mean_cpu_iowait_ecs.json

@@ -6,10 +6,14 @@
   "query": {
     "bool": {
       "filter": {
-        "term":  { "event.dataset": "system.cpu" }
+        "term": {
+          "event.dataset": "system.cpu"
+        }
       },
       "must": {
-        "exists": { "field": "system.cpu.iowait.pct" }
+        "exists": {
+          "field": "system.cpu.iowait.pct"
+        }
       }
     }
   }

x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_max_disk_utilization_ecs.json

@@ -1,16 +1,20 @@
 {
-    "job_id": "JOB_ID",
-    "indices": [
-      "INDEX_PATTERN_NAME"
-    ],
-    "query": {
-      "bool": {
-        "filter": {
-          "term":  { "event.dataset": "system.filesystem" }
-        },
-        "must": {
-          "exists": { "field": "system.filesystem.used.pct" }
+  "job_id": "JOB_ID",
+  "indices": [
+    "INDEX_PATTERN_NAME"
+  ],
+  "query": {
+    "bool": {
+      "filter": {
+        "term": {
+          "event.dataset": "system.filesystem"
+        }
+      },
+      "must": {
+        "exists": {
+          "field": "system.filesystem.used.pct"
         }
       }
     }
   }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/datafeed_metricbeat_outages_ecs.json

@@ -1,13 +1,15 @@
 {
-    "job_id": "JOB_ID",
-    "indices": [
-      "INDEX_PATTERN_NAME"
-    ],
-    "query": {
-      "bool": {
-        "must": {
-          "exists": { "field": "event.dataset" }
+  "job_id": "JOB_ID",
+  "indices": [
+    "INDEX_PATTERN_NAME"
+  ],
+  "query": {
+    "bool": {
+      "must": {
+        "exists": {
+          "field": "event.dataset"
         }
       }
     }
   }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metricbeat_system_ecs/ml/high_mean_cpu_iowait_ecs.json

@@ -1,54 +1,56 @@
 {
-    "job_type": "anomaly_detector",
-    "description": "Metricbeat CPU: Detect unusual increases in cpu time spent in iowait (ECS)",
-    "groups": ["metricbeat"],
-    "analysis_config": {
-      "bucket_span": "10m",
-      "detectors": [
-        {
-          "detector_description": "high mean system.cpu.iowait.pct",
-          "function": "high_mean",
-          "field_name": "system.cpu.iowait.pct",
-          "partition_field_name": "host.name",
-          "custom_rules": [
-            {
-              "actions": [
-                "skip_result"
-              ],
-              "conditions": [
-                {
-                  "applies_to": "actual",
-                  "operator": "lt",
-                  "value": 0.25
-                }
-              ]
-            }
-          ]
-        }
-      ],
-      "influencers": [
-        "host.name"
-      ]
-    },
-    "analysis_limits": {
-      "model_memory_limit": "25mb"
-    },
-    "data_description": {
-      "time_field": "@timestamp",
-      "time_format": "epoch_ms"
-    },
-    "custom_settings": {
-      "created_by": "ml-module-metricbeat-system",
-      "custom_urls": [
-        {
-            "url_name": "Host overview",
-            "time_range": "3h",
-            "url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
-        },
-        {
-          "url_name": "Raw data",
-          "url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.cpu\u0022'),sort:!('@timestamp',desc))"
-        }
-      ]
-    }
+  "job_type": "anomaly_detector",
+  "description": "Metricbeat CPU: Detect unusual increases in cpu time spent in iowait (ECS)",
+  "groups": [
+    "metricbeat"
+  ],
+  "analysis_config": {
+    "bucket_span": "10m",
+    "detectors": [
+      {
+        "detector_description": "high mean system.cpu.iowait.pct",
+        "function": "high_mean",
+        "field_name": "system.cpu.iowait.pct",
+        "partition_field_name": "host.name",
+        "custom_rules": [
+          {
+            "actions": [
+              "skip_result"
+            ],
+            "conditions": [
+              {
+                "applies_to": "actual",
+                "operator": "lt",
+                "value": 0.25
+              }
+            ]
+          }
+        ]
+      }
+    ],
+    "influencers": [
+      "host.name"
+    ]
+  },
+  "analysis_limits": {
+    "model_memory_limit": "25mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp",
+    "time_format": "epoch_ms"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-metricbeat-system",
+    "custom_urls": [
+      {
+        "url_name": "Host overview",
+        "time_range": "3h",
+        "url_value": "dashboards#/view/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs?_g=(time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(filters:!(),query:(language:kuery,query:\u0027host.name:\u0022$host.name$\u0022\u0027))"
+      },
+      {
+        "url_name": "Raw data",
+        "url_value": "discover#/?_g=(refreshInterval:(pause:!t,value:0),time:(from:\u0027$earliest$\u0027,mode:absolute,to:\u0027$latest$\u0027))&_a=(columns:!(_source),index:\u0027INDEX_PATTERN_ID\u0027,interval:auto,query:(language:kuery,query:'event.dataset:\u0022system.cpu\u0022'),sort:!('@timestamp',desc))"
+      }
+    ]
   }
+}