[ML] Adds Metadata and Discovery Analysis Jobs to Security Integration

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/manifest.json

@@ -40,6 +40,46 @@
     {
       "id": "linux_anomalous_user_name_ecs",
       "file": "linux_anomalous_user_name_ecs.json"
+    },
+    {
+      "id": "linux_rare_metadata_process",
+      "file": "linux_rare_metadata_process.json"
+    },
+    {
+      "id": "linux_rare_metadata_user",
+      "file": "linux_rare_metadata_user.json"
+    },
+    {
+      "id": "linux_rare_user_compiler",
+      "file": "linux_rare_user_compiler.json"
+    },
+    {
+      "id": "linux_rare_kernel_module_arguments",
+      "file": "linux_rare_kernel_module_arguments.json"
+    },
+    {
+      "id": "linux_rare_sudo_user",
+      "file": "linux_rare_sudo_user.json"
+    },
+    {
+      "id": "linux_system_user_discovery",
+      "file": "linux_system_user_discovery.json"
+    },
+    {
+      "id": "linux_system_information_discovery",
+      "file": "linux_system_information_discovery.json"
+    },
+    {
+      "id": "linux_system_process_discovery",
+      "file": "linux_system_process_discovery.json"
+    },
+    {
+      "id": "linux_network_connection_discovery",
+      "file": "linux_network_connection_discovery.json"
+    },
+    {
+      "id": "linux_network_configuration_discovery",
+      "file": "linux_network_configuration_discovery.json"
     }
   ],
   "datafeeds": [
@@ -77,6 +117,56 @@
       "id": "datafeed-linux_anomalous_user_name_ecs",
       "file": "datafeed_linux_anomalous_user_name_ecs.json",
       "job_id": "linux_anomalous_user_name_ecs"
+    },
+    {
+      "id": "datafeed-linux_rare_metadata_process",
+      "file": "datafeed_linux_rare_metadata_process.json",
+      "job_id": "linux_rare_metadata_process"
+    },
+    {
+      "id": "datafeed-linux_rare_metadata_user",
+      "file": "datafeed_linux_rare_metadata_user.json",
+      "job_id": "linux_rare_metadata_user"
+    },
+    {
+      "id": "datafeed-linux_rare_user_compiler",
+      "file": "datafeed_linux_rare_user_compiler.json",
+      "job_id": "linux_rare_user_compiler"
+    },
+    {
+      "id": "datafeed-linux_rare_kernel_module_arguments",
+      "file": "datafeed_linux_rare_kernel_module_arguments.json",
+      "job_id": "linux_rare_kernel_module_arguments"
+    },
+    {
+      "id": "datafeed-linux_rare_sudo_user",
+      "file": "datafeed_linux_rare_sudo_user.json",
+      "job_id": "linux_rare_sudo_user"
+    },
+    {
+      "id": "datafeed-linux_system_information_discovery",
+      "file": "datafeed_linux_system_information_discovery.json",
+      "job_id": "linux_system_information_discovery"
+    },
+    {
+      "id": "datafeed-linux_system_process_discovery",
+      "file": "datafeed_linux_system_process_discovery.json",
+      "job_id": "linux_system_process_discovery"
+    },
+    {
+      "id": "datafeed-linux_system_user_discovery",
+      "file": "datafeed_linux_system_user_discovery.json",
+      "job_id": "linux_system_user_discovery"
+    },
+    {
+      "id": "datafeed-linux_network_configuration_discovery",
+      "file": "datafeed_linux_network_configuration_discovery.json",
+      "job_id": "linux_network_configuration_discovery"
+    },
+    {
+      "id": "datafeed-linux_network_connection_discovery",
+      "file": "datafeed_linux_network_connection_discovery.json",
+      "job_id": "linux_network_connection_discovery"
     }
   ]
 }

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_network_configuration_discovery.json

@@ -0,0 +1,26 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "must": [
+              {
+                "bool": {
+                  "should": [
+                    {"term": {"process.name": "arp"}},
+                    {"term": {"process.name": "echo"}},
+                    {"term": {"process.name": "ethtool"}},
+                    {"term": {"process.name": "ifconfig"}},
+                    {"term": {"process.name": "ip"}},
+                    {"term": {"process.name": "iptables"}},
+                    {"term": {"process.name": "ufw"}}
+                  ]
+                }
+              }
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_network_connection_discovery.json

@@ -0,0 +1,23 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "must": [
+              {
+                "bool": {
+                  "should": [
+                    {"term": {"process.name": "netstat"}},
+                    {"term": {"process.name": "ss"}},
+                    {"term": {"process.name": "route"}},
+                    {"term": {"process.name": "showmount"}}
+                  ]
+                }
+              }
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_kernel_module_arguments.json

@@ -0,0 +1,22 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "filter": [{"exists": {"field": "process.title"}}],
+            "must": [
+                {"bool": {
+                    "should": [
+                    {"term": {"process.name": "insmod"}},
+                    {"term": {"process.name": "kmod"}},
+                    {"term": {"process.name": "modprobe"}},
+                    {"term": {"process.name": "rmod"}}
+                    ]
+                }}
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_metadata_process.json

@@ -0,0 +1,12 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+      "bool": {
+        "filter": [{"term": {"destination.ip": "169.254.169.254"}}]
+      }
+    }
+  }

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_metadata_user.json

@@ -0,0 +1,12 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+      "bool": {
+        "filter": [{"term": {"destination.ip": "169.254.169.254"}}]
+      }
+    }
+  }

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_sudo_user.json

@@ -0,0 +1,15 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+      "bool": {
+        "filter": [
+          {"term": {"event.action": "executed"}},
+          {"term": {"process.name": "sudo"}}
+        ]
+      }
+    }
+  }

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_rare_user_compiler.json

@@ -0,0 +1,22 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "filter": [{"term": {"event.action": "executed"}}],
+            "must": [
+              {"bool": {
+                  "should": [
+                    {"term": {"process.name": "compile"}},
+                    {"term": {"process.name": "gcc"}},
+                    {"term": {"process.name": "make"}},
+                    {"term": {"process.name": "yasm"}}
+                  ]
+                }}
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_information_discovery.json

@@ -0,0 +1,31 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "must": [
+              {
+                "bool": {
+                  "should": [
+                    {"term": {"process.name": "cat"}},
+                    {"term": {"process.name": "grep"}},
+                    {"term": {"process.name": "head"}},
+                    {"term": {"process.name": "hostname"}},
+                    {"term": {"process.name": "less"}},
+                    {"term": {"process.name": "ls"}},
+                    {"term": {"process.name": "lsmod"}},
+                    {"term": {"process.name": "more"}},
+                    {"term": {"process.name": "strings"}},
+                    {"term": {"process.name": "tail"}},
+                    {"term": {"process.name": "uptime"}},
+                    {"term": {"process.name": "uname"}}
+                  ]
+                }
+              }
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_process_discovery.json

@@ -0,0 +1,21 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "must": [
+              {
+                "bool": {
+                  "should": [
+                    {"term": {"process.name": "ps"}},
+                    {"term": {"process.name": "top"}}
+                  ]
+                }
+              }
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/datafeed_linux_system_user_discovery.json

@@ -0,0 +1,23 @@
+{
+    "job_id": "JOB_ID",
+    "indices": [
+      "INDEX_PATTERN_NAME"
+    ],
+    "max_empty_searches": 10,
+    "query": {
+        "bool": {
+            "must": [
+              {
+                "bool": {
+                  "should": [
+                    {"term": {"process.name": "users"}},
+                    {"term": {"process.name": "w"}},
+                    {"term": {"process.name": "who"}},
+                    {"term": {"process.name": "whoami"}}
+                  ]
+                }
+              }
+            ]
+        }
+    }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat/ml/linux_network_configuration_discovery.json

@@ -0,0 +1,52 @@
+{
+    "job_type": "anomaly_detector",
+    "description": "Security: Auditbeat - Looks for commands related to system network configuration discovery from an unusual user context. This can be due to uncommon troubleshooting activity or due to a compromised account. A compromised account may be used by a threat actor to engage in system network configuration discovery in order to increase their understanding of connected networks and hosts. This information may be used to shape follow-up behaviors such as lateral movement or additional discovery.",
+    "groups": [
+      "security",
+      "auditbeat",
+      "process"
+    ],
+    "analysis_config": {
+      "bucket_span": "15m",
+      "detectors": [
+        {
+          "detector_description": "rare by \"user.name\"",
+          "function": "rare",
+          "by_field_name": "user.name"
+        }
+      ],
+      "influencers": [
+        "process.name",
+        "host.name",
+        "process.args"
+      ]
+    },
+    "allow_lazy_open": true,
+    "analysis_limits": {
+      "model_memory_limit": "64mb"
+    },
+    "data_description": {
+      "time_field": "@timestamp"
+    },
+    "custom_settings": {
+      "created_by": "ml-module-siem-auditbeat",
+      "custom_urls": [
+        {
+          "url_name": "Host Details by process name",
+          "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+        },
+        {
+          "url_name": "Host Details by user name",
+          "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+        },
+        {
+          "url_name": "Hosts Overview by process name",
+          "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+        },
+        {
+          "url_name": "Hosts Overview by user name",
+          "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
+        }
+      ]
+    }
+  }