Adds Network ML module with four ML jobs for ECS network data (#96480)

* network module

adds the network module with four ml jobs for the 7.13 release

* Update datafeed_high_count_network_denies.json

json formatting

* update test

added the security_network module to the list

* renames

module name change to security_network / Security: Network

* formatting

change hyphen char to underscores

* fixes and name changes

fixes to df queries, descriptions. created_by param

* update tests

tests need the security_network module added

* formatting

change hyphens to underscores

* descriptions

format descriptions

* Update datafeed_high_count_network_events.json

indentation fixes

* Update x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/manifest.json

Co-authored-by: Lisa Cawley <lcawley@elastic.co>

* Update x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/ml/high_count_network_denies.json

Co-authored-by: Lisa Cawley <lcawley@elastic.co>

* Update datafeed_high_count_network_events.json

change to a filter

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Lisa Cawley <lcawley@elastic.co>

x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/logo.json

@@ -0,0 +1,3 @@
+{
+  "icon": "logoSecurity"
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/manifest.json

@@ -0,0 +1,63 @@
+{
+  "id": "security_network",
+  "title": "Security: Network",
+  "description": "Detect anomalous network activity in your ECS-compatible network logs.",
+  "type": "network data",
+  "logoFile": "logo.json",
+  "defaultIndexPattern": [
+    "logs-*",
+    "filebeat-*",
+    "packetbeat-*"
+  ],
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "term": {
+            "event.category": "network"
+          }
+        }
+      ]
+    }
+  },
+  "jobs": [
+    {
+      "id": "high_count_by_destination_country",
+      "file": "high_count_by_destination_country.json"
+    },
+    {
+      "id": "high_count_network_denies",
+      "file": "high_count_network_denies.json"
+    },
+    {
+      "id": "high_count_network_events",
+      "file": "high_count_network_events.json"
+    },
+    {
+      "id": "rare_destination_country",
+      "file": "rare_destination_country.json"
+    }
+  ],
+  "datafeeds": [
+    {
+      "id": "datafeed_high_count_by_destination_country",
+      "file": "datafeed_high_count_by_destination_country.json",
+      "job_id": "high_count_by_destination_country"
+    },
+    {
+      "id": "datafeed_high_count_network_denies",
+      "file": "datafeed_high_count_network_denies.json",
+      "job_id": "high_count_network_denies"
+    },
+    {
+      "id": "datafeed_high_count_network_events",
+      "file": "datafeed_high_count_network_events.json",
+      "job_id": "high_count_network_events"
+    },
+    {
+      "id": "datafeed_rare_destination_country",
+      "file": "datafeed_rare_destination_country.json",
+      "job_id": "rare_destination_country"
+    }
+  ]
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/ml/datafeed_high_count_by_destination_country.json

@@ -0,0 +1,25 @@
+{
+  "job_id": "high_count_by_destination_country",
+  "indices": [
+    "logs-*",
+    "filebeat-*",
+    "packetbeat-*"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "term": {
+            "event.category": "network"
+          }
+        },
+        {
+          "exists": {
+            "field": "destination.geo.country_name"
+          }
+        }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/ml/datafeed_high_count_network_denies.json

@@ -0,0 +1,25 @@
+{
+  "job_id": "high_count_network_denies",
+  "indices": [
+    "logs-*",
+    "filebeat-*",
+    "packetbeat-*"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "term": {
+            "event.category": "network"
+          }
+        },
+        {
+          "term": {
+            "event.outcome": "deny"
+          }
+        }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/ml/datafeed_high_count_network_events.json

@@ -0,0 +1,20 @@
+{
+  "job_id": "high_count_network_events",
+  "indices": [
+    "logs-*",
+    "filebeat-*",
+    "packetbeat-*"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "term": {
+            "event.category": "network"
+          }
+        }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/ml/datafeed_rare_destination_country.json

@@ -0,0 +1,25 @@
+{
+  "job_id": "rare_destination_country",
+  "indices": [
+    "logs-*",
+    "filebeat-*",
+    "packetbeat-*"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "filter": [
+        {
+          "term": {
+            "event.category": "network"
+          }
+        },
+        {
+          "exists": {
+            "field": "destination.geo.country_name"
+          }
+        }
+      ]
+    }
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/ml/high_count_by_destination_country.json

@@ -0,0 +1,35 @@
+{
+  "job_type": "anomaly_detector",
+  "description": "Security: Network - looks for an unusually large spike in network activity to one destination country in the network logs. This could be due to unusually large amounts of reconnaissance or enumeration traffic. Data exfiltration activity may also produce such a surge in traffic to a destination country which does not normally appear in network traffic or business work-flows. Malware instances and persistence mechanisms may communicate with command-and-control (C2) infrastructure in their country of origin, which may be an unusual destination country for the source network.",
+  "groups": [
+    "security",
+    "network"
+  ],
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "high_non_zero_count by \"destination.geo.country_name\"",
+        "function": "high_non_zero_count",
+        "by_field_name": "destination.geo.country_name",
+        "detector_index": 0
+      }
+    ],
+    "influencers": [
+      "destination.geo.country_name",
+      "destination.as.organization.name",
+      "source.ip",
+      "destination.ip"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "32mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-network"
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/ml/high_count_network_denies.json

@@ -0,0 +1,34 @@
+{
+  "job_type": "anomaly_detector",
+  "description": "Security: Network - looks for an unusually large spike in network traffic that was denied by network ACLs or firewall rules. Such a burst of denied traffic is usually either 1) a misconfigured application or firewall or 2) suspicious or malicious activity. Unsuccessful attempts at network transit, in order to connect to command-and-control (C2), or engage in data exfiltration, may produce a burst of failed connections. This could also be due to unusually large amounts of reconnaissance or enumeration traffic.  Denial-of-service attacks or traffic floods may also produce such a surge in traffic.",
+  "groups": [
+    "security",
+    "network"
+  ],
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "high_count",
+        "function": "high_count",
+        "detector_index": 0
+      }
+    ],
+    "influencers": [
+      "destination.geo.country_name",
+      "destination.as.organization.name",
+      "source.ip",
+      "destination.port"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "32mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-network"
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/ml/high_count_network_events.json

@@ -0,0 +1,34 @@
+{
+  "job_type": "anomaly_detector",
+  "description": "Security: Network - looks for an unusually large spike in network traffic. Such a burst of traffic, if not caused by a surge in business activity, can be due to suspicious or malicious activity. Large-scale data exfiltration may produce a burst of network traffic; this could also be due to unusually large amounts of reconnaissance or enumeration traffic.  Denial-of-service attacks or traffic floods may also produce such a surge in traffic.",
+  "groups": [
+    "security",
+    "network"
+  ],
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "high_count",
+        "function": "high_count",
+        "detector_index": 0
+      }
+    ],
+    "influencers": [
+      "destination.geo.country_name",
+      "destination.as.organization.name",
+      "source.ip",
+      "destination.ip"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "32mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-network"
+  }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/security_network/ml/rare_destination_country.json

@@ -0,0 +1,35 @@
+{
+  "job_type": "anomaly_detector",
+  "description": "Security: Network - looks for an unusual destination country name in the network logs. This can be due to initial access, persistence, command-and-control, or exfiltration activity. For example, when a user clicks on a link in a phishing email or opens a malicious document, a request may be sent to download and run a payload from a server in a country which does not normally appear in network traffic or business work-flows. Malware instances and persistence mechanisms may communicate with command-and-control (C2) infrastructure in their country of origin, which may be an unusual destination country for the source network.",
+  "groups": [
+    "security",
+    "network"
+  ],
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "rare by \"destination.geo.country_name\"",
+        "function": "rare",
+        "by_field_name": "destination.geo.country_name",
+        "detector_index": 0
+      }
+    ],
+    "influencers": [
+      "destination.geo.country_name",
+      "destination.as.organization.name",
+      "source.ip",
+      "destination.ip"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "32mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-security-network"
+  }
+}

x-pack/test/api_integration/apis/ml/modules/get_module.ts

@@ -27,6 +27,7 @@ const moduleIds = [
   'sample_data_ecommerce',
   'sample_data_weblogs',
   'security_linux',
+  'security_network',
   'security_windows',
   'siem_auditbeat',
   'siem_auditbeat_auth',

x-pack/test/api_integration/apis/ml/modules/recognize_module.ts

@@ -143,7 +143,7 @@ export default ({ getService }: FtrProviderContext) => {
       user: USER.ML_POWERUSER,
       expected: {
         responseCode: 200,
-        moduleIds: ['security_linux', 'security_windows'],
+        moduleIds: ['security_linux', 'security_network', 'security_windows'],
       },
     },
   ];