Update vulnerability detector index template (#383)

* Update VD index template

* Remove host.os.family

ecs/generate.sh

@@ -13,6 +13,19 @@ show_usage() {
   echo "Example: $0 v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200"
 }
 
+# Function to remove multi-fields from the generated index template
+remove_multi_fields() {
+  local IN_FILE="$1"
+  local OUT_FILE="$2"
+
+  jq 'del(
+    .mappings.properties.host.properties.os.properties.full.fields,
+    .mappings.properties.host.properties.os.properties.name.fields,
+    .mappings.properties.vulnerability.properties.description.fields
+  )' "$IN_FILE" > "$OUT_FILE"
+}
+
+
 # Function to generate mappings
 generate_mappings() {
   local IN_FILES_DIR="$INDEXER_SRC/ecs/$MODULE/fields"
@@ -34,8 +47,21 @@ generate_mappings() {
   echo "Replacing \"match_only_text\" type with \"text\""
   find "$OUT_DIR" -type f -exec sed -i 's/match_only_text/text/g' {} \;
 
+  local IN_FILE="$OUT_DIR/generated/elasticsearch/legacy/template.json"
+  local OUT_FILE="$OUT_DIR/generated/elasticsearch/legacy/template-tmp.json"
+
+  # Delete the "tags" field from the index template
+  echo "Deleting the \"tags\" field from the index template"
+  jq 'del(.mappings.properties.tags)' "$IN_FILE" > "$OUT_FILE"
+  mv "$OUT_FILE" "$IN_FILE"
+
+  # Remove multi-fields from the generated index template
+  echo "Removing multi-fields from the index template"
+  remove_multi_fields "$IN_FILE" "$OUT_FILE"
+  mv "$OUT_FILE" "$IN_FILE"
+
   # Transform legacy index template for OpenSearch compatibility
-  cat "$OUT_DIR/generated/elasticsearch/legacy/template.json" | jq '{
+  cat "$IN_FILE" | jq '{
     "index_patterns": .index_patterns,
     "priority": .order,
     "template": {

ecs/vulnerability-detector/event-generator/event_generator.py

@@ -2,15 +2,13 @@
 
 # This script generates sample events and injects them into the Wazuh Indexer.
 # The events follow the Elastic Common Schema (ECS) format, and contains the following fields:
-#   - ecs
-#   - base
-#   - event
 #   - agent
 #   - package
 #   - host
 #   - vulnerability
+#   - wazuh (custom)
 #
-# This is an ad-hoc script for the vulnearbility module. Extend to support other modules.
+# This is an ad-hoc script for the vulnerability module. Extend to support other modules.
 
 import datetime
 import random
@@ -102,7 +100,6 @@ def generate_random_host():
     version = f'{random.randint(0, 99)}.{random.randint(0, 99)}'
     host = {
         'os': {
-            'family': family,
             'full': f'{family} {version}',
             'kernel': f'{version}kernel{random.randint(0, 99)}',
             'name': f'{family} {version}',
@@ -141,11 +138,6 @@ def generate_random_package():
     return package
 
 
-def generate_random_tags():
-    tags = [f'tag{random.randint(0, 99)}' for _ in range(random.randint(0, 9))]
-    return tags
-
-
 def generate_random_vulnerability():
     id = random.randint(0, 9999)
     vulnerability = {
@@ -156,7 +148,10 @@ def generate_random_vulnerability():
         'id': f'CVE-{id}',
         'reference': f'https://mycve.test.org/cgi-bin/cvename.cgi?name={id}',
         'report_id': f'report-{random.randint(0, 9999)}',
-        'scanner': {'vendor': f'vendor-{random.randint(0, 9)}'},
+        'scanner': {
+            'vendor': f'vendor-{random.randint(0, 9)}',
+            'source': random.choice(['NVD', 'OpenCVE', 'OpenVAS', 'Tenable'])
+        },
         'score': {
             'base': round(random.uniform(0, 10), 1),
             'environmental': round(random.uniform(0, 10), 1),
@@ -166,6 +161,7 @@ def generate_random_vulnerability():
         'severity': random.choice(['Low', 'Medium', 'High', 'Critical']),
         'detected_at': generate_random_date(),
         'published_at': generate_random_date(),
+        'under_evaluation': random.choice([True, False])
     }
     return vulnerability
 
@@ -176,9 +172,9 @@ def generate_random_wazuh():
             'name': f'wazuh-cluster-{random.randint(0,10)}',
             'node': f'wazuh-cluster-node-{random.randint(0,10)}'
         },
-        'manager': {
-            'name': f'wazuh-manager-{random.randint(0,10)}'
-        },
+        # 'manager': {
+        #     'name': f'wazuh-manager-{random.randint(0,10)}'
+        # },
         'schema': {
             'version': '1.7.0'
         },
@@ -190,15 +186,9 @@ def generate_random_data(number):
     data = []
     for _ in range(number):
         event_data = {
-            # '@timestamp': generate_random_date(),
             'agent': generate_random_agent(),
-            # 'ecs': {'version': '1.7.0'},
-            # 'event': generate_random_event(),
             'host': generate_random_host(),
-            # 'labels': generate_random_labels(),
-            'message': f'message{random.randint(0, 99999)}',
             'package': generate_random_package(),
-            'tags': generate_random_tags(),
             'vulnerability': generate_random_vulnerability(),
             'wazuh': generate_random_wazuh()
         }
@@ -227,7 +217,7 @@ def inject_events(ip, port, index, username, password, data):
 
 def main():
     try:
-        number = int(input("How many events do you want to generate? "))
+        number = int(input("How many events do you want to generate? ").strip() or 50)
     except ValueError:
         logging.error("Invalid input. Please enter a valid number.")
         return
@@ -245,11 +235,11 @@ def main():
     inject = input(
         "Do you want to inject the generated data into your indexer? (y/n) ").strip().lower()
     if inject == 'y':
-        ip = input("Enter the IP of your Indexer: ")
-        port = input("Enter the port of your Indexer: ")
-        index = input("Enter the index name: ")
-        username = input("Username: ")
-        password = input("Password: ")
+        ip = input("Enter the IP of your Indexer: ").strip() or "localhost"
+        port = input("Enter the port of your Indexer: ").strip() or 9200
+        index = input("Enter the index name: ").strip() or "wazuh-states-vulnerability-test"
+        username = input("Username: ").strip() or "admin"
+        password = input("Password: ").strip()
         inject_events(ip, port, index, username, password, data)
 
 

ecs/vulnerability-detector/fields/custom/vulnerability.yml

@@ -16,4 +16,14 @@
       type: date
       level: custom
       description: >
-        Vulnerability's publication date.
+        Vulnerability's publication date.
+    - name: under_evaluation
+      type: boolean
+      level: custom
+      description: >
+        Indicates if the vulnerability is awaiting analysis by the NVD.
+    - name: scanner.source
+      type: keyword
+      level: custom
+      description: >
+        The origin of the decision of the scanner (AKA feed used to detect the vulnerability).

ecs/vulnerability-detector/fields/custom/wazuh.yml

@@ -14,11 +14,6 @@
       level: custom
       description: >
         Wazuh cluster node name.
-    - name: manager.name
-      type: keyword
-      level: custom
-      description: >
-        Wazuh manager name. Used by dashboards to filter results on single node deployments.
     - name: schema.version
       type: keyword
       level: custom

ecs/vulnerability-detector/fields/subset.yml

@@ -4,15 +4,20 @@ fields:
   base:
     fields: 
       tags: []
-      message: ""
   agent:
     fields: "*"
   package:
     fields: "*"
   host:
     fields:
       os:
-        fields: "*"
+        fields: 
+          full: ""
+          kernel: ""
+          name: ""
+          platform: ""
+          type: ""
+          version: ""
   vulnerability:
     fields: "*"
   wazuh:

ecs/vulnerability-detector/fields/template-settings-legacy.json

@@ -1,29 +1,20 @@
 {
-  "index_patterns": [
-    "wazuh-states-vulnerabilities"
-  ],
+  "index_patterns": ["wazuh-states-vulnerabilities-*"],
   "order": 1,
   "settings": {
     "index": {
       "codec": "best_compression",
-      "mapping": {
-        "total_fields": {
-          "limit": 1000
-        }
-      },
       "number_of_shards": "1",
       "number_of_replicas": "0",
-      "refresh_interval": "2s",
+      "refresh_interval": "5s",
       "query.default_field": [
-        "base.tags",
         "agent.id",
-        "host.os.family",
-        "host.os.full.text",
+        "host.os.full",
         "host.os.version",
         "package.name",
         "package.version",
         "vulnerability.id",
-        "vulnerability.description.text",
+        "vulnerability.description",
         "vulnerability.severity",
         "wazuh.cluster.name"
       ]

ecs/vulnerability-detector/fields/template-settings.json

@@ -1,30 +1,21 @@
 {
-  "index_patterns": [
-    "wazuh-states-vulnerabilities"
-  ],
+  "index_patterns": ["wazuh-states-vulnerabilities-*"],
   "priority": 1,
   "template": {
     "settings": {
       "index": {
         "codec": "best_compression",
-        "mapping": {
-          "total_fields": {
-            "limit": 2000
-          }
-        },
         "number_of_shards": "1",
         "number_of_replicas": "0",
-        "refresh_interval": "2s",
+        "refresh_interval": "5s",
         "query.default_field": [
-          "base.tags",
           "agent.id",
-          "host.os.family",
-          "host.os.full.text",
+          "host.os.full",
           "host.os.version",
           "package.name",
           "package.version",
           "vulnerability.id",
-          "vulnerability.description.text",
+          "vulnerability.description",
           "vulnerability.severity",
           "wazuh.cluster.name"
         ]