for idaholab#491; added a logstash health check

mmguero-dev · Nov 12, 2024 · 28b6d65 · 28b6d65
1 parent c6e037e
commit 28b6d65
Show file tree

Hide file tree

Showing 5 changed files with 22 additions and 23 deletions.
diff --git a/api/project/__init__.py b/api/project/__init__.py
@@ -169,15 +169,6 @@
 missing_field_map['ip'] = '0.0.0.0'
 missing_field_map['long'] = 0
 
-logstash_default_pipelines = [
-    "malcolm-beats",
-    "malcolm-enrichment",
-    "malcolm-input",
-    "malcolm-output",
-    "malcolm-suricata",
-    "malcolm-zeek",
-]
-
 urllib3.disable_warnings()
 warnings.filterwarnings(
     "ignore",
@@ -937,7 +928,7 @@ def ready():
     logstash_lumberjack
         true or false, the ready status of Logstash's lumberjack protocol listener
     logstash_pipelines
-        true or false, the ready status of Logstash's default pipelines
+        true or false, the ready status of Logstash's pipelines
     netbox
         true or false, the ready status of NetBox
     opensearch
@@ -998,9 +989,9 @@ def ready():
             print(f"{type(e).__name__}: {str(e)} getting freq status")
 
     try:
-        logstashStats = requests.get(f'{logstashUrl}/_node').json()
+        logstashHealth = requests.get(f'{logstashUrl}/_health_report').json()
     except Exception as e:
-        logstashStats = {}
+        logstashHealth = {}
         if debugApi:
             print(f"{type(e).__name__}: {str(e)} getting Logstash node status")
 
@@ -1057,11 +1048,8 @@ def ready():
         filebeat_tcp=filebeatTcpJsonStatus,
         freq=freqStatus,
         logstash_lumberjack=logstashLJStatus,
-        logstash_pipelines=(malcolm_utils.deep_get(logstashStats, ["status"]) == "green")
-        and all(
-            pipeline in malcolm_utils.deep_get(logstashStats, ["pipelines"], {})
-            for pipeline in logstash_default_pipelines
-        ),
+        logstash_pipelines=(malcolm_utils.deep_get(logstashHealth, ["status"]) == "green")
+        and (malcolm_utils.deep_get(logstashHealth, ["indicators", "pipelines", "status"]) == "green"),
         netbox=bool(
             isinstance(netboxStatus, dict)
             and netboxStatus

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
@@ -290,7 +290,7 @@ services:
       target: /usr/share/logstash/malcolm-ruby
       read_only: true
     healthcheck:
-      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9600"]
+      test: ["CMD", "/usr/local/bin/container_health.sh"]
       interval: 30s
       timeout: 15s
       retries: 3

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -254,7 +254,7 @@ services:
       target: /certs/server.key
       read_only: true
     healthcheck:
-      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9600"]
+      test: ["CMD", "/usr/local/bin/container_health.sh"]
       interval: 30s
       timeout: 15s
       retries: 3

diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml
@@ -90,10 +90,9 @@ spec:
           - configMapRef:
               name: logstash-env
         livenessProbe:
-          httpGet:
-            path: /
-            port: 9600
-            scheme: HTTP
+          exec:
+            command:
+            - /usr/local/bin/container_health.sh
           initialDelaySeconds: 600
           periodSeconds: 30
           timeoutSeconds: 15

diff --git a/logstash/scripts/container_health.sh b/logstash/scripts/container_health.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -euo pipefail
+
+JQ_EVAL=$(
+    curl --fail --silent -XGET http://localhost:9600/_health_report | \
+        jq '(.status == "green") and
+            (.indicators.pipelines.status == "green") and
+            (all(.indicators.pipelines.indicators[]; .status == "green"))' 2>/dev/null
+)
+
+[[ "$JQ_EVAL" == "true" ]] && exit 0 && exit 1