Merge pull request #52 from open-runtimes/feat-robust-healthcheck

feat: robust healthcheck
open-runtimes · Dec 20, 2024 · 18ea449 · 18ea449
2 parents a63ef4d + 7c621c5
commit 18ea449
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 51 deletions.
diff --git a/app/http.php b/app/http.php
@@ -215,46 +215,58 @@ function createState(string $dsn): State
 
     $healthy = true;
     foreach ($health->run()->getNodes() as $node) {
-        $hostname = $node->getHostname();
-        $executor = $executors[$hostname] ?? [];
-        $newStatus = $node->isOnline() ? 'online' : 'offline';
-
-        if ($firstCheck || Http::isDevelopment() || $executor['status'] !== $newStatus) {
-            if ($newStatus === 'online') {
-                Console::info('Executor "' . $hostname . '" went online');
-            } else {
-                $message = $node->getState()['message'] ?? 'Unexpected error.';
-                $error = new Exception('Executor "' . $hostname . '" went offline: ' . $message, 500);
-                logError($error, "healthCheckError", $logger, null);
+        try {
+            $hostname = $node->getHostname();
+            $executor = $executors[$hostname] ?? [];
+            $newStatus = $node->isOnline() ? 'online' : 'offline';
+
+            if ($firstCheck || Http::isDevelopment() || $executor['status'] !== $newStatus) {
+                if ($newStatus === 'online') {
+                    Console::info('Executor "' . $hostname . '" went online');
+                } else {
+                    $message = $node->getState()['message'] ?? 'Unexpected error.';
+                    $error = new Exception('Executor "' . $hostname . '" went offline: ' . $message, 500);
+                    logError($error, "healthCheckError", $logger, null);
+                }
             }
-        }
 
-        if (!$node->isOnline()) {
-            $healthy = false;
-        }
+            if (!$node->isOnline()) {
+                $healthy = false;
+            }
 
-        $state->save(
-            resource: RESOURCE_EXECUTORS,
-            name: $hostname,
-            status: $node->isOnline() ? 'online' : 'offline',
-            usage: $node->getState()['usage'] ?? 0
-        );
+            $state->save(
+                resource: RESOURCE_EXECUTORS,
+                name: $hostname,
+                status: $node->isOnline() ? 'online' : 'offline',
+                usage: $node->getState()['usage'] ?? 0
+            );
 
-        $runtimes = [];
+            $runtimes = [];
 
-        Console::log('Executor "' . $hostname . '" healthcheck returned ' . \count($node->getState()['runtimes'] ?? []) . ' runtimes');
-        foreach ($node->getState()['runtimes'] ?? [] as $runtimeId => $runtime) {
-            if (!\is_string($runtimeId) || !\is_array($runtime)) {
-                Console::warning('Invalid runtime data for ' . $hostname . ' runtime ' . $runtimeId);
-                continue;
-            }
+            Console::log('Executor "' . $hostname . '" healthcheck returned ' . \count($node->getState()['runtimes'] ?? []) . ' runtimes');
+            foreach ($node->getState()['runtimes'] ?? [] as $runtimeId => $runtime) {
+                if (!\is_string($runtimeId) || !\is_array($runtime)) {
+                    Console::warning('Invalid runtime data for ' . $hostname . ' runtime ' . $runtimeId);
+                    continue;
+                }
 
-            $runtimes[$runtimeId] = [
-                'status' => $runtime['status'] ?? 'offline',
-                'usage' => $runtime['usage'] ?? 100,
-            ];
+                $runtimes[$runtimeId] = [
+                    'status' => $runtime['status'] ?? 'offline',
+                    'usage' => $runtime['usage'] ?? 100,
+                ];
+            }
+            $state->saveAll(RESOURCE_RUNTIMES . $hostname, $runtimes);
+        } catch (\Throwable $th) {
+            try {
+                $healthy = false;
+                Console::warning('Health check failed for ' . $node->getHostname() . ': ' . $th->getMessage() . ' - removing from state');
+
+                $state->save(RESOURCE_EXECUTORS, $node->getHostname(), 'offline', 100);
+                $state->saveAll(RESOURCE_RUNTIMES . $node->getHostname(), []);
+            } catch (\Throwable $th) {
+                Console::warning('Failed to remove executor from state: ' . $th->getMessage());
+            }
         }
-        $state->saveAll(RESOURCE_RUNTIMES . $hostname, $runtimes);
     }
 
     if (Http::getEnv('OPR_PROXY_HEALTHCHECK_URL', '') !== '' && $healthy) {

diff --git a/composer.json b/composer.json
@@ -27,7 +27,7 @@
         "utopia-php/framework": "0.34.*",
         "utopia-php/registry": "0.6.*",
         "utopia-php/cli": "0.13.*",
-        "utopia-php/logger": "0.6.*",
+        "utopia-php/logger": "0.7.*",
         "utopia-php/balancer": "0.4.*",
         "utopia-php/fetch": "^0.1.0",
         "utopia-php/dsn": "0.2.1"

diff --git a/composer.lock b/composer.lock