Budi 8841 implement lame ducking on app and worker pods

charts/budibase/templates/app-service-deployment.yaml

@@ -30,6 +30,7 @@ spec:
 {{- toYaml .Values.services.apps.templateLabels | indent 8 -}}
 {{ end }}
     spec:
+      terminationGracePeriodSeconds: 60
       containers:
       - env:
         - name: BUDIBASE_ENVIRONMENT

charts/budibase/templates/worker-service-deployment.yaml

@@ -30,6 +30,7 @@ spec:
 {{- toYaml .Values.services.worker.templateLabels | indent 8 -}}
 {{ end }}
     spec:
+      terminationGracePeriodSeconds: 60
       containers:
       - env:
         - name: BUDIBASE_ENVIRONMENT

packages/server/package.json

@@ -81,6 +81,7 @@
     "form-data": "4.0.0",
     "global-agent": "3.0.0",
     "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5",
+    "http-graceful-shutdown": "^3.1.13",
     "ioredis": "5.3.2",
     "isolated-vm": "^4.7.2",
     "jimp": "0.22.12",

packages/server/src/koa.ts

@@ -13,8 +13,9 @@ import {
   timers,
   env as coreEnv,
 } from "@budibase/backend-core"
-import destroyable from "server-destroy"
 import { userAgent } from "koa-useragent"
+import destroyable from "server-destroy"
+import gracefulShutdown from "http-graceful-shutdown"
 
 export default function createKoaApp() {
   const app = new Koa()
@@ -50,35 +51,27 @@ export default function createKoaApp() {
 
   const server = http.createServer(app.callback())
   destroyable(server)
-
-  let shuttingDown = false,
-    errCode = 0
-
-  server.on("close", async () => {
-    // already in process
-    if (shuttingDown) {
-      return
-    }
-    shuttingDown = true
-    console.log("Server Closed")
-    timers.cleanup()
-    await automations.shutdown()
-    await redis.shutdown()
-    events.shutdown()
-    await Thread.shutdown()
-    api.shutdown()
-    if (!env.isTest()) {
-      process.exit(errCode)
-    }
-  })
+  // let shuttingDown = false
+  let errCode = 0
 
   const listener = server.listen(env.PORT || 0)
 
-  const shutdown = () => {
-    server.close()
-    // @ts-ignore
-    server.destroy()
-  }
+  gracefulShutdown(server, {
+    onShutdown: async () => {
+      console.log("Server is shutting down gracefully...")
+      timers.cleanup()
+      await automations.shutdown()
+      await redis.shutdown()
+      events.shutdown()
+      await Thread.shutdown()
+      api.shutdown()
+    },
+    finally: () => {
+      if (!env.isTest()) {
+        process.exit(errCode)
+      }
+    },
+  })
 
   process.on("uncaughtException", err => {
     // @ts-ignore
@@ -88,15 +81,8 @@ export default function createKoaApp() {
     }
     errCode = -1
     logging.logAlert("Uncaught exception.", err)
-    shutdown()
-  })
-
-  process.on("SIGTERM", () => {
-    shutdown()
-  })
-
-  process.on("SIGINT", () => {
-    shutdown()
+    // Trigger graceful shutdown
+    process.kill(process.pid, "SIGTERM")
   })
 
   return { app, server: listener }

packages/worker/src/index.ts

@@ -30,17 +30,18 @@ const koaSession = require("koa-session")
 
 import { userAgent } from "koa-useragent"
 
-import destroyable from "server-destroy"
 import { initPro } from "./initPro"
 import { handleScimBody } from "./middleware/handleScimBody"
+import destroyable from "server-destroy"
+import gracefulShutdown from "http-graceful-shutdown"
 
 if (coreEnv.ENABLE_SSO_MAINTENANCE_MODE) {
   console.warn(
     "Warning: ENABLE_SSO_MAINTENANCE_MODE is set. It is recommended this flag is disabled if maintenance is not in progress"
   )
 }
 
-// this will setup http and https proxies form env variables
+// this will setup http and https proxies from env variables
 bootstrap()
 
 const app: Application = new Application()
@@ -71,28 +72,9 @@ app.use(api.routes())
 const server = http.createServer(app.callback())
 destroyable(server)
 
-let shuttingDown = false,
-  errCode = 0
-server.on("close", async () => {
-  if (shuttingDown) {
-    return
-  }
-  shuttingDown = true
-  console.log("Server Closed")
-  timers.cleanup()
-  events.shutdown()
-  await redis.clients.shutdown()
-  await queue.shutdown()
-  if (!env.isTest()) {
-    process.exit(errCode)
-  }
-})
-
-const shutdown = () => {
-  server.close()
-  server.destroy()
-}
+let errCode = 0
 
+// Start the server
 export default server.listen(parseInt(env.PORT || "4002"), async () => {
   let startupLog = `Worker running on ${JSON.stringify(server.address())}`
   if (env.BUDIBASE_ENVIRONMENT) {
@@ -108,16 +90,26 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
   await events.processors.init(proSdk.auditLogs.write)
 })
 
+gracefulShutdown(server, {
+  signals: "SIGINT SIGTERM",
+  timeout: 30000, // optional: timeout in ms for the shutdown process to finish
+  onShutdown: async () => {
+    console.log("Server is shutting down gracefully...")
+    timers.cleanup()
+    events.shutdown()
+    await redis.clients.shutdown()
+    await queue.shutdown()
+  },
+  finally: () => {
+    if (!env.isTest()) {
+      process.exit(errCode)
+    }
+  },
+})
+
 process.on("uncaughtException", err => {
   errCode = -1
   logging.logAlert("Uncaught exception.", err)
-  shutdown()
-})
-
-process.on("SIGTERM", () => {
-  shutdown()
-})
-
-process.on("SIGINT", () => {
-  shutdown()
+  // Trigger graceful shutdown
+  process.kill(process.pid, "SIGTERM")
 })