fix(core): Handle SIGTERM for graceful shutdowns

libs/infra-express-server/src/lib/infra-express-server.ts

@@ -1,3 +1,4 @@
+import { setupShutdownHooks } from '@island.is/node-utils'
 import express, { Router, Request, Response, NextFunction } from 'express'
 import { collectDefaultMetrics, Histogram } from 'prom-client'
 import { metricsApp } from './metrics-publisher'
@@ -41,11 +42,11 @@ export const runServer = ({
     return next()
   })
 
-  app.get('/liveness', function liveness(req, res) {
+  app.get('/liveness', function liveness(_req, res) {
     res.json({ ok: true })
   })
 
-  app.get('/version', function versionOfCode(req, res) {
+  app.get('/version', function versionOfCode(_req, res) {
     res.json({ version: process.env.REVISION })
   })
 
@@ -56,7 +57,7 @@ export const runServer = ({
 
   // security middleware
   // we should implemente something along the lines of this https://auth0.com/docs/quickstart/backend/nodejs/01-authorization
-  app.use((req, res, next) => {
+  app.use((_req, _res, next) => {
     // we need to secure all routes by default. OAuth?
     next()
   })
@@ -66,9 +67,9 @@ export const runServer = ({
 
   app.use(function errorHandler(
     err: any, // eslint-disable-line  @typescript-eslint/no-explicit-any
-    req: Request,
+    _req: Request,
     res: Response,
-    next: NextFunction, // eslint-disable-line
+    _next: NextFunction,
   ) {
     logger.error(`Status code: ${err.status}, msg: ${err.message}`)
     res.status(err.status || 500)
@@ -84,6 +85,7 @@ export const runServer = ({
   const server = app.listen(servicePort, () => {
     logger.info(`Listening on port ${servicePort}`)
   })
+  setupShutdownHooks(server)
   server.on('error', logger.error)
   return server
 }

libs/infra-nest-server/src/lib/bootstrap.ts

@@ -46,6 +46,9 @@ export const createApp = async ({
     },
   )
 
+  // Handle signals like SIGINT, SIGTERM and SIGHUP gracefully.
+  app.enableShutdownHooks()
+
   if (enableVersioning) {
     app.enableVersioning()
   }

libs/infra-next-server/src/lib/bootstrap.ts

@@ -1,4 +1,5 @@
 import '@island.is/infra-tracing'
+import { setupShutdownHooks } from '@island.is/node-utils'
 
 import createExpressApp, { Express } from 'express'
 
@@ -43,33 +44,22 @@ type BootstrapOptions = {
 const startServer = (app: Express, port = 4200) => {
   const nextPort = parseInt(process.env.PORT || '') || port
   const metricsPort = nextPort + 1
-  app.listen(nextPort, () => {
+  const server = app.listen(nextPort, () => {
     logger.info(
       `Next custom server listening at http://localhost:${nextPort}`,
       {
         context: 'Bootstrap',
       },
     )
   })
+  setupShutdownHooks(server)
   startMetricServer(metricsPort)
 }
 
-const setupExitHook = () => {
-  // Make sure the server doesn't hang after parent process disconnects, eg when
-  // e2e tests are finished.
-  if (process.env.NX_INVOKED_BY_RUNNER === 'true') {
-    process.on('disconnect', () => {
-      process.exit(0)
-    })
-  }
-}
-
 export const bootstrap = async (options: BootstrapOptions) => {
   const dev = process.env.NODE_ENV !== 'production'
   monkeyPatchServerLogging()
 
-  setupExitHook()
-
   const expressApp = createExpressApp()
 
   await setupProxy(expressApp, options.proxyConfig, dev)

libs/node/utils/.eslintrc.json

@@ -0,0 +1,18 @@
+{
+  "extends": ["../../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    }
+  ]
+}

libs/node/utils/README.md

@@ -0,0 +1,24 @@
+# Node Utils
+
+## `setupShutdownHooks`
+
+The `setupShutdownHooks` function configures a server to properly handle shutdown signals, ensuring a graceful termination process. This is particularly useful for preventing the server from hanging when the parent process disconnects, such as after end-to-end tests complete.
+
+### Parameters
+
+- `server` (Server): An instance of an HTTP server that needs to be shut down gracefully upon receiving termination signals.
+- `onShutdown` (Function): Optional callback function that will be executed before the server shuts down. Can be used to perform and wait for other cleanup tasks before the server shuts down.
+
+### Behavior
+
+- Listens for system signals indicating process termination: `SIGHUP`, `SIGINT`, and `SIGTERM`.
+- When executed by an NX runner (detected by `process.env.NX_INVOKED_BY_RUNNER === 'true'`), it also listens for the `disconnect` event.
+- On receiving a termination signal:
+- Logs the signal received.
+- Closes the server using `server.close()`, awaiting closure before continuing.
+- Executes the `onShutdown` callback if provided.
+- Logs the successful server shutdown and exits the process with a status code of `0` upon successful shutdown or `1` if an error occurs.
+
+## Running unit tests
+
+Run `nx test node-utils` to execute the unit tests via [Jest](https://jestjs.io).

libs/node/utils/jest.config.ts

@@ -0,0 +1,11 @@
+/* eslint-disable */
+export default {
+  displayName: 'node-utils',
+  preset: '../../../jest.preset.js',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
+  },
+  moduleFileExtensions: ['ts', 'js', 'html'],
+  coverageDirectory: '../../../coverage/libs/node/utils',
+}

libs/node/utils/project.json

@@ -0,0 +1,19 @@
+{
+  "name": "node-utils",
+  "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "libs/node/utils/src",
+  "projectType": "library",
+  "tags": [],
+  "targets": {
+    "lint": {
+      "executor": "@nx/eslint:lint"
+    },
+    "test": {
+      "executor": "@nx/jest:jest",
+      "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
+      "options": {
+        "jestConfig": "libs/node/utils/jest.config.ts"
+      }
+    }
+  }
+}

libs/node/utils/src/index.ts

@@ -0,0 +1 @@
+export * from './lib/setupShutdownHooks'

libs/node/utils/src/lib/setupShutdownHooks.spec.ts

@@ -0,0 +1,148 @@
+import { setupShutdownHooks } from './setupShutdownHooks'
+import { logger } from '@island.is/logging'
+import { Server } from 'http'
+
+// Mock the logger
+jest.mock('@island.is/logging', () => ({
+  logger: {
+    info: jest.fn(),
+    error: jest.fn(),
+  },
+}))
+
+describe('setupShutdownHooks', () => {
+  let server: Server
+  let processExitSpy: jest.SpyInstance
+  let processOnSpy: jest.SpyInstance
+
+  beforeEach(() => {
+    // Create a mock server
+    server = {
+      close: jest.fn((callback) => callback()),
+    } as unknown as Server
+
+    // Spy on process.exit and process.on
+    processExitSpy = jest
+      .spyOn(process, 'exit')
+      .mockImplementation(() => undefined as never)
+    processOnSpy = jest.spyOn(process, 'on')
+
+    // Clear all mocks before each test
+    jest.clearAllMocks()
+  })
+
+  afterEach(() => {
+    // Restore all mocks after each test
+    jest.restoreAllMocks()
+  })
+
+  it('should set up event listeners for termination signals', () => {
+    // Act
+    setupShutdownHooks(server)
+
+    // Assert
+    expect(processOnSpy).toHaveBeenCalledWith('SIGHUP', expect.any(Function))
+    expect(processOnSpy).toHaveBeenCalledWith('SIGINT', expect.any(Function))
+    expect(processOnSpy).toHaveBeenCalledWith('SIGTERM', expect.any(Function))
+  })
+
+  it('should add disconnect event when NX_INVOKED_BY_RUNNER is true', () => {
+    // Arrange
+    process.env.NX_INVOKED_BY_RUNNER = 'true'
+
+    // Act
+    setupShutdownHooks(server)
+
+    // Assert
+    expect(processOnSpy).toHaveBeenCalledWith(
+      'disconnect',
+      expect.any(Function),
+    )
+
+    // Clean up
+    process.env.NX_INVOKED_BY_RUNNER = undefined
+  })
+
+  it('should handle shutdown gracefully', async () => {
+    // Arrange
+    setupShutdownHooks(server)
+    const handler = processOnSpy.mock.calls.find(
+      ([event]) => event === 'SIGTERM',
+    )[1]
+
+    // Act
+    await handler()
+
+    // Assert
+    expect(logger.info).toHaveBeenCalledWith(
+      'Received SIGTERM, shutting down...',
+    )
+    expect(server.close).toHaveBeenCalled()
+    expect(logger.info).toHaveBeenCalledWith('Server closed successfully')
+    expect(processExitSpy).toHaveBeenCalledWith(0)
+  })
+
+  it('should handle server close errors', async () => {
+    // Arrange
+    const errorServer = {
+      close: jest.fn(() => {
+        throw new Error('Server close failed')
+      }),
+    } as unknown as Server
+    setupShutdownHooks(errorServer)
+    const [[, handler]] = processOnSpy.mock.calls
+
+    // Act
+    await handler()
+
+    // Assert
+    expect(logger.error).toHaveBeenCalledWith(
+      'Error during shutdown:',
+      expect.any(Error),
+    )
+    expect(processExitSpy).toHaveBeenCalledWith(1)
+  })
+
+  it('should execute onShutdown callback during shutdown', async () => {
+    // Arrange
+    const onShutdownMock = jest.fn()
+    setupShutdownHooks(server, onShutdownMock)
+    const handler = processOnSpy.mock.calls.find(
+      ([event]) => event === 'SIGTERM',
+    )[1]
+
+    // Act
+    await handler()
+
+    // Assert
+    expect(onShutdownMock).toHaveBeenCalled()
+    expect(logger.info).toHaveBeenCalledWith(
+      'Received SIGTERM, shutting down...',
+    )
+    expect(server.close).toHaveBeenCalled()
+    expect(logger.info).toHaveBeenCalledWith('Server closed successfully')
+    expect(processExitSpy).toHaveBeenCalledWith(0)
+  })
+
+  it('should handle onShutdown rejection', async () => {
+    // Arrange
+    const onShutdownMock = jest
+      .fn()
+      .mockRejectedValue(new Error('Shutdown hook failed'))
+    setupShutdownHooks(server, onShutdownMock)
+    const handler = processOnSpy.mock.calls.find(
+      ([event]) => event === 'SIGTERM',
+    )[1]
+
+    // Act
+    await handler()
+
+    // Assert
+    expect(onShutdownMock).toHaveBeenCalled()
+    expect(logger.error).toHaveBeenCalledWith(
+      'Error during shutdown:',
+      expect.any(Error),
+    )
+    expect(processExitSpy).toHaveBeenCalledWith(1)
+  })
+})

libs/node/utils/src/lib/setupShutdownHooks.ts

@@ -0,0 +1,44 @@
+import { logger } from '@island.is/logging'
+import { Server } from 'http'
+
+export const setupShutdownHooks = (
+  server: Server,
+  onShutdown?: () => Promise<void>,
+) => {
+  let isShuttingDown = false
+  const terminationEvents: Array<NodeJS.Signals | 'disconnect'> = [
+    'SIGHUP',
+    'SIGINT',
+    'SIGTERM',
+  ]
+
+  // Make sure the server doesn't hang after parent process disconnects, eg when
+  // e2e tests are finished.
+  if (process.env.NX_INVOKED_BY_RUNNER === 'true') {
+    terminationEvents.push('disconnect')
+  }
+
+  const shutdown = async (signal: string) => {
+    if (isShuttingDown) {
+      return
+    }
+    isShuttingDown = true
+
+    try {
+      logger.info(`Received ${signal}, shutting down...`)
+      await new Promise((resolve) => server.close(resolve))
+      if (onShutdown) {
+        await onShutdown()
+      }
+      logger.info('Server closed successfully')
+      process.exit(0)
+    } catch (error) {
+      logger.error('Error during shutdown:', error)
+      process.exit(1)
+    }
+  }
+
+  for (const signal of terminationEvents) {
+    process.on(signal, () => shutdown(signal))
+  }
+}

libs/node/utils/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "module": "commonjs"
+  },
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}