godaddy · aeimer · Jun 16, 2022
@@ -64,6 +64,7 @@ const options = {
   timeout: 1000,                    // [optional = 1000] number of milliseconds before forceful exiting
   signal,                           // [optional = 'SIGTERM'] what signal to listen for relative to shutdown
   signals,                          // [optional = []] array of signals to listen for relative to shutdown
+  useExit0,                         // [optional = false] instead of sending the received signal again without beeing catched, the process will exit(0)
   sendFailuresDuringShutdown,       // [optional = true] whether or not to send failure (503) during shutdown
   beforeShutdown,                   // [optional] called before the HTTP server starts its shutdown
   onSignal,                         // [optional] cleanup function, returning a promise (used to be onSigterm)
@@ -179,17 +180,132 @@ createTerminus(server, options);
 server.listen(PORT || 3000);
 ```
 
+### With cluster (and eg. express)
+
+If you want to use (`cluster`)[https://nodejs.org/api/cluster.html] to use more than one CPU, you need to use `terminus` per worker.
+This is heavily inspired by https://medium.com/@gaurav.lahoti/graceful-shutdown-of-node-js-workers-dd58bbff9e30.
+
+```javascript
+import * as express from 'express';
+import {createTerminus} from "@godaddy/terminus";
+import http from "http";
+import cluster from 'node:cluster';
+import http from 'node:http';
+import { cpus } from 'node:os';
+import process from 'node:process';
+
+function run() {
+  const app = express()
+  // Setup your express here
+  // Last in line, nothing else responded
+  app.use((req, res) => {
+    res.status(404).send('Sorry!');
+  });
+
+  const server = http.createServer(app)
+  const terminusOptions = { ... }
+  createTerminus(server, terminusOptions);
+  server.listen(8080, () => {
+    logger.info(`Express worker with PID: ${process.pid} started.`)
+  })
+}
+
+/**
+ * Shutdown the cluster workers properly
+ */
+async function gracefulClusterShutdown(): Promise<void> {
+  console.log("Starting graceful cluster shutdown")
+  shuttingDownServer = true
+  await shutdownWorkers('SIGTERM')
+  console.log("Successfully finished graceful cluster shutdown")
+  process.exit(0)
+}
+
+/**
+ * Shutdown all worker processes.
+ * From https://medium.com/@gaurav.lahoti/graceful-shutdown-of-node-js-workers-dd58bbff9e30
+ * @param signal Signal to send to the workers
+ */
+function shutdownWorkers (signal: NodeJS.Signals) {
+  return new Promise<void>((resolve) => {
+    if (!cluster.isMaster) { return resolve() }
+
+    const wIds = Object.keys(cluster.workers)
+    if (wIds.length == 0) { return resolve() }
+    // Filter all the valid workers
+    const workers = wIds.map(id => cluster.workers[id]).filter(v => v) as cluster.Worker[]
+    let workersAlive = 0
+    let funcRun = 0
+
+    // Count the number of alive workers and keep looping until the number is zero.
+    const fn = () => {
+      ++funcRun
+      workersAlive = 0
+      workers.forEach(worker => {
+        if (!worker.isDead()) {
+          ++workersAlive
+          if (funcRun == 1)
+            // On the first execution of the function, send the received signal to all the workers
+            // https://github.com/nodejs/node-v0.x-archive/issues/6042#issuecomment-168677045
+            worker.process.kill(signal)
+        }
+      })
+      logger.info(workersAlive + " workers alive")
+      if (workersAlive == 0) {
+        // Clear the interval when all workers are dead
+        clearInterval(interval)
+        return resolve()
+      }
+    }
+    const interval = setInterval(fn, 1000)
+  })
+}
+
+// Startup code
+
+// NodeJS is single threaded, how many CPUs/Threads do we want to use?
+const numCPUs = cpus().length;
+let shuttingDownServer = false;
+
+if (cluster.isPrimary) {
+  console.log(`Primary ${process.pid} is running`);
+
+  // Fork workers.
+  for (let i = 0; i < numCPUs; i++) {
+    cluster.fork();
+  }
+
+  cluster.on('exit', (worker, code, signal) => {
+    if (!shuttingDownServer) {
+      logger.warn(`Worker ${worker.process.pid} died. Forking a new one...`);
+      cluster.fork();
+    }
+  });
+
+  // Graceful shutdown of all workers
+  process.on('SIGTERM', gracefulClusterShutdown)
+} else {
+  run()
+  console.log(`Worker ${process.pid} started`);
+}
+```
+
 ## How to set Terminus up with Kubernetes?
 
 When Kubernetes or a user deletes a Pod, Kubernetes will notify it and wait for `gracePeriod` seconds before killing it.
 
-During that time window (30 seconds by default), the Pod is in the `terminating` state and will be removed from any Services by a controller. The Pod itself needs to catch the `SIGTERM` signal and start failing any readiness probes.
+During that time window (30 seconds by default), the Pod is in the `terminating` state and will be removed from any Services by a controller.
+The Pod itself needs to catch the `SIGTERM` signal and start failing any readiness probes.
 
 > If the ingress controller you use route via the Service, it is not an issue for your case. At the time of this writing, we use the nginx ingress controller which routes traffic directly to the Pods.
 
 During this time, it is possible that load-balancers (like the nginx ingress controller) don't remove the Pods "in time", and when the Pod dies, it kills live connections.
 
-To make sure you don't lose any connections, we recommend delaying the shutdown with the number of milliseconds that's defined by the readiness probe in your deployment configuration. To help with this, terminus exposes an option called `beforeShutdown` that takes any Promise-returning function.
+To make sure you don't lose any connections, we recommend delaying the shutdown with the number of milliseconds that's defined by the readiness probe in your deployment configuration.
+To help with this, terminus exposes an option called `beforeShutdown` that takes any Promise-returning function.
+
+Also it makes sense to use the `useExit0 = true` option to signal Kubernetes that the container exited gracefully.
+Otherwise APM's will send you alerts, in some cases.
 
 ```javascript
 function beforeShutdown () {
@@ -201,7 +317,8 @@ function beforeShutdown () {
   })
 }
 createTerminus(server, {
-  beforeShutdown
+  beforeShutdown,
+  useExit0: true
 })
 ```
 

@@ -0,0 +1,13 @@
+'use strict'
+const http = require('http')
+const server = http.createServer((req, res) => res.end('hello'))
+
+const { createTerminus } = require('../../')
+
+createTerminus(server, {
+  useExit0: true
+})
+
+server.listen(8000, () => {
+  process.kill(process.pid, 'SIGTERM')
+})
@@ -126,7 +126,7 @@ function decorateWithHealthCheck (server, state, options) {
 }
 
 function decorateWithSignalHandler (server, state, options) {
-  const { signals, onSignal, beforeShutdown, onShutdown, timeout, logger } = options
+  const { signals, useExit0, onSignal, beforeShutdown, onShutdown, timeout, logger } = options
 
   stoppable(server, timeout)
 
@@ -140,8 +140,14 @@ function decorateWithSignalHandler (server, state, options) {
         await asyncServerStop()
         await onSignal()
         await onShutdown()
-        signals.forEach(sig => process.removeListener(sig, cleanup))
-        process.kill(process.pid, signal)
+        if (useExit0) {
+          // Exit process
+          process.exit(0)
+        } else {
+          // Resend recieved signal but remove traps beforehand
+          signals.forEach(sig => process.removeListener(sig, cleanup))
+          process.kill(process.pid, signal)
+        }
       } catch (error) {
         logger('error happened during shutdown', error)
         process.exit(1)
@@ -168,6 +174,7 @@ function terminus (server, options = {}) {
   const {
     signal = 'SIGTERM',
     signals = [],
+    useExit0  = false,
     timeout = 1000,
     healthChecks = {},
     sendFailuresDuringShutdown = true,
@@ -201,6 +208,7 @@ function terminus (server, options = {}) {
   if (!signals.includes(signal)) signals.push(signal)
   decorateWithSignalHandler(server, state, {
     signals,
+    useExit0,
     onSignal,
     beforeShutdown,
     onShutdown,

@@ -658,25 +658,39 @@ describe('Terminus', () => {
   })
 
   it('accepts custom headers', async () => {
-
-  createTerminus(server, {
-    healthChecks: {
-      '/health': () => {
-        return Promise.resolve()
-      }
-    },
-    headers: {
-      "Access-Control-Allow-Origin": "*",
-      "Access-Control-Allow-Methods": "OPTIONS, POST, GET",
-    },
+
+    createTerminus(server, {
+      healthChecks: {
+        '/health': () => {
+          return Promise.resolve()
+        }
+      },
+      headers: {
+        "Access-Control-Allow-Origin": "*",
+        "Access-Control-Allow-Methods": "OPTIONS, POST, GET",
+      },
+    })
+    server.listen(8000)
+
+    const res = await fetch('http://localhost:8000/health')
+    expect(res.status).to.eql(200)
+    expect(res.headers.has('Access-Control-Allow-Methods')).to.eql(true)
+    expect(res.headers.get('Access-Control-Allow-Methods')).to.eql('OPTIONS, POST, GET')
+    expect(res.headers.has('Access-Control-Allow-Origin')).to.eql(true)
+    expect(res.headers.get('Access-Control-Allow-Origin')).to.eql('*')
   })
-  server.listen(8000)
-
-  const res = await fetch('http://localhost:8000/health')
-  expect(res.status).to.eql(200)
-  expect(res.headers.has('Access-Control-Allow-Methods')).to.eql(true)
-  expect(res.headers.get('Access-Control-Allow-Methods')).to.eql('OPTIONS, POST, GET')
-  expect(res.headers.has('Access-Control-Allow-Origin')).to.eql(true)
-  expect(res.headers.get('Access-Control-Allow-Origin')).to.eql('*')
+
+  it('allows to exit(0) with useExit0', async () => {
+    let responseAssertionsComplete = false
+
+    // We're only truly finished when the response has been analyzed and the forked http process has exited,
+    // freeing up port 8000 for future tests
+    execFile('node', ['lib/standalone-tests/terminus.useExit0.js'], (error) => {
+      fail(`exit code was non zero ${error.signal}`)
+    })
+    .then(res => {
+      // FIXME: Check if exit code was 0
+      // expect()
+    })
   })
-})
+})