Status handler infrastructure and API impl (#40)

* Status hanlder infrastructure

Implements status handling infrastructure. This allows multiple polling status handlers to exist for different dependencies. The status and detail for these are then combined to form an overall service status which is served by the health endpoint

* Add timeout for service status

* Switch looping structure to a generator

* Rename accumulator to make it clear it\'s a status value

* Small refactor and handled graceful close

* Add API healthcheck

app/env.js

@@ -28,6 +28,8 @@ const vars = envalid.cleanEnv(process.env, {
   API_VERSION: envalid.str({ default: version }),
   API_MAJOR_VERSION: envalid.str({ default: 'v3' }),
   FILE_UPLOAD_MAX_SIZE: envalid.num({ default: 200 * 1024 * 1024 }),
+  SUBSTRATE_STATUS_POLL_PERIOD_MS: envalid.num({ default: 30 * 1000 }),
+  SUBSTRATE_STATUS_TIMEOUT_MS: envalid.num({ default: 2 * 1000 }),
 })
 
 module.exports = {

app/server.js

@@ -11,19 +11,29 @@ const { PORT, API_VERSION, API_MAJOR_VERSION } = require('./env')
 const logger = require('./logger')
 const apiDoc = require('./api-v3/api-doc')
 const apiService = require('./api-v3/services/apiService')
+const { startStatusHandlers } = require('./serviceStatus')
+const { serviceState } = require('./util/statusPoll')
 const { verifyJwks } = require('./util/appUtil')
 
 async function createHttpServer() {
   const requestLogger = pinoHttp({ logger })
   const app = express()
+  const statusHandler = await startStatusHandlers()
 
   app.use(cors())
   app.use(compression())
   app.use(bodyParser.json())
 
+  const serviceStatusStrings = {
+    [serviceState.UP]: 'ok',
+    [serviceState.DOWN]: 'down',
+    [serviceState.ERROR]: 'error',
+  }
   app.get('/health', async (req, res) => {
-    res.status(200).send({ version: API_VERSION, status: 'ok' })
-    return
+    const status = statusHandler.status
+    const detail = statusHandler.detail
+    const code = status === serviceState.UP ? 200 : 503
+    res.status(code).send({ version: API_VERSION, status: serviceStatusStrings[status] || 'error', detail })
   })
 
   app.use((req, res, next) => {
@@ -84,20 +94,52 @@ async function createHttpServer() {
     }
   })
 
-  return app
+  return { app, statusHandler }
 }
 
 async function startServer() {
-  const app = await createHttpServer()
+  try {
+    const { app, statusHandler } = await createHttpServer()
 
-  app.listen(PORT, (err) => {
-    if (err) {
-      logger.error('Error  starting app:', err)
-      throw err
-    } else {
-      logger.info(`Server is listening on port ${PORT}`)
+    const server = await new Promise((resolve, reject) => {
+      let resolved = false
+      const server = app.listen(PORT, (err) => {
+        if (err) {
+          if (!resolved) {
+            resolved = true
+            reject(err)
+          }
+        }
+        logger.info(`Listening on port ${PORT} `)
+        if (!resolved) {
+          resolved = true
+          resolve(server)
+        }
+      })
+      server.on('error', (err) => {
+        if (!resolved) {
+          resolved = true
+          reject(err)
+        }
+      })
+    })
+
+    const closeHandler = (exitCode) => async () => {
+      server.close(async () => {
+        await statusHandler.close()
+        process.exit(exitCode)
+      })
     }
-  })
+
+    const setupGracefulExit = ({ sigName, exitCode }) => {
+      process.on(sigName, closeHandler(exitCode))
+    }
+    setupGracefulExit({ sigName: 'SIGINT', server, exitCode: 0 })
+    setupGracefulExit({ sigName: 'SIGTERM', server, exitCode: 143 })
+  } catch (err) {
+    logger.fatal('Fatal error during initialisation: %s', err.message)
+    process.exit(1)
+  }
 }
 
 module.exports = { startServer, createHttpServer }

app/serviceStatus/apiStatus.js

@@ -0,0 +1,32 @@
+const { startStatusHandler, serviceState } = require('../util/statusPoll')
+const { substrateApi } = require('../util/substrateApi')
+const { SUBSTRATE_STATUS_POLL_PERIOD_MS, SUBSTRATE_STATUS_TIMEOUT_MS } = require('../env')
+
+const getStatus = async () => {
+  await substrateApi.isReady
+  const [chain, runtime] = await Promise.all([substrateApi.runtimeChain, substrateApi.runtimeVersion])
+  return {
+    status: serviceState.UP,
+    detail: {
+      chain,
+      runtime: {
+        name: runtime.specName,
+        versions: {
+          spec: runtime.specVersion.toNumber(),
+          impl: runtime.implVersion.toNumber(),
+          authoring: runtime.authoringVersion.toNumber(),
+          transaction: runtime.transactionVersion.toNumber(),
+        },
+      },
+    },
+  }
+}
+
+const startApiStatus = () =>
+  startStatusHandler({
+    getStatus,
+    pollingPeriodMs: SUBSTRATE_STATUS_POLL_PERIOD_MS,
+    serviceTimeoutMs: SUBSTRATE_STATUS_TIMEOUT_MS,
+  })
+
+module.exports = startApiStatus

app/serviceStatus/index.js

@@ -0,0 +1,13 @@
+const startApiStatus = require('./apiStatus')
+const { buildCombinedHandler } = require('../util/statusPoll')
+
+const startStatusHandlers = async () => {
+  const handlers = new Map()
+  handlers.set('api', await startApiStatus())
+
+  return buildCombinedHandler(handlers)
+}
+
+module.exports = {
+  startStatusHandlers,
+}

app/util/statusPoll.js

@@ -0,0 +1,97 @@
+const serviceState = {
+  UP: Symbol('status-up'),
+  DOWN: Symbol('status-down'),
+  ERROR: Symbol('status-error'),
+}
+const stateSymbols = new Set(Object.values(serviceState))
+
+const delay = (delayMs, result) => new Promise((resolve) => setTimeout(resolve, delayMs, result))
+
+const mkStatusGenerator = async function* ({ getStatus, serviceTimeoutMs }) {
+  while (true) {
+    try {
+      const newStatus = await Promise.race([
+        getStatus(),
+        delay(serviceTimeoutMs, { status: serviceState.ERROR, detail: null }),
+      ])
+
+      if (stateSymbols.has(newStatus.status)) {
+        yield {
+          status: newStatus.status,
+          detail: newStatus.detail === undefined ? null : newStatus.detail,
+        }
+        continue
+      }
+      throw new Error('Status is not a valid value')
+    } catch (err) {
+      yield {
+        status: serviceState.ERROR,
+        detail: null,
+      }
+    }
+  }
+}
+
+const startStatusHandler = async ({ pollingPeriodMs, serviceTimeoutMs, getStatus }) => {
+  let status = null
+  const statusGenerator = mkStatusGenerator({ getStatus, serviceTimeoutMs })
+  status = (await statusGenerator.next()).value
+
+  const statusLoop = async function () {
+    await delay(pollingPeriodMs)
+    for await (const newStatus of statusGenerator) {
+      status = newStatus
+      await delay(pollingPeriodMs)
+    }
+  }
+  statusLoop()
+
+  return {
+    get status() {
+      return status.status
+    },
+    get detail() {
+      return status.detail
+    },
+    close: () => {
+      statusGenerator.return()
+    },
+  }
+}
+
+const buildCombinedHandler = async (handlerMap) => {
+  const getStatus = () =>
+    [...handlerMap].reduce((accStatus, [, h]) => {
+      const handlerStatus = h.status
+      if (accStatus === serviceState.UP) {
+        return handlerStatus
+      }
+      if (accStatus === serviceState.DOWN) {
+        return accStatus
+      }
+      if (handlerStatus === serviceState.DOWN) {
+        return handlerStatus
+      }
+      return accStatus
+    }, serviceState.UP)
+
+  return {
+    get status() {
+      return getStatus()
+    },
+    get detail() {
+      return Object.fromEntries([...handlerMap].map(([name, { detail }]) => [name, detail]))
+    },
+    close: () => {
+      for (const handler of handlerMap.values()) {
+        handler.close()
+      }
+    },
+  }
+}
+
+module.exports = {
+  serviceState,
+  startStatusHandler,
+  buildCombinedHandler,
+}

docker-compose.yaml

@@ -7,7 +7,7 @@ services:
       - 8080:8080
       - 5001:5001
   dscp-node:
-    image: ghcr.io/digicatapult/dscp-node:v3.0.0
+    image: ghcr.io/digicatapult/dscp-node:latest
     command:
       --base-path /data/
       --dev

helm/dscp-api/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 name: dscp-api
-appVersion: '4.0.1'
+appVersion: '4.0.2'
 description: A Helm chart for dscp-api
-version: '4.0.1'
+version: '4.0.2'
 type: application
 dependencies:
   - name: dscp-node

helm/dscp-api/values.yaml

@@ -24,7 +24,7 @@ replicaCount: 1
 image:
   repository: ghcr.io/digicatapult/dscp-api
   pullPolicy: IfNotPresent
-  tag: 'v4.0.1'
+  tag: 'v4.0.2'
 
 dscpNode:
   enabled: false