healthcheck improvements

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:17.7.1-slim
+FROM node:17-slim
 WORKDIR /opt/app/
 COPY package.json ./
 RUN npm install

app.js

@@ -11,7 +11,7 @@ assert.ok(process.env.JAMBONES_NETWORK_CIDR || process.env.K8S, 'missing JAMBONE
 const Srf = require('drachtio-srf');
 const srf = new Srf('sbc-outbound');
 const CIDRMatcher = require('cidr-matcher');
-const {pingMsTeamsGateways, equalsIgnoreOrder} = require('./lib/utils');
+const {equalsIgnoreOrder, pingMsTeamsGateways, createHealthCheckApp, systemHealth} = require('./lib/utils');
 const opts = Object.assign({
   timestamp: () => {return `, "time": "${new Date().toISOString()}"`;}
 }, {level: process.env.JAMBONES_LOGLEVEL || 'info'});
@@ -32,6 +32,7 @@ const CallSession = require('./lib/call-session');
 const setNameRtp = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:active-rtp`;
 const rtpServers = [];
 const {
+  ping,
   performLcr,
   lookupAllTeamsFQDNs,
   lookupAccountBySipRealm,
@@ -47,6 +48,7 @@ const {
   connectionLimit: process.env.JAMBONES_MYSQL_CONNECTION_LIMIT || 10
 }, logger);
 const {
+  client: redisClient,
   createHash,
   retrieveHash,
   incrKey,
@@ -68,6 +70,7 @@ srf.locals = {...srf.locals,
   queryCdrs,
   activeCallIds,
   dbHelpers: {
+    ping,
     performLcr,
     lookupAllTeamsFQDNs,
     lookupAccountBySipRealm,
@@ -137,13 +140,32 @@ srf.invite((req, res) => {
   session.connect();
 });
 
-if (process.env.K8S) {
+if (process.env.K8S || process.env.HTTP_PORT) {
   const PORT = process.env.HTTP_PORT || 3000;
-  const getCount = () => activeCallIds.size;
   const healthCheck = require('@jambonz/http-health-check');
-  healthCheck({port: PORT, logger, path: '/', fn: getCount});
-}
 
+  const getCount = () => srf.locals.activeCallIds.size;
+
+  createHealthCheckApp(PORT, logger)
+    .then((app) => {
+      healthCheck({
+        app,
+        logger,
+        path: '/',
+        fn: getCount
+      });
+      healthCheck({
+        app,
+        logger,
+        path: '/system-health',
+        fn: systemHealth.bind(null, redisClient, ping, getCount)
+      });
+      return;
+    })
+    .catch((err) => {
+      logger.error({err}, 'Error creating health check server');
+    });
+}
 if ('test' !== process.env.NODE_ENV) {
   /* update call stats periodically */
   setInterval(() => {

lib/call-session.js

@@ -43,7 +43,8 @@ const initCdr = (srf, req) => {
     attempted_at: Date.now(),
     direction: 'outbound',
     host: srf.locals.sipAddress,
-    remote_host: uri.host
+    remote_host: uri.host,
+    trace_id: req.get('X-Trace-ID') || '00000000000000000000000000000000'
   };
 };
 
@@ -286,6 +287,7 @@ class CallSession extends Emitter {
               'all',
               '-X-MS-Teams-FQDN',
               '-X-MS-Teams-Tenant-FQDN',
+              '-X-Trace-ID',
               'X-CID',
               '-Allow',
               '-Session-Expires',
@@ -387,7 +389,7 @@ class CallSession extends Emitter {
               this.writeCdrs({...this.req.locals.cdr,
                 terminated_at: Date.now(),
                 termination_reason: 487 === status ? 'caller abandoned' : 'failed',
-                sip_status: status,
+                sip_status: status
               }).catch((err) => this.logger.error({err}, 'Error writing cdr for call failure'));
             }
           }
@@ -419,12 +421,14 @@ class CallSession extends Emitter {
     this.uas = uas;
     this.uac = uac;
     [uas, uac].forEach((dlg) => {
-      dlg.on('destroy', () => {
-        this.logger.info('call ended');
+      dlg.on('destroy', async() => {
+        const other = dlg.other;
         this.rtpEngineResource.destroy();
         this.activeCallIds.delete(this.req.get('Call-ID'));
         this.unsubscribeDTMF(this.logger, this.req.get('Call-ID'), this.rtpEngineOpts.uac.tag);
-        dlg.other.destroy();
+        try {
+          await other.destroy();
+        } catch (err) {}
 
         this.decrKey(this.callCountKey)
           .then((count) => {
@@ -445,6 +449,14 @@ class CallSession extends Emitter {
             duration: Math.floor((now - callStart) / 1000)
           }).catch((err) => this.logger.error({err}, 'Error writing cdr for completed call'));
         }
+        /* de-link the 2 Dialogs for GC */
+        dlg.removeAllListeners();
+        other.removeAllListeners();
+        dlg.other = null;
+        other.other = null;
+
+        this.logger.info(`call ended with normal termination, there are ${this.activeCallIds.size} active`);
+
         this.srf.endSession(this.req);
       });
     });

lib/middleware.js

@@ -17,7 +17,12 @@ module.exports = (srf, logger, opts) => {
     req.locals = req.locals || {};
     const callId = req.get('Call-ID');
     req.locals.account_sid = req.get('X-Account-Sid');
-    req.locals.logger = logger.child({callId, account_sid: req.locals.account_sid});
+    const traceId = req.locals.trace_id = req.get('X-Trace-ID');
+    req.locals.logger = logger.child({
+      callId,
+      traceId,
+      account_sid:
+      req.locals.account_sid});
 
     if (!req.locals.account_sid) {
       logger.info('missing X-Account-Sid on outbound call');

lib/utils.js

@@ -84,10 +84,32 @@ const equalsIgnoreOrder = (a, b) => {
   return true;
 };
 
+const  systemHealth = async(redisClient, ping, getCount) => {
+  await Promise.all([redisClient.ping(), ping()]);
+  return getCount();
+};
+
+const createHealthCheckApp = (port, logger) => {
+  const express = require('express');
+  const app = express();
+
+  app.use(express.urlencoded({ extended: true }));
+  app.use(express.json());
+
+  return new Promise((resolve) => {
+    app.listen(port, () => {
+      logger.info(`Health check server started at http://localhost:${port}`);
+      resolve(app);
+    });
+  });
+};
+
 module.exports = {
   makeRtpEngineOpts,
   selectHostPort,
   pingMsTeamsGateways,
   makeCallCountKey,
-  equalsIgnoreOrder
+  equalsIgnoreOrder,
+  systemHealth,
+  createHealthCheckApp
 };

package-lock.json

@@ -1,21 +1,21 @@
 {
   "name": "sbc-outbound",
-  "version": "v0.7.4",
+  "version": "v0.7.5",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "sbc-outbound",
-      "version": "v0.7.4",
+      "version": "v0.7.5",
       "license": "MIT",
       "dependencies": {
-        "@jambonz/db-helpers": "^0.6.17",
+        "@jambonz/db-helpers": "^0.6.18",
         "@jambonz/http-health-check": "^0.0.1",
         "@jambonz/mw-registrar": "0.2.2",
         "@jambonz/realtimedb-helpers": "^0.4.24",
         "@jambonz/rtpengine-utils": "^0.3.1",
         "@jambonz/stats-collector": "^0.1.6",
-        "@jambonz/time-series": "^0.1.6",
+        "@jambonz/time-series": "^0.1.9",
         "cidr-matcher": "^2.1.1",
         "debug": "^4.3.3",
         "drachtio-fn-b2b-sugar": "^0.0.12",
@@ -1375,9 +1375,9 @@
       }
     },
     "node_modules/@jambonz/db-helpers": {
-      "version": "0.6.17",
-      "resolved": "https://registry.npmjs.org/@jambonz/db-helpers/-/db-helpers-0.6.17.tgz",
-      "integrity": "sha512-3L5F6SFVpjn59CO23MpqMydTy1ox2FRe/kry9Zfv8kjTNTSMmfn01c51hjoXcX1v2kdx61qkGvCN6pHlc2LoSQ==",
+      "version": "0.6.18",
+      "resolved": "https://registry.npmjs.org/@jambonz/db-helpers/-/db-helpers-0.6.18.tgz",
+      "integrity": "sha512-9wTkOxqIbNd95+0yznGFPPTRih/xPRnX7ajrJMxzKqzzHQ/CUQgOfjfGcxmOBx3UxaWBmxyHQeYA3oZMPPyX+Q==",
       "dependencies": {
         "cidr-matcher": "^2.1.1",
         "debug": "^4.3.3",
@@ -1449,8 +1449,9 @@
       }
     },
     "node_modules/@jambonz/time-series": {
-      "version": "0.1.6",
-      "integrity": "sha512-EFO1ZFUazHLCzDYkAbpke0IfmPRogLkGg+KGWwz3SGpnuiltfohQ3TsJ5KG8Gp9mXqdpkyAioWSdaJxGDquthQ==",
+      "version": "0.1.9",
+      "resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.1.9.tgz",
+      "integrity": "sha512-FLD7mLGEMToG7s6LGWr/GtNp61753RBxyYuKSxCVI6G14bm6ydgcBdG85h8EGTVQlIp9dnwbn5ebSVsGb8a46w==",
       "dependencies": {
         "debug": "^4.3.1",
         "influx": "^5.8.0"
@@ -4093,8 +4094,9 @@
       }
     },
     "node_modules/minimist": {
-      "version": "1.2.5",
-      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
+      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q=="
     },
     "node_modules/mkdirp": {
       "version": "0.5.5",
@@ -4187,9 +4189,9 @@
       }
     },
     "node_modules/node-forge": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz",
-      "integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==",
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.0.tgz",
+      "integrity": "sha512-08ARB91bUi6zNKzVmaj3QO7cr397uiDT2nJ63cHjyNtCTWIgvS47j3eT0WfzUwS9+6Z5YshRaoasFkXCKrIYbA==",
       "engines": {
         "node": ">= 6.13.0"
       }
@@ -6806,9 +6808,9 @@
       "dev": true
     },
     "@jambonz/db-helpers": {
-      "version": "0.6.17",
-      "resolved": "https://registry.npmjs.org/@jambonz/db-helpers/-/db-helpers-0.6.17.tgz",
-      "integrity": "sha512-3L5F6SFVpjn59CO23MpqMydTy1ox2FRe/kry9Zfv8kjTNTSMmfn01c51hjoXcX1v2kdx61qkGvCN6pHlc2LoSQ==",
+      "version": "0.6.18",
+      "resolved": "https://registry.npmjs.org/@jambonz/db-helpers/-/db-helpers-0.6.18.tgz",
+      "integrity": "sha512-9wTkOxqIbNd95+0yznGFPPTRih/xPRnX7ajrJMxzKqzzHQ/CUQgOfjfGcxmOBx3UxaWBmxyHQeYA3oZMPPyX+Q==",
       "requires": {
         "cidr-matcher": "^2.1.1",
         "debug": "^4.3.3",
@@ -6874,8 +6876,9 @@
       }
     },
     "@jambonz/time-series": {
-      "version": "0.1.6",
-      "integrity": "sha512-EFO1ZFUazHLCzDYkAbpke0IfmPRogLkGg+KGWwz3SGpnuiltfohQ3TsJ5KG8Gp9mXqdpkyAioWSdaJxGDquthQ==",
+      "version": "0.1.9",
+      "resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.1.9.tgz",
+      "integrity": "sha512-FLD7mLGEMToG7s6LGWr/GtNp61753RBxyYuKSxCVI6G14bm6ydgcBdG85h8EGTVQlIp9dnwbn5ebSVsGb8a46w==",
       "requires": {
         "debug": "^4.3.1",
         "influx": "^5.8.0"
@@ -8849,8 +8852,9 @@
       }
     },
     "minimist": {
-      "version": "1.2.5",
-      "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz",
+      "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q=="
     },
     "mkdirp": {
       "version": "0.5.5",
@@ -8922,9 +8926,9 @@
       }
     },
     "node-forge": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz",
-      "integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w=="
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.0.tgz",
+      "integrity": "sha512-08ARB91bUi6zNKzVmaj3QO7cr397uiDT2nJ63cHjyNtCTWIgvS47j3eT0WfzUwS9+6Z5YshRaoasFkXCKrIYbA=="
     },
     "node-noop": {
       "version": "0.0.1",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "sbc-outbound",
-  "version": "v0.7.4",
+  "version": "v0.7.5",
   "main": "app.js",
   "engines": {
     "node": ">= 12.0.0"
@@ -22,18 +22,18 @@
   "description": "jambonz session border controller application for outbound calls",
   "scripts": {
     "start": "node app",
-    "test": "NODE_ENV=test JAMBONZ_HOSTING=1 JAMBONES_NETWORK_CIDR=127.0.0.1/32 JAMBONES_MYSQL_HOST=127.0.0.1 JAMBONES_MYSQL_USER=jambones_test JAMBONES_MYSQL_PASSWORD=jambones_test JAMBONES_MYSQL_DATABASE=jambones_test JAMBONES_REDIS_HOST=localhost JAMBONES_REDIS_PORT=16379 JAMBONES_TIME_SERIES_HOST=127.0.0.1 JAMBONES_LOGLEVEL=error DRACHTIO_SECRET=cymru DRACHTIO_HOST=127.0.0.1 DRACHTIO_PORT=9060 JAMBONES_RTPENGINES=127.0.0.1:12222 node test/ ",
+    "test": "NODE_ENV=test HTTP_PORT=3050 JAMBONZ_HOSTING=1 JAMBONES_NETWORK_CIDR=127.0.0.1/32 JAMBONES_MYSQL_HOST=127.0.0.1 JAMBONES_MYSQL_USER=jambones_test JAMBONES_MYSQL_PASSWORD=jambones_test JAMBONES_MYSQL_DATABASE=jambones_test JAMBONES_REDIS_HOST=localhost JAMBONES_REDIS_PORT=16379 JAMBONES_TIME_SERIES_HOST=127.0.0.1 JAMBONES_LOGLEVEL=error DRACHTIO_SECRET=cymru DRACHTIO_HOST=127.0.0.1 DRACHTIO_PORT=9060 JAMBONES_RTPENGINES=127.0.0.1:12222 node test/ ",
     "coverage": "./node_modules/.bin/nyc --reporter html --report-dir ./coverage npm run test",
     "jslint": "eslint app.js lib"
   },
   "dependencies": {
-    "@jambonz/db-helpers": "^0.6.17",
+    "@jambonz/db-helpers": "^0.6.18",
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/mw-registrar": "0.2.2",
     "@jambonz/realtimedb-helpers": "^0.4.24",
     "@jambonz/rtpengine-utils": "^0.3.1",
     "@jambonz/stats-collector": "^0.1.6",
-    "@jambonz/time-series": "^0.1.6",
+    "@jambonz/time-series": "^0.1.9",
     "cidr-matcher": "^2.1.1",
     "debug": "^4.3.3",
     "drachtio-fn-b2b-sugar": "^0.0.12",

test/sip-tests.js

@@ -2,7 +2,8 @@ const test = require('tape');
 const { output, sippUac } = require('./sipp')('test_sbc-outbound');
 const {execSync} = require('child_process');
 const debug = require('debug')('jambonz:sbc-outbound');
-const consoleLogger = {error: console.error, info: console.log, debug: console.log};
+const bent = require('bent');
+const getJSON = bent('json');
 
 process.on('unhandledRejection', (reason, p) => {
   console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
@@ -29,6 +30,11 @@ test('sbc-outbound tests', async(t) => {
   try {
     await connect(srf);
   
+    let obj = await getJSON('http://127.0.0.1:3050/');
+    t.ok(obj.calls === 0, 'HTTP GET / works (current call count)')
+    obj = await getJSON('http://127.0.0.1:3050/system-health');
+    t.ok(obj.calls === 0, 'HTTP GET /system-health works (health check)')
+
     /* call to unregistered user */
     debug('successfully connected to drachtio server');
     await sippUac('uac-pcap-device-404.xml');