feat: report call count to redis for server-wide drain coordination

A server may run several sbc-inbound and sbc-outbound processes.  When
one of the sbc-inbound processes handles an AWS autoscale drain it must
complete the terminating lifecycle hook only when the whole server is
idle, not just its own process — otherwise the instance is terminated
while sibling processes still have calls in progress.

Each process now writes its own call count to a per-pid redis key
(120s expiry, refreshed every 15s from activeCallIds.size) and
registers that key in a per-host set, which the draining process
enumerates and sums.  See the companion change in sbc-inbound.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.gitignore

@@ -34,6 +34,7 @@ build/Release
 node_modules
 
 .DS_Store
+.vscode
 
 examples/*
 CLAUDE.md

README.md

@@ -29,9 +29,19 @@ Configuration is provided via environment variables:
 |JAMBONES_RTPENGINES| commans-separated list of ip:ng-port for rtpengines (e.g. '172.31.32.10:22222')|yes|
 |JAMBONES_TIME_SERIES_HOST| influxdb host |yes|
 |JAMBONES_RECORD_ALL_CALLS| enable auto record calls |no|
+|JAMBONES_CODEC_OFFER_WITH_ORDER| comma-separated codec list to use as the outbound offer toward the carrier; the original codecs from the feature server are stripped first (e.g. `'opus,PCMU,PCMA,telephone-event'`) |no|
+|JAMBONES_CODEC_TRANSCODE| comma-separated codec list that rtpengine should add to the outbound offer and transcode on the fly when the carrier selects them — used to support codecs the feature server does not speak (e.g. `'AMR-WB/16000'`). Requires an rtpengine build that includes the corresponding codec module. |no|
+|JAMBONES_ACCEPT_AND_TRANSCODE| comma-separated codec list to accept on the outbound leg and transcode to PCMU/PCMA toward the feature server |no|
+|JAMBONES_ACCEPT_G729| if set, accept G.729 on the outbound leg and transcode to PCMU/PCMA (shorthand for `JAMBONES_ACCEPT_AND_TRANSCODE=g729`) |no|
 |K8S| service running as kubernetes service |no|
 |K8S_RTPENGINE_SERVICE_NAME| rtpengine service name(required for K8S) |no|
 
+#### Codec transcoding notes
+
+`JAMBONES_CODEC_OFFER_WITH_ORDER` and `JAMBONES_CODEC_TRANSCODE` can be combined. When both are set, rtpengine first strips the original codecs from the SDP, then adds the codecs listed in `JAMBONES_CODEC_OFFER_WITH_ORDER` to the offer, and finally adds the codecs from `JAMBONES_CODEC_TRANSCODE` with transcoding enabled. When only `JAMBONES_CODEC_TRANSCODE` is set, the original codecs from the feature server are kept in the offer and the transcode codecs are appended.
+
+Codecs that require licensed or optional rtpengine modules (AMR, AMR-WB, G.729, etc.) only work if rtpengine was compiled with support for them. Carrier-specific fmtp parameters (for example, `octet-align=1` for AMR-WB) can be appended to the codec name as needed (e.g. `AMR-WB/16000;octet-align=1`).
+
 ### running under pm2
 Typically, this application runs under [pm2](https://pm2.io) using an [ecosystem.config.js](https://pm2.keymetrics.io/docs/usage/application-declaration/) file similar to this:
 ```js

app.js

@@ -70,7 +70,8 @@ const {
   isMemberOfSet,
   addKey,
   deleteKey,
-  retrieveKey
+  retrieveKey,
+  addToSet
 } = require('@jambonz/realtimedb-helpers')({}, logger);
 
 const activeCallIds = new Map();
@@ -124,6 +125,17 @@ const {getRtpEngine, setRtpEngines} = require('@jambonz/rtpengine-utils')([], lo
 });
 srf.locals.getRtpEngine = getRtpEngine;
 
+/* report our call count to redis so a draining sbc-inbound process can count
+   calls across all sbc-inbound and sbc-outbound processes on this server */
+if (!process.env.K8S && 'test' !== process.env.NODE_ENV) {
+  srf.locals.callCountReporter = require('./lib/call-count-reporter')({
+    logger,
+    addKey,
+    addToSet,
+    getCount: () => activeCallIds.size
+  });
+}
+
 if (process.env.DRACHTIO_HOST && !process.env.K8S) {
   const cidrs = process.env.JAMBONES_NETWORK_CIDR
     .split(',')

data/srtp-transcoding.json

@@ -4,7 +4,7 @@
     "ICE": "force",
     "SDES": "off",
     "flags": ["generate mid", "SDES-no", "port latching"],
-    "rtcp-mux": ["require"]  
+    "rtcp-mux": ["offer"]
   },
   "teams": {
     "transport-protocol": "RTP/SAVP",

lib/call-count-reporter.js

@@ -0,0 +1,31 @@
+const os = require('os');
+
+/**
+ * Periodically report this process's count of calls in progress to redis.
+ * A server may host several sbc-inbound and sbc-outbound processes; when one
+ * of them handles an autoscale drain it needs to know when the entire server
+ * has no calls in progress, not just its own process.  Each process writes
+ * its own count under a per-pid key (with a short expiry, so keys from dead
+ * processes evaporate) and registers that key in a per-host set that the
+ * draining process can enumerate.
+ */
+const REPORT_INTERVAL = 15000;
+const KEY_EXPIRY_SECS = 120;
+
+module.exports = ({logger, addKey, addToSet, getCount}) => {
+  const prefix = process.env.JAMBONES_CLUSTER_ID || 'default';
+  const setName = `${prefix}:call-count-keys:${os.hostname()}`;
+  const key = `${prefix}:call-count:${os.hostname()}:${process.pid}`;
+
+  const report = () => {
+    addKey(key, `${getCount()}`, KEY_EXPIRY_SECS)
+      .catch((err) => logger.error({err}, 'call-count-reporter: error writing call count'));
+  };
+
+  addToSet(setName, key)
+    .catch((err) => logger.error({err}, `call-count-reporter: error adding ${key} to ${setName}`));
+  setInterval(report, REPORT_INTERVAL);
+  report();
+
+  return {key, setName};
+};

lib/call-session.js

@@ -110,6 +110,7 @@ class CallSession extends Emitter {
     this.decrKey = req.srf.locals.realtimeDbHelpers.decrKey;
     this.addKey = req.srf.locals.realtimeDbHelpers.addKey;
     this.retrieveKey = req.srf.locals.realtimeDbHelpers.retrieveKey;
+    this.retrieveHash = req.srf.locals.realtimeDbHelpers.retrieveHash;
 
     const {
       lookupOutboundCarrierForAccount,
@@ -236,6 +237,24 @@ class CallSession extends Emitter {
         this.logger.info(`sending call to registered user ${destUri}`);
       }
       else if (this.req.locals.target === 'forward') {
+        /* the feature-server sets X-Jambonz-SRTP (from the dial verb's srtpEncryption
+           option) to request encrypted media on a per-call basis */
+        const srtpMode = this.req.get('X-Jambonz-SRTP');
+        if (srtpMode) {
+          /* SDES (RTP/SAVP): pass teams=true to select the SDES srtp profile, matching
+             the proven carrier tls/srtp path above. This offers a=crypto to the target
+             (what SIP endpoints such as LiveKit expect), not DTLS-SRTP. */
+          this.logger.info({uri: this.req.uri, srtpMode},
+            'using SRTP (SDES) for forwarded call per X-Jambonz-SRTP');
+          this.rtpEngineOpts = makeRtpEngineOpts(this.req, false, true, false, true);
+          encryptedMedia = true;
+        }
+        else if (process.env.JAMBONES_SIPS_FORWARD_SRTP && this.req.uri.startsWith('sips:')) {
+          /* legacy global opt-in for sips: forwards (uses the DTLS srtp profile) */
+          this.logger.info({uri: this.req.uri}, 'using SRTP/TLS for forwarded sips: call');
+          this.rtpEngineOpts = makeRtpEngineOpts(this.req, false, true, false, false);
+          encryptedMedia = true;
+        }
         uris = [{
           private_network: await isPrivateVoipNetwork(this.req.uri),
           uri: this.req.uri
@@ -351,7 +370,8 @@ class CallSession extends Emitter {
                  * like a rare use case -- encryption is usually an all or nothing requirement.
                  */
                 this.logger.info({u}, `using SRTP for outbound call, pad crypto: ${o.pad_crypto ? 'yes' : 'no'}`);
-                this.rtpEngineOpts = makeRtpEngineOpts(this.req, false, true, o.pad_crypto, true);
+                // eslint-disable-next-line max-len
+                this.rtpEngineOpts = makeRtpEngineOpts(this.req, false, true, o.pad_crypto, true, o.remove_ice, o.dtls_off);
                 encryptedMedia = true;
               }
             });
@@ -524,6 +544,7 @@ class CallSession extends Emitter {
               '-X-Preferred-From-Host',
               '-X-Jambonz-FS-UUID',
               '-X-Voip-Carrier-Sid',
+              '-X-Jambonz-SRTP',
               '-X-SIP-Proxy'
             ],
             proxyResponseHeaders: [
@@ -658,7 +679,21 @@ class CallSession extends Emitter {
         }
       }
     } catch (err) {
-      if ('abandonded' !== err.message) this.logger.error({err}, `Error setting up outbound call to: ${uris}`);
+      if ('abandoned' !== err.message) this.logger.error({err}, `Error setting up outbound call to: ${uris}`);
+
+      /* the call count was incremented on 'init' but we bailed out before connecting (e.g. caller
+         hung up before the B leg answered), so we must decrement it here - no failure response is
+         sent on this path, so the res.once('end') safety net in middleware would not fire.
+         nudgeCallCounts is a no-op if the count was already decremented elsewhere. */
+      const {writeCallCount, writeCallCountSP, writeCallCountApp} = this.req.srf.locals;
+      nudgeCallCounts(this.req, 'failure', {
+        service_provider_sid: this.service_provider_sid,
+        account_sid: this.account_sid,
+        application_sid: this.application_sid,
+        callId: this.req.locals.callId
+      }, this.decrKey, {writeCallCountSP, writeCallCount, writeCallCountApp})
+        .catch((err) => this.logger.error(err, 'Error decrementing call counts'));
+
       this.emit('failed');
       this.srf.endSession(this.req);
       this.rtpEngineResource.destroy();
@@ -723,10 +758,13 @@ class CallSession extends Emitter {
         if (this.req.locals.cdr) {
           const now = Date.now();
           const day = new Date();
-          const recordAllCalls = this.req.locals.record_all_calls;
+          // eslint-disable-next-line max-len
+          const {hasRecording = false} = await this.retrieveHash(`call:${this.account_sid}:${this.req.locals.cdr.call_sid}`) ?? {};
+          const recordAllCalls = this.req.locals.record_all_calls || hasRecording;
+          const record_format = this.req.locals.account.record_format || 'mp3';
           let recording_url = `/Accounts/${this.account_sid}/RecentCalls/${this.req.locals.cdr.call_sid}/record`;
           recording_url += `/${day.getFullYear()}/${(day.getMonth() + 1).toString().padStart(2, '0')}`;
-          recording_url += `/${day.getDate().toString().padStart(2, '0')}/${recordAllCalls}`;
+          recording_url += `/${day.getDate().toString().padStart(2, '0')}/${record_format}`;
           this.writeCdrs({...this.req.locals.cdr,
             terminated_at: now,
             termination_reason: dlg.type === 'uas' ? 'caller hungup' : 'called party hungup',
@@ -882,8 +920,14 @@ Duration=${payload.duration} `
         process.env.JAMBONES_ACCEPT_G729 ? 'g729' : '';
       offerMedia = { ...offerMedia, ...(acceptCodecs && dlg.type === 'uac' &&
         { codec: { mask: acceptCodecs, transcode: 'pcmu,pcma' } })};
-      answerMedia = { ...answerMedia, ...(process.env.JAMBONES_CODEC_OFFER_WITH_ORDER && dlg.type === 'uac' &&
-        { codec: {offer: process.env.JAMBONES_CODEC_OFFER_WITH_ORDER.split(','), strip: 'all' }})};
+      answerMedia = { ...answerMedia, ...((process.env.JAMBONES_CODEC_OFFER_WITH_ORDER ||
+        process.env.JAMBONES_CODEC_TRANSCODE) && dlg.type === 'uac' &&
+        { codec: {
+          ...(process.env.JAMBONES_CODEC_OFFER_WITH_ORDER &&
+            { offer: process.env.JAMBONES_CODEC_OFFER_WITH_ORDER.split(','), strip: 'all' }),
+          ...(process.env.JAMBONES_CODEC_TRANSCODE &&
+            { transcode: process.env.JAMBONES_CODEC_TRANSCODE.split(',') })
+        }})};
       let opts = {
         ...this.rtpEngineOpts.common,
         ...offerMedia,
@@ -1208,7 +1252,6 @@ Duration=${payload.duration} `
         ...(req.has('X-Retain-Call-Sid') && {'X-Retain-Call-Sid': req.get('X-Retain-Call-Sid')}),
         ...(req.has('X-Account-Sid') && {'X-Account-Sid': req.get('X-Account-Sid')})
       };
-
       const uac = await this.srf.createUAC(referTo.uri, {localSdp: dlg.local.sdp, headers});
       this.uas = uac;
       uac.type = 'uas';
@@ -1217,7 +1260,7 @@ Duration=${payload.duration} `
       uac.on('info', this._onInfo.bind(this, uac));
       uac.on('modify', this._onReinvite.bind(this, uac));
       uac.on('refer', this._onFeatureServerTransfer.bind(this, uac));
-      uac.on('destroy', () => {
+      uac.on('destroy', async() => {
         this.logger.info('call ended with normal termination');
         this.rtpEngineResource.destroy();
         this.activeCallIds.delete(this.req.get('Call-ID'));
@@ -1226,10 +1269,13 @@ Duration=${payload.duration} `
         if (this.req.locals.cdr) {
           const now = Date.now();
           const day = new Date();
-          const recordAllCalls = this.req.locals.record_all_calls;
+          // eslint-disable-next-line max-len
+          const {hasRecording = false} = await this.retrieveHash(`call:${this.account_sid}:${this.req.locals.cdr.call_sid}`) ?? {} ;
+          const recordAllCalls = this.req.locals.record_all_calls || hasRecording;
+          const record_format = this.req.locals.account.record_format || 'mp3';
           let recording_url = `/Accounts/${this.account_sid}/RecentCalls/${this.req.locals.cdr.call_sid}/record`;
           recording_url += `/${day.getFullYear()}/${(day.getMonth() + 1).toString().padStart(2, '0')}`;
-          recording_url += `/${day.getDate().toString().padStart(2, '0')}/${recordAllCalls}`;
+          recording_url += `/${day.getDate().toString().padStart(2, '0')}/${record_format}`;
           this.writeCdrs({...this.req.locals.cdr,
             terminated_at: now,
             termination_reason: 'caller hungup',

lib/utils.js

@@ -5,7 +5,8 @@ const CIDRMatcher = require('cidr-matcher');
 const dns = require('dns');
 const sdpTransform = require('sdp-transform');
 
-function makeRtpEngineOpts(req, srcIsUsingSrtp, dstIsUsingSrtp, padCrypto, teams) {
+// eslint-disable-next-line max-len
+function makeRtpEngineOpts(req, srcIsUsingSrtp, dstIsUsingSrtp, padCrypto, teams, remove_ice = false, dtls_off = false) {
   const from = req.getParsedHeader('from');
   const rtpCopy = JSON.parse(JSON.stringify(rtpCharacteristics));
   const srtpCopy = JSON.parse(JSON.stringify(srtpCharacteristics));
@@ -16,6 +17,14 @@ function makeRtpEngineOpts(req, srcIsUsingSrtp, dstIsUsingSrtp, padCrypto, teams
   }
 
   const srtpOpts = teams ? srtpCopy['teams'] : srtpCopy['default'];
+
+  if (remove_ice) {
+    srtpOpts.ICE = 'remove';
+  }
+  if (dtls_off) {
+    srtpOpts.DTLS = 'off';
+  }
+
   const dstOpts = JSON.parse(JSON.stringify(dstIsUsingSrtp ? srtpOpts : rtpCopy));
   const srcOpts = JSON.parse(JSON.stringify(srcIsUsingSrtp ? srtpOpts : rtpCopy));
 
@@ -49,6 +58,10 @@ function makeRtpEngineOpts(req, srcIsUsingSrtp, dstIsUsingSrtp, padCrypto, teams
         offer: process.env.JAMBONES_CODEC_OFFER_WITH_ORDER.split(','),
         strip: 'all'
       }),
+    ...(process.env.JAMBONES_CODEC_TRANSCODE &&
+      {
+        transcode: process.env.JAMBONES_CODEC_TRANSCODE.split(',')
+      }),
   };
 
   return {

package.json

@@ -1,9 +1,9 @@
 {
   "name": "sbc-outbound",
-  "version": "0.9.5",
+  "version": "0.9.9",
   "main": "app.js",
   "engines": {
-    "node": ">= 18.0.0"
+    "node": ">= 20.0.0"
   },
   "keywords": [
     "sip",
@@ -31,7 +31,7 @@
     "@jambonz/db-helpers": "^0.9.18",
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/mw-registrar": "0.2.7",
-    "@jambonz/realtimedb-helpers": "^0.8.13",
+    "@jambonz/realtimedb-helpers": "^0.8.21",
     "@jambonz/rtpengine-utils": "^0.4.4",
     "@jambonz/siprec-client-utils": "^0.2.10",
     "@jambonz/stats-collector": "^0.1.10",
@@ -39,7 +39,7 @@
     "cidr-matcher": "^2.1.1",
     "debug": "^4.4.3",
     "drachtio-fn-b2b-sugar": "^0.2.1",
-    "drachtio-srf": "^5.0.12",
+    "drachtio-srf": "^5.0.21",
     "express": "^4.21.2",
     "pino": "^10.1.0",
     "sdp-transform": "^2.15.0"
@@ -48,7 +48,7 @@
     "bent": "^7.3.12",
     "eslint": "^9.17.0",
     "eslint-plugin-promise": "^7.2.1",
-    "nyc": "^15.1.0",
-    "tape": "^5.7.5"
+    "nyc": "^17.1.0",
+    "tape": "^5.9.0"
   }
 }