0.9.10

* fix: keep siprec recording alive across a feature server transfer

A cross-feature-server move (enqueue/dequeue or conference) re-negotiates the
feature-server leg in _onFeatureServerTransfer, but nothing rebuilt the rtpengine
subscription the SIPREC recording forks from, so the recorder went silent from the
moment the call moved. The fresh destroy handler installed on the new leg also
dropped the _stopRecording() call the original handlers have, so the SIPREC dialog
was never BYEd and the recorder had to wait out its media timeout.

Rebuild the subscription after the transfer re-negotiates media, and stop the
recording when the transferred leg ends. The resubscribe call is guarded so this
is safe to deploy before @jambonz/siprec-client-utils is bumped.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: drop the siprec re-subscribe, keep the missing teardown

Running the cross-feature-server move against a real two-feature-server cluster
showed the rtpengine subscription survives the REFER on its own: with
resubscribe() removed, the fork still carried the post-transfer conversation
(50 packets/s on both streams, and the words spoken after the move transcribed
straight out of the recording). rtpengine keeps non-offer-answer subscriptions
across an answer, so there was never anything to rebuild - the earlier commit
was fixing a fault that does not exist.

What does fail, and what this branch still fixes, is the teardown: the destroy
handler installed on the transferred leg never stopped the recording, so the
recorder was left without a BYE.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

app.js

@@ -155,6 +155,17 @@ srf.locals = {
 };
 const activeCallIds = srf.locals.activeCallIds;
 
+/* report our call count to redis so a draining process can count calls across
+   all sbc-inbound and sbc-outbound processes on this server */
+if (!process.env.K8S && 'test' !== process.env.NODE_ENV) {
+  srf.locals.callCountReporter = require('./lib/call-count-reporter')({
+    logger,
+    addKey,
+    addToSet,
+    getCount: () => activeCallIds.size
+  });
+}
+
 const {
   initLocals,
   handleSipRec,

lib/autoscale-manager.js

@@ -23,23 +23,55 @@ module.exports = (logger) => {
             const {srf} = require('..');
             const {activeCallIds, removeFromRedis} = srf.locals;
 
+            /* reject new INVITEs with 503 so senders fail over to another SBC */
+            srf.locals.dryUpCalls = true;
+
             /* remove our private IP from the set of active SBCs so rtp and fs know we are gone */
             removeFromRedis();
 
-            /* if we have zero calls, we can complete the scale-in right now */
-            const calls = activeCallIds.size;
-            if (0 === calls) {
-              logger.info('scale-in can complete immediately as we have no calls in progress');
-              lifecycleEmitter.completeScaleIn();
-            }
-            else {
-              logger.info(`${calls} calls in progress; scale-in will complete when they are done`);
-            }
+            /* count calls in progress across all sbc-inbound and sbc-outbound
+               processes on this server, if they are reporting; otherwise
+               fall back to counting only our own */
+            const countServerCalls = async() => {
+              const reporter = srf.locals.callCountReporter;
+              if (!reporter) return activeCallIds.size;
+              const {retrieveSet, retrieveKey} = srf.locals.realtimeDbHelpers;
+              const keys = await retrieveSet(reporter.setName);
+              let count = 0;
+              for (const key of keys) {
+                count += parseInt(await retrieveKey(key), 10) || 0;
+              }
+              return Math.max(count, activeCallIds.size);
+            };
+
+            /* poll until calls have dried up, then complete the scale-in;
+               require two consecutive zero readings since reported counts
+               may be up to 15s stale */
+            let consecutiveZeroCounts = 0;
+            const timer = setInterval(async() => {
+              try {
+                const calls = await countServerCalls();
+                if (0 === calls) {
+                  if (++consecutiveZeroCounts >= 2) {
+                    clearInterval(timer);
+                    logger.info('scale-in complete now that calls have dried up');
+                    lifecycleEmitter.completeScaleIn();
+                  }
+                }
+                else {
+                  consecutiveZeroCounts = 0;
+                  logger.info(`${calls} calls in progress on this server; scale-in will complete when they are done`);
+                }
+              } catch (err) {
+                logger.error({err}, 'Error counting calls in progress during scale-in');
+              }
+            }, 20000);
           })
           .on(LifeCycleEvents.StandbyEnter, () => {
             lifecycleEmitter.dryUpCalls = true;
             const {srf} = require('..');
             const {removeFromRedis} = srf.locals;
+            srf.locals.dryUpCalls = true;
             removeFromRedis();
 
             logger.info('AWS enter pending state notification: begin drying up calls');
@@ -48,6 +80,7 @@ module.exports = (logger) => {
             lifecycleEmitter.dryUpCalls = false;
             const {srf} = require('..');
             const {addToRedis} = srf.locals;
+            srf.locals.dryUpCalls = false;
             addToRedis();
 
             logger.info('AWS exit pending state notification: re-enable calls');

lib/call-count-reporter.js

@@ -0,0 +1,31 @@
+const os = require('os');
+
+/**
+ * Periodically report this process's count of calls in progress to redis.
+ * A server may host several sbc-inbound and sbc-outbound processes; when one
+ * of them handles an autoscale drain it needs to know when the entire server
+ * has no calls in progress, not just its own process.  Each process writes
+ * its own count under a per-pid key (with a short expiry, so keys from dead
+ * processes evaporate) and registers that key in a per-host set that the
+ * draining process can enumerate.
+ */
+const REPORT_INTERVAL = 15000;
+const KEY_EXPIRY_SECS = 120;
+
+module.exports = ({logger, addKey, addToSet, getCount}) => {
+  const prefix = process.env.JAMBONES_CLUSTER_ID || 'default';
+  const setName = `${prefix}:call-count-keys:${os.hostname()}`;
+  const key = `${prefix}:call-count:${os.hostname()}:${process.pid}`;
+
+  const report = () => {
+    addKey(key, `${getCount()}`, KEY_EXPIRY_SECS)
+      .catch((err) => logger.error({err}, 'call-count-reporter: error writing call count'));
+  };
+
+  addToSet(setName, key)
+    .catch((err) => logger.error({err}, `call-count-reporter: error adding ${key} to ${setName}`));
+  setInterval(report, REPORT_INTERVAL);
+  report();
+
+  return {key, setName};
+};

lib/call-session.js

@@ -1109,6 +1109,7 @@ Duration=${payload.duration} `
         this.rtpEngineResource.destroy();
         this.activeCallIds.delete(this.req.get('Call-ID'));
         uac.other.destroy();
+        this._stopRecording();
         this.srf.endSession(this.req);
       });
 

lib/middleware.js

@@ -34,6 +34,15 @@ module.exports = function(srf, logger) {
 
   const initLocals = (req, res, next) => {
     const callId = req.get('Call-ID');
+
+    /* if we are drying up calls prior to scale-in, reject new INVITEs so the
+       sender fails over to another SBC; allow INVITE with Replaces through
+       since it targets a call already in progress on this server */
+    if (srf.locals.dryUpCalls && !req.has('Replaces')) {
+      logger.info({callId}, 'rejecting INVITE with 503 as we are drying up calls before scale-in');
+      return res.send(503);
+    }
+
     req.locals = req.locals || {callId};
     req.locals.nudge = 0;
 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "sbc-inbound",
-  "version": "0.9.8",
+  "version": "0.9.10",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "sbc-inbound",
-      "version": "0.9.8",
+      "version": "0.9.10",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-auto-scaling": "^3.549.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "sbc-inbound",
-  "version": "0.9.8",
+  "version": "0.9.10",
   "main": "app.js",
   "engines": {
     "node": ">= 20.0.0"