refined the method for syncing call count updates and added debugging (#163)

* refined the method for syncing call count updates and added debugging

* wip

* include callId in debug key

.github/workflows/build.yml

@@ -6,12 +6,17 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
         with:
-          node-version: lts/*
+          node-version: 20.x
       - run: npm ci
       - run: npm run jslint
+      - name: Install Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+          docker-compose --version
       - run: npm test
 
  

.husky/pre-commit

@@ -1,4 +1,4 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
 
-npm run jslint
+npm run jslint

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 Drachtio Communications Services, LLC
+Copyright (c) 2018-2024 FirstFive8, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

app.js

@@ -50,7 +50,8 @@ const {
   lookupSipGatewaysByCarrier,
   lookupCarrierBySid,
   queryCallLimits,
-  lookupCarrierByAccountLcr
+  lookupCarrierByAccountLcr,
+  lookupSystemInformation
 } = require('@jambonz/db-helpers')({
   host: process.env.JAMBONES_MYSQL_HOST,
   port: process.env.JAMBONES_MYSQL_PORT || 3306,
@@ -67,6 +68,9 @@ const {
   decrKey,
   retrieveSet,
   isMemberOfSet,
+  addKey,
+  deleteKey,
+  retrieveKey
 } = require('@jambonz/realtimedb-helpers')({}, logger);
 
 const activeCallIds = new Map();
@@ -84,6 +88,7 @@ srf.locals = {...srf.locals,
   queryCdrs,
   activeCallIds,
   idleEmitter,
+  privateNetworkCidr: process.env.PRIVATE_VOIP_NETWORK_CIDR || null,
   dbHelpers: {
     ping,
     lookupOutboundCarrierForAccount,
@@ -94,9 +99,14 @@ srf.locals = {...srf.locals,
     lookupSipGatewaysByCarrier,
     lookupCarrierBySid,
     queryCallLimits,
-    lookupCarrierByAccountLcr
+    lookupCarrierByAccountLcr,
+    lookupSystemInformation
   },
   realtimeDbHelpers: {
+    client: redisClient,
+    addKey,
+    deleteKey,
+    retrieveKey,
     createHash,
     retrieveHash,
     incrKey,
@@ -183,10 +193,22 @@ if (process.env.K8S || process.env.HTTP_PORT) {
     });
 }
 if ('test' !== process.env.NODE_ENV) {
-  /* update call stats periodically */
-  setInterval(() => {
+  /* update call stats periodically as well as definition of private network cidr */
+  setInterval(async() => {
     stats.gauge('sbc.sip.calls.count', activeCallIds.size, ['direction:outbound',
       `instance_id:${process.env.INSTANCE_ID || 0}`]);
+
+    const r = await lookupSystemInformation();
+    if (r) {
+      if (r.private_network_cidr !== srf.locals.privateNetworkCidr) {
+        logger.info(`updating private network cidr from ${srf.locals.privateNetworkCidr} to ${r.private_network_cidr}`);
+        srf.locals.privateNetworkCidr = r.private_network_cidr;
+      }
+      // Update system log level
+      if (r.log_level) {
+        logger.level = r.log_level;
+      }
+    }
   }, 20000);
 }
 
@@ -261,4 +283,4 @@ function handle(signal) {
   }
 }
 
-module.exports = {srf};
+module.exports = {srf, logger};

lib/call-session.js

@@ -1,36 +1,61 @@
 const Emitter = require('events');
 const sdpTransform = require('sdp-transform');
 const SrsClient = require('@jambonz/siprec-client-utils');
-const {makeRtpEngineOpts, nudgeCallCounts, isPrivateVoipNetwork} = require('./utils');
+const {
+  makeRtpEngineOpts,
+  nudgeCallCounts,
+  isPrivateVoipNetwork,
+  isBlackListedSipGateway,
+  makeFullMediaReleaseKey,
+  makePartnerFullMediaReleaseKey
+} = require('./utils');
+const { MediaPath } = require('./constants.json');
 const {forwardInDialogRequests} = require('drachtio-fn-b2b-sugar');
 const {SipError, stringifyUri, parseUri} = require('drachtio-srf');
 const debug = require('debug')('jambonz:sbc-outbound');
 
 const makeInviteInProgressKey = (callid) => `sbc-out-iip${callid}`;
 const IMMUTABLE_HEADERS = ['via', 'from', 'to', 'call-id', 'cseq', 'max-forwards', 'content-length'];
-/**
- * this is to make sure the outgoing From has the number in the incoming From
- * and not the incoming PAI
- */
-const createBLegFromHeader = (req, teams, register_from_domain = null) => {
+
+const createBLegFromHeader = ({
+  logger,
+  req,
+  host,
+  fromUser,
+  register_from_domain,
+  transport,
+  teams = false,
+  scheme = 'sip'
+}) => {
   const from = req.getParsedHeader('From');
   const uri = parseUri(from.uri);
-  let user = uri.user || 'anonymous';
-  let host = 'localhost';
-  if (teams) {
-    host = req.get('X-MS-Teams-Tenant-FQDN');
-  }
-  else if (req.has('X-Preferred-From-User') || req.has('X-Preferred-From-Host')) {
-    user = req.get('X-Preferred-From-User') || user;
-    host = req.get('X-Preferred-From-Host') || host;
-  } else if (register_from_domain) {
-    host = register_from_domain;
+  const transportParam = transport ? `;transport=${transport}` : '';
+
+  logger.debug({from, fromUser, uri, host, scheme, transport, teams}, 'createBLegFromHeader');
+  /* user */
+  const user = fromUser || req.get('X-Preferred-From-User') || uri.user || 'anonymous';
+
+  /* host */
+  if (!host) {
+    if (teams) {
+      host = req.get('X-MS-Teams-Tenant-FQDN');
+    }
+    else if (req.has('X-Preferred-From-Host')) {
+      host = req.get('X-Preferred-From-Host');
+    } else if (register_from_domain) {
+      host = register_from_domain;
+    }
+    else {
+      host = 'localhost';
+    }
   }
+
   if (from.name) {
-    return `${from.name} <sip:${user}@${host}>`;
+    return `${from.name} <${scheme}:${user}@${host}${transportParam}>`;
   }
-  return `sip:${user}@${host}`;
+  return `<${scheme}:${user}@${host}${transportParam}>`;
 };
+
 const createBLegToHeader = (req, teams) => {
   const to = req.getParsedHeader('To');
   const host = teams ? req.get('X-MS-Teams-Tenant-FQDN') : 'localhost';
@@ -54,6 +79,7 @@ const initCdr = (req, invite) => {
     account_sid: req.get('X-Account-Sid'),
     call_sid: req.get('X-Call-Sid'),
     sip_callid: invite.get('Call-ID'),
+    ...(req.has('X-CID') && {sip_parent_callid: req.get('X-CID')}),
     ...(applicationSid && {application_sid: applicationSid}),
     from,
     to,
@@ -71,7 +97,10 @@ const updateRtpEngineFlags = (sdp, opts) => {
   try {
     const parsed = sdpTransform.parse(sdp);
     const codec = parsed.media[0].rtp[0].codec;
-    if (['PCMU', 'PCMA'].includes(codec)) opts.flags.push(`codec-accept-${codec}`);
+    if (['PCMU', 'PCMA'].includes(codec)) {
+      opts.flags.push(`codec-accept-${codec}`);
+      opts.flags.push('codec-accept-telephone-event');
+    }
   } catch (err) {}
   return opts;
 };
@@ -90,6 +119,8 @@ class CallSession extends Emitter {
     this.writeCdrs = this.srf.locals.writeCdrs;
 
     this.decrKey = req.srf.locals.realtimeDbHelpers.decrKey;
+    this.addKey = req.srf.locals.realtimeDbHelpers.addKey;
+    this.retrieveKey = req.srf.locals.realtimeDbHelpers.retrieveKey;
 
     const {
       lookupOutboundCarrierForAccount,
@@ -102,7 +133,8 @@ class CallSession extends Emitter {
     this.lookupSipGatewaysByCarrier = lookupSipGatewaysByCarrier;
     this.lookupCarrierByAccountLcr = lookupCarrierByAccountLcr;
 
-    this._mediaReleased = false;
+    this._mediaPath = MediaPath.FullMedia;
+
     this.recordingNoAnswerTimeout = (process.env.JAMBONES_RECORDING_NO_ANSWER_TIMEOUT || 2) * 1000;
   }
 
@@ -123,12 +155,7 @@ class CallSession extends Emitter {
   }
 
   get isMediaReleased() {
-    return this._mediaReleased;
-  }
-
-  get calleeIsUsingSrtp() {
-    const tp = this.rtpEngineOpts?.uac?.mediaOpts['transport-protocol'];
-    return tp && -1 !== tp.indexOf('SAVP');
+    return this._mediaPath !== MediaPath.FullMedia;
   }
 
   subscribeForDTMF(dlg) {
@@ -168,7 +195,7 @@ class CallSession extends Emitter {
       subscribeAnswer,
       unsubscribe
     } = engine;
-    const {createHash, retrieveHash} = this.srf.locals.realtimeDbHelpers;
+    const {client, createHash, retrieveHash} = this.srf.locals.realtimeDbHelpers;
     this.offer = offer;
     this.answer = answer;
     this.del = del;
@@ -186,15 +213,14 @@ class CallSession extends Emitter {
 
     this.rtpEngineOpts = makeRtpEngineOpts(this.req, false, this.useWss || teams, false, teams);
     this.rtpEngineResource = {destroy: this.del.bind(null, this.rtpEngineOpts.common)};
-    let proxy, uris;
+    let proxy, uris = [];
     const mapGateways = new Map();
+    let encryptedMedia = false;
 
     try {
       // determine where to send the call
       debug(`connecting call: ${JSON.stringify(this.req.locals)}`);
-      let headers = {
-        'From': createBLegFromHeader(this.req, teams),
-        'Contact': createBLegFromHeader(this.req, teams),
+      const headers = {
         'To': createBLegToHeader(this.req, teams),
         Allow: 'INVITE, ACK, OPTIONS, CANCEL, BYE, NOTIFY, UPDATE, PRACK',
         'X-Account-Sid': this.account_sid
@@ -218,6 +244,7 @@ class CallSession extends Emitter {
         if (!contact.includes('transport=ws')) {
           proxy = this.req.locals.registration.proxy;
         }
+        this.logger.info(`sending call to registered user ${destUri}`);
       }
       else if (this.req.locals.target === 'forward') {
         uris = [{
@@ -242,10 +269,6 @@ class CallSession extends Emitter {
           private_network: false,
           uri: `sip:${this.req.calledNumber}@sip.pstnhub.microsoft.com`
         }];
-        headers = {
-          ...headers,
-          Contact: `sip:${this.req.calledNumber}@${this.req.get('X-MS-Teams-Tenant-FQDN')}:5061;transport=tls`
-        };
       }
       else {
         try {
@@ -280,7 +303,13 @@ class CallSession extends Emitter {
           }
           const vc = await this.lookupCarrierBySid(voip_carrier_sid);
           const gateways = await this.lookupSipGatewaysByCarrier(voip_carrier_sid);
-          const gws = (gateways || [])
+          const goodGateways = [];
+          for (const g of gateways) {
+            if (!await isBlackListedSipGateway(client, this.logger, g.sip_gateway_sid)) {
+              goodGateways.push(g);
+            }
+          }
+          const gws = (goodGateways || [])
             .filter((gw) => gw.outbound);
           if (gws.length) {
             uris = [];
@@ -289,17 +318,18 @@ class CallSession extends Emitter {
                 this.req.calledNumber.slice(1) :
                 this.req.calledNumber;
               const prefix = vc.tech_prefix || '';
-              const protocol =  o.protocol?.startsWith('tls') ? 'tls' : (o.protocol || 'udp');
+              const transport =  o.protocol?.startsWith('tls') ? 'tls' : (o.protocol || 'udp');
               const hostport = !o.port || 5060 === o.port ? o.ipv4 : `${o.ipv4}:${o.port}`;
               const prependPlus = vc.e164_leading_plus && !this.req.calledNumber.startsWith('0') ? '+' : '';
-              const transport = `transport=${protocol}`;
-              const scheme = (protocol === 'tls' && !process.env.JAMBONES_USE_BEST_EFFORT_TLS) ? 'sips' : 'sip';
-              const u = `${scheme}:${prefix}${prependPlus}${calledNumber}@${hostport};${transport}`;
+              const scheme = transport === 'tls' && !process.env.JAMBONES_USE_BEST_EFFORT_TLS && o.use_sips_scheme ?
+                'sips' : 'sip';
+              let u = `${scheme}:${prefix}${prependPlus}${calledNumber}@${hostport};transport=${transport}`;
               const obj = {
                 name: vc.name,
                 diversion: vc.diversion,
                 hostport,
-                protocol: o.protocol,
+                transport,
+                scheme,
                 register_from_domain: vc.register_from_domain
               };
               if (vc.register_username && vc.register_password) {
@@ -308,8 +338,14 @@ class CallSession extends Emitter {
                   password: vc.register_password
                 };
               }
+              if (vc.requires_register && vc.register_sip_realm?.length > 0) {
+                proxy = u;
+                u = `${scheme}:${prefix}${prependPlus}${calledNumber}@${vc.register_sip_realm};transport=${transport}`;
+                this.logger.debug({uri: u}, `using outbound proxy for this registered trunk: ${proxy}`);
+              }
               mapGateways.set(u, obj);
               uris.push(u);
+              this.logger.debug({gateway: o}, `pushed uri ${u}`);
               if (o.protocol === 'tls/srtp') {
                 /**  TODO: this is a bit of a hack in the sense that we are not
                  * supporting a scenario where you have a carrier with several outbound
@@ -319,9 +355,10 @@ class CallSession extends Emitter {
                  */
                 this.logger.info({u}, `using SRTP for outbound call, pad crypto: ${o.pad_crypto ? 'yes' : 'no'}`);
                 this.rtpEngineOpts = makeRtpEngineOpts(this.req, false, true, o.pad_crypto, true);
+                encryptedMedia = true;
               }
             });
-            // Check private network for each gw
+            /* Check private network for each gw */
             uris = await Promise.all(uris.map(async(u) => {
               return {
                 private_network: await isPrivateVoipNetwork(u),
@@ -332,6 +369,7 @@ class CallSession extends Emitter {
           }
           else {
             this.logger.info({voip_carrier_sid}, 'no outbound gateways found for requested carrier');
+            this.res.send(603);
           }
         } catch (err) {
           debug(err);
@@ -342,14 +380,12 @@ class CallSession extends Emitter {
         debug(`sending call to PSTN ${uris}`);
       }
 
-      // private_network should be called at last
+      /* private_network should be called at last - try public first */
       uris = uris.sort((a, b) => a.private_network - b.private_network);
       const toPrivate = uris.some((u) => u.private_network === true);
       const toPublic = uris.some((u) => u.private_network === false);
       let isOfferUpdatedToPrivate = toPrivate && !toPublic;
 
-
-      // rtpengine 'offer'
       const opts = updateRtpEngineFlags(this.req.body, {
         ...this.rtpEngineOpts.common,
         ...this.rtpEngineOpts.uac.mediaOpts,
@@ -358,7 +394,6 @@ class CallSession extends Emitter {
         sdp: this.req.body
       });
       let response = await this.offer(opts);
-      debug(`response from rtpengine to offer ${JSON.stringify(response)}`);
       this.logger.debug({offer: opts, response}, 'initial offer to rtpengine');
       if ('ok' !== response.result) {
         this.logger.error(`rtpengine offer failed with ${JSON.stringify(response)}`);
@@ -371,10 +406,12 @@ class CallSession extends Emitter {
       // crank through the list of gateways until connected, exhausted or caller hangs up
       let earlyMedia = false;
       while (uris.length) {
-        let hdrs = { ...headers};
+        let hdrs = { ...headers };
         const {private_network, uri} = uris.shift();
+
+        /* if we've exhausted attempts to public endpoints and are switching to trying private, we need new rtp */
         if (private_network && !isOfferUpdatedToPrivate) {
-          // Cannot make call to all public Uris, now come to talk with private network Uris
+          this.logger.info('switching to attempt to deliver call via private network now..');
           this.rtpEngineResource.destroy()
             .catch((err) => this.logger.info({err}, 'Error destroying rtpe to re-connect to private network'));
           response = await this.offer({
@@ -383,8 +420,8 @@ class CallSession extends Emitter {
           });
           isOfferUpdatedToPrivate = true;
         }
-        const gw = mapGateways.get(uri);
-        const passFailure = 0 === uris.length;  // only a single target
+
+        /* on the second and subsequent attempts, use the same Call-ID and CSeq from the first attempt */
         if (0 === uris.length) {
           try {
             const key = makeInviteInProgressKey(this.req.get('Call-ID'));
@@ -400,32 +437,76 @@ class CallSession extends Emitter {
             this.logger.info({err}, 'Error retrieving iip key');
           }
         }
-        // INVITE request line and To header should be the same.
-        hdrs = {...hdrs, 'To': uri};
-        if (gw) {
-          this.logger.info({gw}, `sending INVITE to ${uri} via carrier ${gw.name}`);
-          if (gw.diversion) {
-            let div = gw.diversion;
-            if (div.startsWith('+')) {
-              div = `<sip:${div}@${gw.hostport}>;reason=unknown;counter=1;privacy=off`;
-            }
-            else div = `<sip:+${div}@${gw.hostport}>;reason=unknown;counter=1;privacy=off`;
-            hdrs = {
-              ...hdrs,
-              'Diversion': div
-            };
-          }
-          if (gw.register_from_domain) {
-            hdrs = {
-              ...hdrs,
-              'From': createBLegFromHeader(this.req, teams, gw.register_from_domain)
-            };
-          }
-        }
-        else this.logger.info(`sending INVITE to ${uri} via proxy ${proxy})`);
-        try {
-          const responseHeaders = this.privateSipAddress ? {Contact: `<sip:${this.privateSipAddress}>`} : {};
 
+        /* INVITE request line and To header should be the same. */
+        hdrs = {...hdrs, 'To': uri};
+
+        /* only now can we set Contact & From header since they depend on transport and scheme of gw */
+        const gw = mapGateways.get(uri);
+        if (gw) {
+          const {scheme, transport} = gw;
+          this.logger.info({gw}, `sending INVITE to ${uri} via carrier ${gw.name}`);
+          hdrs = {
+            ...hdrs,
+            From: gw.register_from_domain ?
+              createBLegFromHeader({
+                logger: this.logger,
+                req: this.req,
+                register_from_domain: gw.register_from_domain,
+                scheme,
+                transport,
+                ...(private_network && {host: this.privateSipAddress})
+              }) :
+              createBLegFromHeader({
+                logger: this.logger,
+                req: this.req,
+                scheme,
+                transport,
+                ...(private_network && {host: this.privateSipAddress})
+              }),
+            Contact: createBLegFromHeader({
+              logger: this.logger,
+              req: this.req,
+              ...(gw.auth?.username && {fromUser: gw.auth?.username}),
+              scheme,
+              transport,
+              ...(private_network && {host: this.privateSipAddress})
+            }),
+            ...(gw.diversion && {
+              Diversion: gw.diversion.startsWith('+') ?
+                `<sip:${gw.diversion}@${gw.hostport}>;reason=unknown;counter=1;privacy=off` :
+                `<sip:+${gw.diversion}@${gw.hostport}>;reason=unknown;counter=1;privacy=off`
+            })
+          };
+        }
+        else if (teams) {
+          hdrs = {
+            ...hdrs,
+            'From': createBLegFromHeader({logger: this.logger, req: this.req, teams: true, transport: 'tls'}),
+            'Contact': `sip:${this.req.calledNumber}@${this.req.get('X-MS-Teams-Tenant-FQDN')}:5061;transport=tls`
+          };
+        }
+        else {
+          hdrs = {
+            ...hdrs,
+            'From': createBLegFromHeader({
+              logger: this.logger,
+              req: this.req,
+              ...(private_network && {host: this.privateSipAddress})
+            }),
+            'Contact': createBLegFromHeader({
+              logger: this.logger,
+              req: this.req,
+              ...(private_network && {host: this.privateSipAddress})
+            })
+          };
+          const p = proxy ? ` via ${proxy}` : '';
+          this.logger.info({uri, p}, `sending INVITE to ${uri}${p}`);
+        }
+
+        /* now launch an outbound call attempt */
+        const passFailure = 0 === uris.length;  // only propagate failure on last attempt
+        try {
           const {uas, uac} = await this.srf.createB2BUA(this.req, this.res, uri, {
             proxy,
             passFailure,
@@ -450,8 +531,12 @@ class CallSession extends Emitter {
               '-Allow',
               '-Session-Expires'
             ],
+            // Add X-CID header to feature server response.
+            // to allow calling/status hooks contains sbc_callid.
+            responseHeaders: (uacRes, headers) => {
+              headers['X-CID'] = uacRes.get('Call-ID');
+            },
             headers: hdrs,
-            responseHeaders,
             auth: gw ? gw.auth : undefined,
             localSdpB: response.sdp,
             localSdpA: async(sdp, res) => {
@@ -504,6 +589,9 @@ class CallSession extends Emitter {
               } catch (err) {
                 this.logger.error({err}, 'Error saving Call-ID/CSeq');
               }
+
+              this.contactHeader = inv.get('Contact');
+              this.logger.info(`outbound call attempt to ${uri} has contact header ${this.contactHeader}`);
             },
             cbProvisional: (response) => {
               if (!earlyMedia && [180, 183].includes(response.status) && response.body) earlyMedia = true;
@@ -523,13 +611,24 @@ class CallSession extends Emitter {
           // these are all final failure scenarios
           if (uris.length === 0 ||          // exhausted all targets
             earlyMedia ||                   // failure after early media
+            encryptedMedia ||               // cant crank back when using encrypted media as keys have been exchanged
             !(err instanceof SipError) ||   // unexpected error
             err.status === 487) {           // caller hung up
 
             const abandoned = err.message && err.message.includes('rtpengine failed: Unknown call-id');
             const status = err.status || (abandoned ? 487 : 500);
             if (err instanceof SipError) this.logger.info(`final call failure  ${status}`);
-            else if (!abandoned) this.logger.error(err, 'unexpected call failure');
+            else if (!abandoned) {
+              const {writeCallCount, writeCallCountSP, writeCallCountApp} = this.req.srf.locals;
+              this.logger.error({err}, 'unexpected call failure');
+              nudgeCallCounts(this.req, 'failure', {
+                service_provider_sid: this.service_provider_sid,
+                account_sid: this.account_sid,
+                application_sid: this.application_sid,
+                callId: this.req.locals.callId
+              }, this.decrKey, {writeCallCountSP, writeCallCount, writeCallCountApp})
+                .catch((err) => this.logger.error(err, 'Error decrementing call counts'));
+            }
             debug(`got final outdial error: ${err}`);
             if (!passFailure) this.res.send(status);
             this.emit('failed');
@@ -554,7 +653,7 @@ class CallSession extends Emitter {
         }
       }
     } catch (err) {
-      if ('abandonded' !== err.message) this.logger.error(err, `Error setting up outbound call to: ${uris}`);
+      if ('abandonded' !== err.message) this.logger.error({err}, `Error setting up outbound call to: ${uris}`);
       this.emit('failed');
       this.srf.endSession(this.req);
       this.rtpEngineResource.destroy();
@@ -573,6 +672,15 @@ class CallSession extends Emitter {
         answered_at: callStart
       };
     }
+
+    /* save far end SDP for later use if we do a full media release */
+    if (process.env.JAMBONES_ENABLE_FULL_MEDIA_RELEASE) {
+      const key = makeFullMediaReleaseKey(this.req.get('X-CID'));
+      const sdp = uac.remote.sdp;
+      this.logger.info({key, sdp}, 'saving far end sdp for full media release feature');
+      this.addKey(key, sdp, 3600).catch((err) => this.logger.error(err, 'Error saving far end sdp'));
+    }
+
     this.uas = uas;
     this.uac = uac;
     [uas, uac].forEach((dlg) => {
@@ -588,7 +696,6 @@ class CallSession extends Emitter {
             if (!IMMUTABLE_HEADERS.includes(h)) headers[h] = bye.headers[h];
           });
           await other.destroy({headers});
-          await other.destroy();
         } catch (err) {}
 
         const trackingOn = process.env.JAMBONES_TRACK_ACCOUNT_CALLS ||
@@ -597,10 +704,12 @@ class CallSession extends Emitter {
 
         if (process.env.JAMBONES_HOSTING || trackingOn) {
           const {writeCallCount, writeCallCountSP, writeCallCountApp} = this.req.srf.locals;
-          await nudgeCallCounts(this.logger, {
+          this.logger.debug('decrementing call count at end of call');
+          await nudgeCallCounts(this.req, 'complete', {
             service_provider_sid: this.service_provider_sid,
             account_sid: this.account_sid,
-            application_sid: this.application_sid
+            application_sid: this.application_sid,
+            callId: this.req.locals.callId
           }, this.decrKey, {writeCallCountSP, writeCallCount, writeCallCountApp})
             .catch((err) => this.logger.error(err, 'Error decrementing call counts'));
         }
@@ -679,12 +788,19 @@ class CallSession extends Emitter {
   async _onRefer(dlg, req, res) {
     /* REFER coming in from a sip device, forward to feature server */
     try {
+      const customHeaders = Object.keys(req.headers)
+        .filter((h) => h.toLowerCase().startsWith('x-'))
+        .reduce((acc, h) => {
+          acc[h] = req.get(h);
+          return acc;
+        }, {});
       const response = await dlg.other.request({
         method: 'REFER',
         headers: {
           'Refer-To': req.get('Refer-To'),
           'Referred-By': req.get('Referred-By'),
-          'User-Agent': req.get('User-Agent')
+          'User-Agent': req.get('User-Agent'),
+          ...customHeaders
         }
       });
       res.send(response.status, response.reason);
@@ -725,16 +841,44 @@ Duration=${payload.duration} `
     try {
       const reason = req.get('X-Reason');
       const isReleasingMedia = reason && dlg.type === 'uas' && ['release-media', 'anchor-media'].includes(reason);
+      const isFullMediaRelease = reason === 'release-media-entirely' && process.env.JAMBONES_ENABLE_FULL_MEDIA_RELEASE;
       const fromTag = dlg.type === 'uas' ? this.rtpEngineOpts.uas.tag : this.rtpEngineOpts.uac.tag;
       const toTag = dlg.type === 'uas' ? this.rtpEngineOpts.uac.tag : this.rtpEngineOpts.uas.tag;
-      const offerMedia = dlg.type === 'uas' ? this.rtpEngineOpts.uac.mediaOpts : this.rtpEngineOpts.uas.mediaOpts;
-      const answerMedia = dlg.type === 'uas' ? this.rtpEngineOpts.uas.mediaOpts : this.rtpEngineOpts.uac.mediaOpts;
+      let offerMedia = dlg.type === 'uas' ? this.rtpEngineOpts.uac.mediaOpts : this.rtpEngineOpts.uas.mediaOpts;
+      let answerMedia = dlg.type === 'uas' ? this.rtpEngineOpts.uas.mediaOpts : this.rtpEngineOpts.uac.mediaOpts;
       const direction =  dlg.type === 'uas' ? ['private', 'public'] : ['public', 'private'];
+
+      if (isFullMediaRelease) {
+        const a_sdp = await this.retrieveKey(makePartnerFullMediaReleaseKey(this.req.get('X-CID')));
+        this.logger.info({a_sdp}, 'reinvite ourselves out of the media path with this reinvite offer');
+        const answerSdp = await dlg.other.modify(a_sdp);
+        this.logger.info({answerSdp}, 'far end response to full media release');
+        res.send(200, {
+          body: dlg.local.sdp,
+          headers: {
+            'Contact': this.contactHeader
+          }
+        });
+        /* no media going through us now we can destroy the rtpengine resource */
+        this.rtpEngineResource.destroy().catch((err) => {
+          this.logger.info({err}, 'Error destroying rtpengine resource after full media release');
+        });
+        this._mediaPath = MediaPath.NoMedia;
+        return;
+      }
+
       if (isReleasingMedia) {
         if (!offerMedia.flags.includes('port latching')) offerMedia.flags.push('port latching');
         if (!offerMedia.flags.includes('asymmetric')) offerMedia.flags.push('asymmetric');
         offerMedia.flags = offerMedia.flags.filter((f) => f !== 'media handover');
       }
+      const acceptCodecs = process.env.JAMBONES_ACCEPT_AND_TRANSCODE ?
+        process.env.JAMBONES_ACCEPT_AND_TRANSCODE :
+        process.env.JAMBONES_ACCEPT_G729 ? 'g729' : '';
+      offerMedia = { ...offerMedia, ...(acceptCodecs && dlg.type === 'uac' &&
+        { codec: { mask: acceptCodecs, transcode: 'pcmu,pcma' } })};
+      answerMedia = { ...answerMedia, ...(process.env.JAMBONES_CODEC_OFFER_WITH_ORDER && dlg.type === 'uac' &&
+        { codec: {offer: process.env.JAMBONES_CODEC_OFFER_WITH_ORDER.split(','), strip: 'all' }})};
       let opts = {
         ...this.rtpEngineOpts.common,
         ...offerMedia,
@@ -755,17 +899,20 @@ Duration=${payload.duration} `
 
       /* if this is a re-invite from the FS to change media anchoring, avoid sending the reinvite out */
       let sdp;
-      if (isReleasingMedia && !this.calleeIsUsingSrtp) {
+      //HL 2024-11-13: previously forwarded re-invites to webrtc clients but further testing has shown to be unnecessary
+      //if (isReleasingMedia && !this.calleeIsUsingSrtp) {
+
+      //DH 2024-11- 18: if we are going from no-media to either partial or full media, we need reinvite the far end
+      if (isReleasingMedia && this._mediaPath !== MediaPath.NoMedia) {
         this.logger.info(`got a reinvite from FS to ${reason}`);
         sdp = dlg.other.remote.sdp;
         if (!answerMedia.flags.includes('port latching')) answerMedia.flags.push('port latching');
         if (!answerMedia.flags.includes('asymmetric')) answerMedia.flags.push('asymmetric');
         answerMedia.flags = answerMedia.flags.filter((f) => f !== 'media handover');
-        this._mediaReleased = 'release-media' === reason;
+        this._mediaPath = 'release-media' === reason ? MediaPath.PartialMedia : MediaPath.FullMedia;
       }
       else {
         sdp = await dlg.other.modify(response.sdp);
-        this.logger.info({sdp}, 'CallSession:_onReinvite: got sdp from 200 OK to invite we sent');
       }
       opts = {
         ...this.rtpEngineOpts.common,
@@ -775,12 +922,19 @@ Duration=${payload.duration} `
         sdp
       };
       response = await this.answer(opts);
+      /* now remove asymmetric as B party (looking at you Genesys ring group) may need port re-learning on invites  */
+      answerMedia.flags = answerMedia.flags.filter((f) => f !== 'asymmetric');
       if ('ok' !== response.result) {
         res.send(488);
         throw new Error(`_onReinvite: rtpengine failed: ${JSON.stringify(response)}`);
       }
       this.logger.debug({opts, sdp: response.sdp}, 'CallSession:_onReinvite: (answer) sending back upstream');
-      res.send(200, {body: response.sdp});
+      res.send(200, {
+        body: response.sdp,
+        headers: {
+          'Contact': this.contactHeader
+        }
+      });
     } catch (err) {
       res.send(err.status || 500);
       this.logger.error(err, 'Error handling reinvite');
@@ -860,8 +1014,6 @@ Duration=${payload.duration} `
             try {
               this._startRecordingNoAnswerTimer(res);
               await Promise.any(this.srsClients.map((c) => c.start()));
-              // Only take who success accept request.
-              this.srsClients = this.srsClients.filter((c) => c.activated);
               succeeded = true;
             } catch (err) {
               this.logger.error({err}, 'Error starting SipRec call recording');
@@ -869,14 +1021,18 @@ Duration=${payload.duration} `
             }
           }
           else if (reason === 'stopCallRecording') {
-            if (!this.srsClients.length) {
+            if (!this.srsClients.length || !this.srsClients.some((c) => c.activated)) {
               res.send(400);
               this.logger.info('discarding stopCallRecording request because we are not recording');
               return;
             }
             try {
               this._startRecordingNoAnswerTimer(res);
-              await Promise.any(this.srsClients.map((c) => c.stop()));
+              await Promise.any(this.srsClients.map((c) => {
+                if (c.activated) {
+                  c.stop();
+                }
+              }));
               succeeded = true;
             } catch (err) {
               this.logger.error({err}, 'Error stopping SipRec call recording');
@@ -885,14 +1041,18 @@ Duration=${payload.duration} `
             this.srsClients = [];
           }
           else if (reason === 'pauseCallRecording') {
-            if (!this.srsClients.length || this.srsClients.every((c) => c.paused)) {
+            if (!this.srsClients.length || !this.srsClients.some((c) => c.activated && !c.paused)) {
               this.logger.info('discarding invalid pauseCallRecording request');
               res.send(400);
               return;
             }
             try {
               this._startRecordingNoAnswerTimer(res);
-              await Promise.any(this.srsClients.map((c) => c.pause({headers})));
+              await Promise.any(this.srsClients.map((c) => {
+                if (c.activated && !c.paused) {
+                  c.pause({headers});
+                }
+              }));
               succeeded = true;
             } catch (err) {
               this.logger.error({err}, 'Error pausing SipRec call recording');
@@ -900,14 +1060,18 @@ Duration=${payload.duration} `
             }
           }
           else if (reason === 'resumeCallRecording') {
-            if (!this.srsClients.length || !this.srsClients.every((c) => c.paused)) {
+            if (!this.srsClients.length || !this.srsClients.some((c) => c.activated && c.paused)) {
               res.send(400);
               this.logger.info('discarding invalid resumeCallRecording request');
               return;
             }
             try {
               this._startRecordingNoAnswerTimer(res);
-              await Promise.any(this.srsClients.map((c) => c.resume({headers})));
+              await Promise.any(this.srsClients.map((c) => {
+                if (c.activated && c.paused) {
+                  c.resume({headers});
+                }
+              }));
               succeeded = true;
             } catch (err) {
               this.logger.error({err}, 'Error resuming SipRec call recording');
@@ -997,7 +1161,7 @@ Duration=${payload.duration} `
       // eslint-disable-next-line no-unused-vars
       const { via, from, to, 'call-id': callid, cseq, 'max-forwards': maxforwards,
       // eslint-disable-next-line no-unused-vars
-        'content-length': _contentlength, 'refer-to': _referto, 'referred-by': _referredby,
+        'content-length': _contentlength, 'refer-to': _referto, 'referred-by': referredby,
         // eslint-disable-next-line no-unused-vars
         'X-Refer-To-Leave-Untouched': _leave,
         ...customHeaders
@@ -1009,18 +1173,16 @@ Duration=${payload.duration} `
       const arr = /context-(.*)/.exec(uri.user);
       if (!arr) {
         /* call transfer requested */
-        const referredBy = req.getParsedHeader('Referred-By');
-        if (!referredBy) return res.send(400);
-        const u = parseUri(referredBy.uri);
-        const farEnd = parseUri(this.connectedUri);
-        uri.host = farEnd.host;
-        uri.port = farEnd.port;
+        if (!req.has('Referred-By')) return res.send(400);
+
+        /* delete contact if it was there from feature server */
+        delete customHeaders['contact'];
 
         const response = await this.uac.request({
           method: 'REFER',
           headers: {
-            'Refer-To': stringifyUri(uri),
-            'Referred-By': stringifyUri(u),
+            'Refer-To': `<${stringifyUri(uri)}>`,
+            'Referred-By': referredby,
             ...customHeaders
           }
         });

lib/constants.json

@@ -0,0 +1,7 @@
+{
+  "MediaPath": {
+  "NoMedia": "no-media",
+  "PartialMedia": "partial-media",
+  "FullMedia": "full-media"
+  }
+}

lib/middleware.js

@@ -17,6 +17,8 @@ module.exports = (srf, logger, redisClient) => {
   const initLocals = async(req, res, next) => {
     req.locals = req.locals || {};
     const callId = req.get('Call-ID');
+    req.locals.nudge = 0;
+    req.locals.callId = callId;
     req.locals.account_sid = req.get('X-Account-Sid');
     req.locals.application_sid = req.get('X-Application-Sid');
     req.locals.record_all_calls = req.get('X-Record-All-Calls');
@@ -77,7 +79,11 @@ module.exports = (srf, logger, redisClient) => {
     });
 
     try {
-      req.locals.account = await lookupAccountBySid(req.locals.account_sid);
+      const account = await lookupAccountBySid(req.locals.account_sid);
+      req.locals.account = account;
+      if (account.enable_debug_log) {
+        req.locals.logger.level = 'debug';
+      }
       req.locals.service_provider_sid = req.locals.account.service_provider_sid;
     } catch (err) {
       req.locals.logger.error({err}, `Error looking up account sid ${req.locals.account_sid}`);
@@ -103,10 +109,11 @@ module.exports = (srf, logger, redisClient) => {
       /* decrement count if INVITE is later rejected */
       res.once('end', async({status}) => {
         if (status > 200) {
-          nudgeCallCounts(logger, {
+          nudgeCallCounts(req, 'failure', {
             service_provider_sid,
             account_sid,
-            application_sid
+            application_sid,
+            callId: req.locals.callId
           }, decrKey, {writeCallCountSP, writeCallCount, writeCallCountApp})
             .catch((err) => logger.error(err, 'Error decrementing call counts'));
           const tags = ['accepted:no', `sipStatus:${status}`];
@@ -119,10 +126,11 @@ module.exports = (srf, logger, redisClient) => {
       });
 
       /* increment the call count */
-      const  {callsSP, calls} = await nudgeCallCounts(logger, {
+      const  {callsSP, calls} = await nudgeCallCounts(req, 'init', {
         service_provider_sid,
         account_sid,
-        application_sid
+        application_sid,
+        callId: req.locals.callId
       }, incrKey, {writeCallCountSP, writeCallCount, writeCallCountApp});
 
       /* compare to account's limit, though avoid db hit when call count is low */
@@ -223,7 +231,10 @@ module.exports = (srf, logger, redisClient) => {
           /* redirect to the correct SBC where this user is connected */
           const proxyAddress = selectHostPort(reg.sbcAddress, 'tcp');
           const redirectUri =  `<sip:${proxyAddress[1]}>`;
-          logger.info(`redirecting call to SBC at ${redirectUri}`);
+          logger.info({
+            myHostPort: req.server.hostport,
+            registeredHostPort: reg.sbcAddress,
+          }, `redirecting call to SBC at ${redirectUri}`);
           return res.send(302, {headers: {Contact: redirectUri}});
         }
         req.locals.registration = reg;

lib/utils.js

@@ -4,9 +4,6 @@ const debug = require('debug')('jambonz:sbc-outbound');
 const CIDRMatcher = require('cidr-matcher');
 const dns = require('dns');
 
-const cidrMatcher = process.env.PRIVATE_VOIP_NETWORK_CIDR ?
-  new CIDRMatcher(process.env.PRIVATE_VOIP_NETWORK_CIDR.split(',')) : null;
-
 function makeRtpEngineOpts(req, srcIsUsingSrtp, dstIsUsingSrtp, padCrypto, teams) {
   const from = req.getParsedHeader('from');
   const rtpCopy = JSON.parse(JSON.stringify(rtpCharacteristics));
@@ -43,7 +40,11 @@ function makeRtpEngineOpts(req, srcIsUsingSrtp, dstIsUsingSrtp, padCrypto, teams
     },
     uac: {
       tag: null,
-      mediaOpts: dstOpts
+      mediaOpts: {
+        ...dstOpts,
+        ...(process.env.JAMBONES_CODEC_OFFER_WITH_ORDER &&
+          { codec: { offer: process.env.JAMBONES_CODEC_OFFER_WITH_ORDER.split(','), strip: 'all' } }),
+      }
     }
   };
 }
@@ -130,13 +131,96 @@ const createHealthCheckApp = (port, logger) => {
   });
 };
 
-const nudgeCallCounts = async(logger, sids, nudgeOperator, writers) => {
-  const {service_provider_sid, account_sid, application_sid} = sids;
+/**
+ * nudgeCallCounts - increment or decrement call counts in redis
+ *
+ *              current nudge value
+ * -----------------------------------------
+ *  why    |     -1     |    0    |    1    |
+ * -----------------------------------------
+ *  init   |    no-op   |    +1   |    N/A  |
+ * failure |    N/A     |    -1   |    -1   |
+ * complete|    N/A     |    N/A  |    -1   |
+ *
+ *
+ */
+
+const shouldNudge = (why, req) => {
+  const {nudge, logger} = req.locals;
+  let modifyCount = false;
+  const originalNudge = nudge;
+
+  switch (why) {
+    case 'init':
+      if (nudge === 0) {
+        // normal case: new call, increment call count
+        req.locals.nudge = 1;
+        modifyCount = true;
+      }
+      else if (nudge === -1) {
+        // extremely quick cancel, don't increment call count
+        req.locals.nudge = 0;
+      }
+      else {
+        logger.info(`shouldNudge: unexpected nudge value ${nudge} for ${why}`);
+      }
+      break;
+    case 'failure':
+      if (nudge === 1) {
+        // normal case of call failed for any reason, decrement call count
+        req.locals.nudge = 0;
+        modifyCount = true;
+      }
+      else if (nudge === 0) {
+        // very quick failure dont decrement call count
+        req.locals.nudge = -1;
+      }
+      else {
+        logger.info(`shouldNudge: unexpected nudge value ${nudge} for ${why}`);
+      }
+      break;
+    case 'complete':
+      if (nudge === 1) {
+        // normal case of call completed, decrement call count
+        req.locals.nudge = 0;
+        modifyCount = true;
+      }
+      else {
+        logger.info(`shouldNudge: unexpected nudge value ${nudge} for ${why}`);
+      }
+      break;
+    default:
+      logger.info(`shouldNudge: unexpected why value ${why}`);
+      break;
+  }
+
+  logger.info(`shouldNudge: '${why}': updating count: ${modifyCount}, nudge: ${originalNudge} -> ${req.locals.nudge}`);
+  return modifyCount;
+};
+
+const nudgeCallCounts = async(req, why, sids, nudgeOperator, writers) => {
+  const {logger} = req.locals;
+  const {service_provider_sid, account_sid, application_sid, callId} = sids;
   const {writeCallCount, writeCallCountSP, writeCallCountApp} = writers;
   const nudges = [];
   const writes = [];
 
-  logger.debug(sids, 'nudgeCallCounts');
+  if (!shouldNudge(why, req)) {
+    return {callsSP: null, calls: null, callsApp: null};
+  }
+
+  if (process.env.JAMBONES_DEBUG_CALL_COUNTS) {
+    const {srf} = require('..');
+    const {addKey, deleteKey} = srf.locals.realtimeDbHelpers;
+
+    if (why === 'init') {
+      // save for 3 days
+      await addKey(`debug:outcalls:${account_sid}:${callId}`, new Date().toISOString(), 259200);
+    }
+    else {
+      await deleteKey(`debug:outcalls:${account_sid}:${callId}`);
+    }
+  }
 
   if (process.env.JAMBONES_TRACK_SP_CALLS) {
     const key = makeSPCallCountKey(service_provider_sid);
@@ -191,8 +275,12 @@ const nudgeCallCounts = async(logger, sids, nudgeOperator, writers) => {
 };
 
 const isPrivateVoipNetwork = async(uri) => {
-  if (cidrMatcher) {
+  const {srf, logger} = require('..');
+  const {privateNetworkCidr} = srf.locals;
+
+  if (privateNetworkCidr) {
     try {
+      const matcher = new CIDRMatcher(privateNetworkCidr.split(','));
       const arr = /sips?:.*@(.*?)(:\d+)?(;.*)$/.exec(uri);
       if (arr) {
         const input = arr[1];
@@ -203,16 +291,38 @@ const isPrivateVoipNetwork = async(uri) => {
           addresses = await dns.resolve4(input);
         }
         for (const ip of addresses) {
-          if (cidrMatcher.contains(ip)) {
+          if (matcher.contains(ip)) {
             return true;
           }
         }
       }
-    } catch (err) {}
+    } catch (err) {
+      logger.info({err, privateNetworkCidr},
+        'Error checking private network CIDR, probably misconfigured must be a comma separated list of CIDRs');
+    }
   }
   return false;
 };
 
+function makeBlacklistGatewayKey(key) {
+  return `blacklist-sip-gateway:${key}`;
+}
+
+async function isBlackListedSipGateway(client, logger, sip_gateway_sid) {
+  try {
+    return await client.exists(makeBlacklistGatewayKey(sip_gateway_sid)) === 1;
+  } catch (err) {
+    logger.error({err}, `isBlackListedSipGateway: error while checking blacklist for ${sip_gateway_sid}`);
+  }
+}
+
+const makeFullMediaReleaseKey = (callId) => {
+  return `b_sdp:${callId}`;
+};
+const makePartnerFullMediaReleaseKey = (callId) => {
+  return `a_sdp:${callId}`;
+};
+
 module.exports = {
   makeRtpEngineOpts,
   selectHostPort,
@@ -223,5 +333,8 @@ module.exports = {
   systemHealth,
   createHealthCheckApp,
   nudgeCallCounts,
-  isPrivateVoipNetwork
+  isPrivateVoipNetwork,
+  isBlackListedSipGateway,
+  makeFullMediaReleaseKey,
+  makePartnerFullMediaReleaseKey
 };

package.json

@@ -1,9 +1,9 @@
 {
   "name": "sbc-outbound",
-  "version": "0.8.5",
+  "version": "0.9.3",
   "main": "app.js",
   "engines": {
-    "node": ">= 12.0.0"
+    "node": ">= 18.0.0"
   },
   "keywords": [
     "sip",
@@ -27,27 +27,27 @@
     "jslint": "eslint app.js lib --fix"    
   },
   "dependencies": {
-    "@jambonz/db-helpers": "^0.9.3",
-    "@jambonz/realtimedb-helpers": "^0.8.7",
+    "@jambonz/db-helpers": "^0.9.7",
+    "@jambonz/realtimedb-helpers": "^0.8.9",
     "@jambonz/http-health-check": "^0.0.1",
-    "@jambonz/mw-registrar": "0.2.4",
-    "@jambonz/rtpengine-utils": "^0.4.3",
-    "@jambonz/siprec-client-utils": "^0.2.7",
-    "@jambonz/stats-collector": "^0.1.9",
-    "@jambonz/time-series": "^0.2.5",
+    "@jambonz/mw-registrar": "0.2.7",
+    "@jambonz/rtpengine-utils": "^0.4.4",
+    "@jambonz/siprec-client-utils": "^0.2.9",
+    "@jambonz/stats-collector": "^0.1.10",
+    "@jambonz/time-series": "^0.2.10",
     "cidr-matcher": "^2.1.1",
     "debug": "^4.3.4",
-    "drachtio-fn-b2b-sugar": "^0.0.12",
-    "drachtio-srf": "^4.5.31",
-    "express": "^4.18.1",
-    "pino": "^7.11.0",
-    "sdp-transform": "^2.14.1"
+    "drachtio-fn-b2b-sugar": "^0.2.1",
+    "drachtio-srf": "^5.0.1",
+    "express": "^4.19.2",
+    "pino": "^8.20.0",
+    "sdp-transform": "^2.15.0"
   },
   "devDependencies": {
     "bent": "^7.3.12",
     "eslint": "^7.32.0",
-    "eslint-plugin-promise": "^5.2.0",
+    "eslint-plugin-promise": "^6.1.1",
     "nyc": "^15.1.0",
-    "tape": "^5.5.3"
+    "tape": "^5.7.5"
   }
 }

test/db/jambones-sql.sql

@@ -1,4 +1,5 @@
 /* SQLEditor (MySQL (2))*/
+
 SET FOREIGN_KEY_CHECKS=0;
 
 DROP TABLE IF EXISTS account_static_ips;
@@ -13,6 +14,8 @@ DROP TABLE IF EXISTS beta_invite_codes;
 
 DROP TABLE IF EXISTS call_routes;
 
+DROP TABLE IF EXISTS clients;
+
 DROP TABLE IF EXISTS dns_records;
 
 DROP TABLE IF EXISTS lcr;
@@ -51,6 +54,8 @@ DROP TABLE IF EXISTS signup_history;
 
 DROP TABLE IF EXISTS smpp_addresses;
 
+DROP TABLE IF EXISTS google_custom_voices;
+
 DROP TABLE IF EXISTS speech_credentials;
 
 DROP TABLE IF EXISTS system_information;
@@ -127,6 +132,19 @@ application_sid CHAR(36) NOT NULL,
 PRIMARY KEY (call_route_sid)
 ) COMMENT='a regex-based pattern match for call routing';
 
+CREATE TABLE clients
+(
+client_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+is_active BOOLEAN NOT NULL DEFAULT 1,
+username VARCHAR(64),
+password VARCHAR(1024),
+allow_direct_app_calling BOOLEAN NOT NULL DEFAULT 1,
+allow_direct_queue_calling BOOLEAN NOT NULL DEFAULT 1,
+allow_direct_user_calling BOOLEAN NOT NULL DEFAULT 1,
+PRIMARY KEY (client_sid)
+);
+
 CREATE TABLE dns_records
 (
 dns_record_sid CHAR(36) NOT NULL UNIQUE ,
@@ -322,14 +340,27 @@ last_tested DATETIME,
 tts_tested_ok BOOLEAN,
 stt_tested_ok BOOLEAN,
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+label VARCHAR(64),
 PRIMARY KEY (speech_credential_sid)
 );
 
+CREATE TABLE google_custom_voices
+(
+google_custom_voice_sid CHAR(36) NOT NULL UNIQUE ,
+speech_credential_sid CHAR(36) NOT NULL,
+model VARCHAR(512) NOT NULL,
+reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
+name VARCHAR(64) NOT NULL,
+PRIMARY KEY (google_custom_voice_sid)
+);
+
 CREATE TABLE system_information
 (
 domain_name VARCHAR(255),
 sip_domain_name VARCHAR(255),
-monitoring_domain_name VARCHAR(255)
+monitoring_domain_name VARCHAR(255),
+private_network_cidr VARCHAR(8192),
+log_level ENUM('info', 'debug') NOT NULL DEFAULT 'info'
 );
 
 CREATE TABLE users
@@ -411,7 +442,7 @@ PRIMARY KEY (smpp_gateway_sid)
 CREATE TABLE phone_numbers
 (
 phone_number_sid CHAR(36) UNIQUE ,
-number VARCHAR(132) NOT NULL UNIQUE ,
+number VARCHAR(132) NOT NULL,
 voip_carrier_sid CHAR(36),
 account_sid CHAR(36),
 application_sid CHAR(36),
@@ -424,11 +455,14 @@ CREATE TABLE sip_gateways
 sip_gateway_sid CHAR(36),
 ipv4 VARCHAR(128) NOT NULL COMMENT 'ip address or DNS name of the gateway.  For gateways providing inbound calling service, ip address is required.',
 netmask INTEGER NOT NULL DEFAULT 32,
-port INTEGER NOT NULL DEFAULT 5060 COMMENT 'sip signaling port',
+port INTEGER COMMENT 'sip signaling port',
 inbound BOOLEAN NOT NULL COMMENT 'if true, whitelist this IP to allow inbound calls from the gateway',
 outbound BOOLEAN NOT NULL COMMENT 'if true, include in least-cost routing when placing calls to the PSTN',
 voip_carrier_sid CHAR(36) NOT NULL,
 is_active BOOLEAN NOT NULL DEFAULT 1,
+send_options_ping BOOLEAN NOT NULL DEFAULT 0,
+use_sips_scheme BOOLEAN NOT NULL DEFAULT 0,
+pad_crypto BOOLEAN NOT NULL DEFAULT 0,
 protocol ENUM('udp','tcp','tls', 'tls/srtp') DEFAULT 'udp' COMMENT 'Outbound call protocol',
 PRIMARY KEY (sip_gateway_sid)
 ) COMMENT='A whitelisted sip gateway used for origination/termination';
@@ -465,10 +499,21 @@ messaging_hook_sid CHAR(36) COMMENT 'webhook to call for inbound SMS/MMS ',
 app_json TEXT,
 speech_synthesis_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_synthesis_language VARCHAR(12) NOT NULL DEFAULT 'en-US',
-speech_synthesis_voice VARCHAR(64),
+speech_synthesis_voice VARCHAR(256),
+speech_synthesis_label VARCHAR(64),
 speech_recognizer_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_recognizer_language VARCHAR(64) NOT NULL DEFAULT 'en-US',
+speech_recognizer_label VARCHAR(64),
+use_for_fallback_speech BOOLEAN DEFAULT false,
+fallback_speech_synthesis_vendor VARCHAR(64),
+fallback_speech_synthesis_language VARCHAR(12),
+fallback_speech_synthesis_voice VARCHAR(256),
+fallback_speech_synthesis_label VARCHAR(64),
+fallback_speech_recognizer_vendor VARCHAR(64),
+fallback_speech_recognizer_language VARCHAR(64),
+fallback_speech_recognizer_label VARCHAR(64),
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
 PRIMARY KEY (application_sid)
 ) COMMENT='A defined set of behaviors to be applied to phone calls ';
 
@@ -506,6 +551,10 @@ subspace_client_secret VARCHAR(255),
 subspace_sip_teleport_id VARCHAR(255),
 subspace_sip_teleport_destinations VARCHAR(255),
 siprec_hook_sid CHAR(36),
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
+record_format VARCHAR(16) NOT NULL DEFAULT 'mp3',
+bucket_credential VARCHAR(8192) COMMENT 'credential used to authenticate with storage service',
+enable_debug_log BOOLEAN NOT NULL DEFAULT false,
 PRIMARY KEY (account_sid)
 ) COMMENT='An enterprise that uses the platform for comm services';
 
@@ -526,6 +575,9 @@ ALTER TABLE call_routes ADD FOREIGN KEY account_sid_idxfk_3 (account_sid) REFERE
 
 ALTER TABLE call_routes ADD FOREIGN KEY application_sid_idxfk (application_sid) REFERENCES applications (application_sid);
 
+CREATE INDEX client_sid_idx ON clients (client_sid);
+ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid);
+
 CREATE INDEX dns_record_sid_idx ON dns_records (dns_record_sid);
 ALTER TABLE dns_records ADD FOREIGN KEY account_sid_idxfk_4 (account_sid) REFERENCES accounts (account_sid);
 
@@ -590,8 +642,6 @@ CREATE INDEX smpp_address_sid_idx ON smpp_addresses (smpp_address_sid);
 CREATE INDEX service_provider_sid_idx ON smpp_addresses (service_provider_sid);
 ALTER TABLE smpp_addresses ADD FOREIGN KEY service_provider_sid_idxfk_4 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
-CREATE UNIQUE INDEX speech_credentials_idx_1 ON speech_credentials (vendor,account_sid);
-
 CREATE INDEX speech_credential_sid_idx ON speech_credentials (speech_credential_sid);
 CREATE INDEX service_provider_sid_idx ON speech_credentials (service_provider_sid);
 ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
@@ -599,6 +649,10 @@ ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (ser
 CREATE INDEX account_sid_idx ON speech_credentials (account_sid);
 ALTER TABLE speech_credentials ADD FOREIGN KEY account_sid_idxfk_8 (account_sid) REFERENCES accounts (account_sid);
 
+CREATE INDEX google_custom_voice_sid_idx ON google_custom_voices (google_custom_voice_sid);
+CREATE INDEX speech_credential_sid_idx ON google_custom_voices (speech_credential_sid);
+ALTER TABLE google_custom_voices ADD FOREIGN KEY speech_credential_sid_idxfk (speech_credential_sid) REFERENCES speech_credentials (speech_credential_sid) ON DELETE CASCADE;
+
 CREATE INDEX user_sid_idx ON users (user_sid);
 CREATE INDEX email_idx ON users (email);
 CREATE INDEX phone_idx ON users (phone);
@@ -628,6 +682,8 @@ CREATE INDEX smpp_gateway_sid_idx ON smpp_gateways (smpp_gateway_sid);
 CREATE INDEX voip_carrier_sid_idx ON smpp_gateways (voip_carrier_sid);
 ALTER TABLE smpp_gateways ADD FOREIGN KEY voip_carrier_sid_idxfk (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
 
+CREATE UNIQUE INDEX phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid);
+
 CREATE INDEX phone_number_sid_idx ON phone_numbers (phone_number_sid);
 CREATE INDEX number_idx ON phone_numbers (number);
 CREATE INDEX voip_carrier_sid_idx ON phone_numbers (voip_carrier_sid);

test/docker-compose-testbed.yaml

@@ -24,7 +24,7 @@ services:
         ipv4_address: 172.39.0.2
 
   sbc:
-    image: drachtio/drachtio-server:latest
+    image: drachtio/drachtio-server:0.8.26
     command: drachtio --contact "sip:*;transport=udp" --loglevel debug --sofia-loglevel 9
     ports:
       - "9060:9022/tcp"

test/scenarios/uac-pcap-carrier-fail-blacklist.xml

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+<!DOCTYPE scenario SYSTEM "sipp.dtd">
+
+<scenario name="UAC with media">
+
+  <send retrans="500">
+    <![CDATA[
+
+      INVITE sip:16173333456@127.0.0.1 SIP/2.0
+      Via: SIP/2.0/[transport] [local_ip]:[local_port];branch=[branch]
+      From: sipp <sip:sipp@[local_ip]:[local_port]>;tag=[pid]SIPpTag09[call_number]
+      To: <sip:16173333456@127.0.0.1>
+      Call-ID: [call_id]
+      CSeq: 1 INVITE
+      Contact: sip:sipp@[local_ip]:[local_port]
+      Max-Forwards: 70
+      X-Account-Sid: ed649e33-e771-403a-8c99-1780eabbc803
+      X-Call-Sid: ff49e33-e771-403a-8c99-1780eabbc803
+      X-Jambonz-Routing: phone
+      Subject: uac-pcap-carrier-fail-limits
+      Content-Type: application/sdp
+      Content-Length: [len]
+
+      v=0
+      o=user1 53655765 2353687637 IN IP[local_ip_type] [local_ip]
+      s=-
+      c=IN IP[local_ip_type] [local_ip]
+      t=0 0
+      m=audio [auto_media_port] RTP/AVP 8 101
+      a=rtpmap:8 PCMA/8000
+      a=rtpmap:101 telephone-event/8000
+      a=fmtp:101 0-11,16
+
+    ]]>
+  </send>
+
+  <recv response="100" optional="true">
+  </recv>
+
+  <recv response="603" rtd="true" crlf="true">
+  </recv>
+
+  <send>
+    <![CDATA[
+
+      ACK sip:sip:+16173333456@127.0.0.1 SIP/2.0
+      [last_Via]
+      From: sipp <sip:sipp@[local_ip]:[local_port]>;tag=[pid]SIPpTag09[call_number]
+      To: <sip:sip:+16173333456@127.0.0.1>[peer_tag_param]
+      Call-ID: [call_id]
+      CSeq: 1 ACK
+      Subject: uac-pcap-carrier-fail-blacklist
+      Content-Length: 0
+
+    ]]>
+  </send>
+
+
+</scenario>

test/sip-tests.js

@@ -26,6 +26,7 @@ function connect(connectable) {
 test('sbc-outbound tests', async(t) => {
   const {srf} = require('../app');
   const { queryCdrs } = srf.locals;
+  const redisClient = srf.locals.realtimeDbHelpers.client;
 
   try {
     await connect(srf);
@@ -95,7 +96,17 @@ test('sbc-outbound tests', async(t) => {
     /* fails when session limit exceeded */
     await sippUac('uac-pcap-carrier-fail-limits.xml');
     t.pass('fails when max calls in progress');
-      
+
+    // re-rack test data
+    execSync(`mysql -h 127.0.0.1 -u root  --protocol=tcp -D jambones_test < ${__dirname}/db/jambones-sql.sql`);
+    execSync(`mysql -h 127.0.0.1 -u root  --protocol=tcp -D jambones_test < ${__dirname}/db/populate-test-data.sql`);
+
+    // Black list good carrier for some seconds
+    await redisClient.setex('blacklist-sip-gateway:124a5339-c62c-4075-9e19-f4de70a96597', 3, '');
+    await sippUac('uac-pcap-carrier-fail-blacklist.xml');
+    t.pass('fails when carrier is blacklisted');
+    await redisClient.del('blacklist-sip-gateway:124a5339-c62c-4075-9e19-f4de70a96597');
+
     await waitFor(25);
 
     const res = await queryCdrs({account_sid: 'ed649e33-e771-403a-8c99-1780eabbc803'});