Feat/deepgram voice agent (#1006)

* wip

* wip

* wip

.gitignore

@@ -42,4 +42,5 @@ ecosystem.config.js
 test/credentials/*.json
 run-tests.sh
 run-coverage.sh
-.vscode
+.vscode
+.env

app.js

@@ -25,10 +25,22 @@ const opts = {
 };
 const pino = require('pino');
 const logger = pino(opts, pino.destination({sync: false}));
-const {LifeCycleEvents, FS_UUID_SET_NAME} = require('./lib/utils/constants');
+const {LifeCycleEvents, FS_UUID_SET_NAME, SystemState, FEATURE_SERVER} = require('./lib/utils/constants');
 const installSrfLocals = require('./lib/utils/install-srf-locals');
 installSrfLocals(srf, logger);
 
+const writeSystemAlerts = srf.locals?.writeSystemAlerts;
+if (writeSystemAlerts) {
+  writeSystemAlerts({
+    system_component: FEATURE_SERVER,
+    state : SystemState.Online,
+    fields : {
+      detail: `feature-server with process_id ${process.pid} started`,
+      host: srf.locals?.ipv4
+    }
+  });
+}
+
 const {
   initLocals,
   createRootSpan,
@@ -100,12 +112,19 @@ createHttpListener(logger, srf)
   });
 
 
-setInterval(async() => {
+const monInterval = setInterval(async() => {
   srf.locals.stats.gauge('fs.sip.calls.count', sessionTracker.count);
-  // Checking system log level
-  const systemInformation = await srf.locals.dbHelpers.lookupSystemInformation();
-  if (systemInformation && systemInformation.log_level) {
-    logger.level = systemInformation.log_level;
+  try {
+    const systemInformation = await srf.locals.dbHelpers.lookupSystemInformation();
+    if (systemInformation && systemInformation.log_level) {
+      logger.level = systemInformation.log_level;
+    }
+  } catch (err) {
+    if (process.env.NODE_ENV === 'test') {
+      clearInterval(monInterval);
+      logger.error('all tests complete');
+    }
+    else logger.error({err}, 'Error checking system log level in database');
   }
 }, 20000);
 
@@ -117,13 +136,25 @@ const disconnect = () => {
     srf.locals.mediaservers?.forEach((ms) => ms.disconnect());
   });
 };
-
 process.on('SIGTERM', handle);
+process.on('SIGINT', handle);
 
-function handle(signal) {
+async function handle(signal) {
   const {removeFromSet} = srf.locals.dbHelpers;
   srf.locals.disabled = true;
   logger.info(`got signal ${signal}`);
+  const writeSystemAlerts = srf.locals?.writeSystemAlerts;
+  if (writeSystemAlerts) {
+    // it has to be synchronous call, or else by the time system saves the app terminates
+    await writeSystemAlerts({
+      system_component: FEATURE_SERVER,
+      state : SystemState.Offline,
+      fields : {
+        detail: `feature-server with process_id ${process.pid} stopped, signal ${signal}`,
+        host: srf.locals?.ipv4
+      }
+    });
+  }
   const setName = `${(JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
   const fsServiceUrlSetName = `${(JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
   if (setName && srf.locals.localSipAddress) {

lib/http-routes/api/create-call.js

@@ -208,10 +208,13 @@ router.post('/',
 
       /**
      * create our application object -
-     * not from the database as per an inbound call,
-     * but from the provided params in the request
+     * we merge the inbound call application,
+     * with the provided app params from the request body
      */
-      const app = req.body;
+      const app = {
+        ...application,
+        ...req.body
+      };
 
       /**
      * attach our requestor and notifier objects

lib/session/adulting-call-session.js

@@ -45,8 +45,10 @@ class AdultingCallSession extends CallSession {
     return this.sd.ep;
   }
 
-  /* see note above */
-  set ep(newEp) {}
+  // When adulting session kicked from conference, replaceEndpoint is a must
+  set ep(newEp) {
+    this.sd.ep = newEp;
+  }
 
   get callSid() {
     return this.callInfo.callSid;

lib/session/call-session.js

@@ -2,6 +2,7 @@ const Emitter = require('events');
 const fs = require('fs');
 const {
   CallDirection,
+  MediaPath,
   TaskPreconditions,
   CallStatus,
   TaskName,
@@ -29,9 +30,10 @@ const {
 } = require('../config');
 const bent = require('bent');
 const BackgroundTaskManager = require('../utils/background-task-manager');
+const dbUtils = require('../utils/db-utils');
 const BADPRECONDITIONS = 'preconditions not met';
 const CALLER_CANCELLED_ERR_MSG = 'Response not sent due to unknown transaction';
-
+const { NonFatalTaskError} = require('../utils/error');
 const sqlRetrieveQueueEventHook = `SELECT * FROM webhooks 
 WHERE webhook_sid = 
 (
@@ -398,6 +400,13 @@ class CallSession extends Emitter {
     return this.application.transferredCall === true;
   }
 
+  /**
+   * returns true if this session is an inbound call session
+   */
+  get isInboundCallSession() {
+    return this.constructor.name === 'InboundCallSession';
+  }
+
   /**
    * returns true if this session is a ConfirmCallSession
    */
@@ -426,6 +435,10 @@ class CallSession extends Emitter {
     return this.constructor.name === 'SmsCallSession';
   }
 
+  get is3pccInvite() {
+    return this.isInboundCallSession && this.req?.body?.length === 0;
+  }
+
   get webhook_secret() {
     return this.accountInfo?.account?.webhook_secret;
   }
@@ -1040,6 +1053,16 @@ class CallSession extends Emitter {
   async exec() {
     this.logger.info({tasks: listTaskNames(this.tasks)}, `CallSession:exec starting ${this.tasks.length} tasks`);
 
+    // calculate if inbandDTMF tone is used
+    const voip_carrier_sid = this.req?.has('X-Voip-Carrier-Sid') ? this.req.get('X-Voip-Carrier-Sid') :
+      this.req?.has('X-Requested-Carrier-Sid') ? this.req.get('X-Requested-Carrier-Sid') : null;
+
+    if (voip_carrier_sid) {
+      const {lookupVoipCarrierBySid} = dbUtils(this.logger, this.srf);
+      const [voipCarrier] = await lookupVoipCarrierBySid(voip_carrier_sid);
+      this.inbandDtmfEnabled = voipCarrier?.dtmf_type === 'tones';
+    }
+
     while (this.tasks.length && !this.callGone) {
       const taskNum = ++this.taskIdx;
       const stackNum = this.stackIdx;
@@ -1080,6 +1103,8 @@ class CallSession extends Emitter {
         this.currentTask = null;
         if (err.message?.includes(BADPRECONDITIONS)) {
           this.logger.info(`CallSession:exec task #${stackNum}:${taskNum}: ${task.name}: ${err.message}`);
+        } else if (err instanceof NonFatalTaskError) {
+          this.logger.error(err, `Error executing task  #${stackNum}:${taskNum}: ${task.name}`);
         }
         else {
           this.logger.error(err, `Error executing task  #${stackNum}:${taskNum}: ${task.name}`);
@@ -1589,6 +1614,15 @@ Duration=${duration} `
     this.logger.info({response}, '_lccBoostAudioSignal: response from freeswitch');
   }
 
+  async _lccMediaPath(desiredPath) {
+    const task = this.currentTask;
+    if (!task || task.name !== TaskName.Dial) {
+      return this.logger.info('CallSession:_lccMediaPath - invalid command since we are not in a dial verb');
+    }
+    task.updateMediaPath(desiredPath)
+      .catch((err) => this.logger.error(err, 'CallSession:_lccMediaPath'));
+  }
+
   _lccToolOutput(tool_call_id, opts, callSid) {
     // only valid if we are in an LLM verb
     const task = this.currentTask;
@@ -1672,6 +1706,9 @@ Duration=${duration} `
     else if (opts.boostAudioSignal) {
       return this._lccBoostAudioSignal(opts, callSid);
     }
+    else if (opts.media_path) {
+      return this._lccMediaPath(opts.media_path, callSid);
+    }
     else if (opts.llm_tool_output) {
       return this._lccToolOutput(opts.tool_call_id, opts.llm_tool_output, callSid);
     }
@@ -1975,6 +2012,13 @@ Duration=${duration} `
           });
         break;
 
+      case 'media:path':
+        this._lccMediaPath(data, call_sid)
+          .catch((err) => {
+            this.logger.info({err, data}, 'CallSession:_onCommand - error setting media path');
+          });
+        break;
+
       case 'llm:tool-output':
         this._lccToolOutput(tool_call_id, data, call_sid);
         break;
@@ -2040,6 +2084,8 @@ Duration=${duration} `
         };
       }
 
+      this._enableInbandDtmfIfRequired(this.ep);
+
       // we are going from an early media connection to answer
       if (this.direction === CallDirection.Inbound) {
         // only do this for inbound call.
@@ -2063,7 +2109,6 @@ Duration=${duration} `
       //ep.cs = this;
       this.ep = ep;
       this.logger.info(`allocated endpoint ${ep.uuid}`);
-
       this._configMsEndpoint();
 
       this.ep.on('destroy', () => {
@@ -2225,22 +2270,43 @@ Duration=${duration} `
         },
         localSdp: this.ep.local.sdp
       });
-      this.logger.debug('answered call');
-      this.dlg.on('destroy', this._callerHungup.bind(this));
-      this.wrapDialog(this.dlg);
-      this.dlg.callSid = this.callSid;
-      this.emit('callStatusChange', {sipStatus: 200, sipReason: 'OK', callStatus: CallStatus.InProgress});
 
-      if (this.recordOptions && this.recordState === RecordState.RecordingOff) {
-        this.startRecording();
+      const tidyUp = () => {
+        this.dlg.on('destroy', this._callerHungup.bind(this));
+        this.wrapDialog(this.dlg);
+        this.dlg.callSid = this.callSid;
+        this.emit('callStatusChange', {sipStatus: 200, sipReason: 'OK', callStatus: CallStatus.InProgress});
+
+        if (this.recordOptions && this.recordState === RecordState.RecordingOff) {
+          this.startRecording();
+        }
+        this.dlg.on('modify', this._onReinvite.bind(this));
+        this.dlg.on('refer', this._onRefer.bind(this));
+        if (this.sipRequestWithinDialogHook) {
+          this.dlg.on('info', this._onRequestWithinDialog.bind(this));
+          this.dlg.on('message', this._onRequestWithinDialog.bind(this));
+        }
+        this.logger.debug(`CallSession:propagateAnswer - answered callSid ${this.callSid}`);
+      };
+
+      if (this.is3pccInvite) {
+        return new Promise((resolve, reject) => {
+          this.dlg.once('ack', (ackRequest) => {
+            this.logger.debug({body: ackRequest.body}, 'received ACK for 3pcc invite');
+            this.ep.modify(ackRequest.body)
+              .then(() => {
+                tidyUp();
+                return resolve();
+              })
+              .catch((err) => {
+                this.logger.info({err}, 'Error modifying endpoint with ACK');
+                reject(err);
+              });
+          });
+          this.logger.debug('this is a 3pcc invite');
+        });
       }
-      this.dlg.on('modify', this._onReinvite.bind(this));
-      this.dlg.on('refer', this._onRefer.bind(this));
-      if (this.sipRequestWithinDialogHook) {
-        this.dlg.on('info', this._onRequestWithinDialog.bind(this));
-        this.dlg.on('message', this._onRequestWithinDialog.bind(this));
-      }
-      this.logger.debug(`CallSession:propagateAnswer - answered callSid ${this.callSid}`);
+      tidyUp();
     }
     else {
       this.logger.debug('CallSession:propagateAnswer - call already answered - re-anchor media with a reinvite');
@@ -2318,6 +2384,12 @@ Duration=${duration} `
       try {
         const to = parseUri(req.getParsedHeader('Refer-To').uri);
         const by = parseUri(req.getParsedHeader('Referred-By').uri);
+        const customHeaders = Object.keys(req.headers)
+          .filter((h) => h.toLowerCase().startsWith('x-'))
+          .reduce((acc, h) => {
+            acc[h] = req.get(h);
+            return acc;
+          }, {});
         const b3 = this.b3;
         const httpHeaders = b3 && {b3};
         const json = await this.requestor.request('verb:hook', this._referHook, {
@@ -2330,6 +2402,7 @@ Duration=${duration} `
             referred_by_user: by.scheme === 'tel' ? by.number : by.user,
             referring_call_sid: this.callSid,
             referred_call_sid: null,
+            ...customHeaders
           }
         }, httpHeaders);
 
@@ -2505,27 +2578,35 @@ Duration=${duration} `
     };
   }
 
-  async releaseMediaToSBC(remoteSdp) {
+  async releaseMediaToSBC(remoteSdp, releaseMediaEntirely) {
     assert(this.dlg && this.dlg.connected && this.ep && typeof remoteSdp === 'string');
     await this.dlg.modify(remoteSdp, {
       headers: {
-        'X-Reason': 'release-media'
+        'X-Reason': releaseMediaEntirely ? 'release-media-entirely' : 'release-media'
       }
     });
-    this.ep.destroy()
-      .then(() => this.ep = null)
-      .catch((err) => this.logger.error({err}, 'CallSession:releaseMediaToSBC: Error destroying endpoint'));
+    try {
+      await this.ep.destroy();
+    } catch (err) {
+      this.logger.error({err}, 'CallSession:releaseMediaToSBC: Error destroying endpoint');
+    }
+    this.ep = null;
   }
 
-  async reAnchorMedia() {
+  async reAnchorMedia(currentMediaRoute = MediaPath.PartialMedia) {
     assert(this.dlg && this.dlg.connected && !this.ep);
+
     this.ep = await this.ms.createEndpoint({remoteSdp: this.dlg.remote.sdp});
+    this._configMsEndpoint();
     await this.dlg.modify(this.ep.local.sdp, {
       headers: {
         'X-Reason': 'anchor-media'
       }
     });
-    this._configMsEndpoint();
+
+    if (currentMediaRoute === MediaPath.NoMedia) {
+      await this.ep.modify(this.dlg.remote.sdp);
+    }
   }
 
   async handleReinviteAfterMediaReleased(req, res) {
@@ -2592,6 +2673,7 @@ Duration=${duration} `
   }
 
   _configMsEndpoint() {
+    this._enableInbandDtmfIfRequired(this.ep);
     const opts = {
       ...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
       ...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'})
@@ -2601,6 +2683,20 @@ Duration=${duration} `
     }
   }
 
+  async _enableInbandDtmfIfRequired(ep) {
+    if (ep.inbandDtmfEnabled) return;
+    // only enable inband dtmf detection if voip carrier dtmf_type === tones
+    if (this.inbandDtmfEnabled) {
+      // https://developer.signalwire.com/freeswitch/FreeSWITCH-Explained/Modules/mod-dptools/6587132/#0-about
+      try {
+        ep.execute('start_dtmf');
+        ep.inbandDtmfEnabled = true;
+      } catch (err) {
+        this.logger.info(err, 'CallSession:_enableInbandDtmf - error enable inband DTMF');
+      }
+    }
+  }
+
   /**
    * notifyTaskError - only used when websocket connection is used instead of webhooks
    */

lib/session/confirm-call-session.js

@@ -8,7 +8,7 @@ const CallSession = require('./call-session');
 
  */
 class ConfirmCallSession extends CallSession {
-  constructor({logger, application, dlg, ep, tasks, callInfo, accountInfo, memberId, confName, rootSpan}) {
+  constructor({logger, application, dlg, ep, tasks, callInfo, accountInfo, memberId, confName, rootSpan, req}) {
     super({
       logger,
       application,
@@ -23,6 +23,7 @@ class ConfirmCallSession extends CallSession {
     });
     this.dlg = dlg;
     this.ep = ep;
+    this.req = req;
   }
 
   /**

lib/tasks/conference.js

@@ -653,7 +653,8 @@ class Conference extends Task {
         memberId: this.memberId,
         confName: this.confName,
         tasks,
-        rootSpan: cs.rootSpan
+        rootSpan: cs.rootSpan,
+        req: cs.req
       });
       await this._playSession.exec();
       this._playSession = null;

lib/tasks/dial.js

@@ -6,6 +6,7 @@ const {
   TaskName,
   TaskPreconditions,
   MAX_SIMRINGS,
+  MediaPath,
   KillReason
 } = require('../utils/constants');
 const assert = require('assert');
@@ -108,6 +109,7 @@ class TaskDial extends Task {
     this.proxy = this.data.proxy;
     this.tag = this.data.tag;
     this.boostAudioSignal = this.data.boostAudioSignal;
+    this._mediaPath = MediaPath.FullMedia;
 
     if (this.dtmfHook) {
       const {parentDtmfCollector, childDtmfCollector} = parseDtmfOptions(logger, this.data.dtmfCapture || {});
@@ -155,17 +157,21 @@ class TaskDial extends Task {
 
   get canReleaseMedia() {
     const keepAnchor = this.data.anchorMedia ||
-    this.cs.isBackGroundListen ||
-    this.cs.onHoldMusic ||
-    ANCHOR_MEDIA_ALWAYS ||
-    this.listenTask ||
-    this.dubTasks ||
-    this.transcribeTask ||
-    this.startAmd;
+      this.cs.isBackGroundListen ||
+      this.cs.onHoldMusic ||
+      ANCHOR_MEDIA_ALWAYS ||
+      this.listenTask ||
+      this.dubTasks ||
+      this.transcribeTask ||
+      this.startAmd;
 
     return !keepAnchor;
   }
 
+  get shouldExitMediaPathEntirely() {
+    return this.data.exitMediaPath;
+  }
+
   get summary() {
     if (this.target.length === 1) {
       const target = this.target[0];
@@ -186,6 +192,16 @@ class TaskDial extends Task {
 
   async exec(cs) {
     await super.exec(cs);
+
+    if (this.data.anchorMedia && this.data.exitMediaPath) {
+      this.logger.info('Dial:exec - incompatible anchorMedia and exitMediaPath are both set, will obey anchorMedia');
+      delete this.data.exitMediaPath;
+    }
+    if (!this.canReleaseMedia && this.data.exitMediaPath) {
+      this.logger.info(
+        'Dial:exec - exitMediaPath is set so features such as transcribe and record will not work on this call');
+    }
+
     try {
       if (this.listenTask) {
         const {span, ctx} = this.startChildSpan(`nested:${this.listenTask.summary}`);
@@ -303,7 +319,7 @@ class TaskDial extends Task {
       if (!cs.callGone && this.epOther) {
 
         /* if we can release the media back to the SBC, do so now */
-        if (this.canReleaseMedia) this._releaseMedia(cs, this.sd);
+        if (this.canReleaseMedia) this._releaseMedia(cs, this.sd, this.shouldExitMediaPathEntirely);
         else this.epOther.bridge(this.ep);
       }
     } catch (err) {
@@ -345,6 +361,12 @@ class TaskDial extends Task {
         const by = parseUri(req.getParsedHeader('Referred-By').uri);
         const referredBy = req.get('Referred-By');
         const userAgent = req.get('User-Agent');
+        const customHeaders = Object.keys(req.headers)
+          .filter((h) => h.toLowerCase().startsWith('x-'))
+          .reduce((acc, h) => {
+            acc[h] = req.get(h);
+            return acc;
+          }, {});
         this.logger.info({to}, 'refer to parsed');
         const json = await cs.requestor.request('verb:hook', this.referHook, {
           ...(callInfo.toJSON()),
@@ -355,7 +377,8 @@ class TaskDial extends Task {
             ...(userAgent && {sip_user_agent: userAgent}),
             ...(by && {referred_by_user: by.scheme === 'tel' ? by.number : by.user}),
             referring_call_sid,
-            referred_call_sid
+            referred_call_sid,
+            ...customHeaders
           }
         }, httpHeaders);
         if (json && Array.isArray(json)) {
@@ -752,7 +775,7 @@ class TaskDial extends Task {
             // Offhold, time to release media
             const newSdp = await this.ep.modify(req.body);
             await res.send(200, {body: newSdp});
-            await this._releaseMedia(this.cs, this.sd);
+            await this._releaseMedia(this.cs, this.sd, this.shouldExitMediaPathEntirely);
             this.isOutgoingLegHold = false;
           } else {
             this.logger.debug('Dial: _onReinvite receive unhold Request, update media server');
@@ -861,7 +884,9 @@ class TaskDial extends Task {
     }
 
     /* if we can release the media back to the SBC, do so now */
-    if (this.canReleaseMedia) setTimeout(this._releaseMedia.bind(this, cs, sd), 200);
+    if (this.canReleaseMedia || this.shouldExitMediaPathEntirely) {
+      setTimeout(this._releaseMedia.bind(this, cs, sd, this.shouldExitMediaPathEntirely), 200);
+    }
   }
 
   _bridgeEarlyMedia(sd) {
@@ -873,22 +898,57 @@ class TaskDial extends Task {
     }
   }
 
+  /* public api */
+  async updateMediaPath(desiredPath) {
+    this.logger.info(`Dial:updateMediaPath - ${this._mediaPath} => ${desiredPath}`);
+    switch (desiredPath) {
+      case MediaPath.NoMedia:
+        assert(this._mediaPath !== MediaPath.NoMedia, 'updateMediaPath: already no-media');
+        await this._releaseMedia(this.cs, this.sd, true);
+        break;
+
+      case MediaPath.PartialMedia:
+        assert(this._mediaPath !== MediaPath.PartialMedia, 'updateMediaPath: already partial-media');
+        if (this._mediaPath === MediaPath.FullMedia) {
+          await this._releaseMedia(this.cs, this.sd, false);
+        }
+        else {
+          // to go from no-media to partial-media we need to go through full-media first
+          await this.reAnchorMedia(this.cs, this.sd);
+          await this._releaseMedia(this.cs, this.sd, false);
+        }
+        assert(!this.epOther, 'updateMediaPath: epOther should be null');
+        assert(!this.ep, 'updateMediaPath: ep should be null');
+
+        break;
+      case MediaPath.FullMedia:
+        assert(this._mediaPath !== MediaPath.FullMedia, 'updateMediaPath: already full-media');
+        await this.reAnchorMedia(this.cs, this.sd);
+        break;
+
+      default:
+        assert(false, `updateMediaPath: invalid path request ${desiredPath}`);
+    }
+  }
+
   /**
    * Release the media from freeswitch
    * @param {*} cs
    * @param {*} sd
    */
-  async _releaseMedia(cs, sd) {
+  async _releaseMedia(cs, sd, releaseEntirely = false) {
     assert(cs.ep && sd.ep);
 
     try {
       // Wait until we got new SDP from B leg to ofter to A Leg
       const aLegSdp = cs.ep.remote.sdp;
-      await sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp);
+      await sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp, releaseEntirely);
       const bLegSdp = sd.dlg.remote.sdp;
-      await cs.releaseMediaToSBC(bLegSdp);
+      await cs.releaseMediaToSBC(bLegSdp, releaseEntirely);
       this.epOther = null;
-      this.logger.info('Dial:_releaseMedia - successfully released media from freewitch');
+      this._mediaPath = releaseEntirely ? MediaPath.NoMedia : MediaPath.PartialMedia;
+      this.logger.info(
+        `Dial:_releaseMedia - successfully released media from freewitch, media path is now ${this._mediaPath}`);
     } catch (err) {
       this.logger.info({err}, 'Dial:_releaseMedia error');
     }
@@ -898,8 +958,14 @@ class TaskDial extends Task {
     if (cs.ep && sd.ep) return;
 
     this.logger.info('Dial:reAnchorMedia - re-anchoring media to freewitch');
-    await Promise.all([sd.reAnchorMedia(), cs.reAnchorMedia()]);
+    await Promise.all([sd.reAnchorMedia(this._mediaPath), cs.reAnchorMedia(this._mediaPath)]);
     this.epOther = cs.ep;
+
+    this.epOther.bridge(this.ep);
+
+    this._mediaPath = MediaPath.FullMedia;
+    this.logger.info(
+      `Dial:_releaseMedia - successfully re-anchored media to freewitch, media path is now ${this._mediaPath}`);
   }
 
   // Handle RE-INVITE hold from caller leg.
@@ -918,11 +984,12 @@ class TaskDial extends Task {
         }
         this._onHoldHook(req);
       } else if (!isOnhold(req.body)) {
-        if (this.epOther && this.ep && this.isIncomingLegHold && this.canReleaseMedia) {
+        if (this.epOther && this.ep && this.isIncomingLegHold &&
+          (this.canReleaseMedia || this.shouldExitMediaPathEntirely)) {
           // Offhold, time to release media
           const newSdp = await this.epOther.modify(req.body);
           await res.send(200, {body: newSdp});
-          await this._releaseMedia(this.cs, this.sd);
+          await this._releaseMedia(this.cs, this.sd, this.shouldExitMediaPathEntirely);
           isHandled = true;
         }
         this.isIncomingLegHold = false;
@@ -981,7 +1048,8 @@ class TaskDial extends Task {
               callInfo: this.cs.callInfo,
               accountInfo: this.cs.accountInfo,
               tasks,
-              rootSpan: this.cs.rootSpan
+              rootSpan: this.cs.rootSpan,
+              req: this.cs.req
             });
             await this._onHoldSession.exec();
             this._onHoldSession = null;

lib/tasks/enqueue.js

@@ -369,7 +369,8 @@ class TaskEnqueue extends Task {
         callInfo: cs.callInfo,
         accountInfo: cs.accountInfo,
         tasks: tasksToRun,
-        rootSpan: cs.rootSpan
+        rootSpan: cs.rootSpan,
+        req: cs.req
       });
       await this._playSession.exec();
       this._playSession = null;

lib/tasks/gather.js

@@ -155,7 +155,8 @@ class TaskGather extends SttTask {
       const {hints, hintsBoost} = cs.globalSttHints;
       const setOfHints = new Set((this.data.recognizer.hints || [])
         .concat(hints)
-        .filter((h) => typeof h === 'string' && h.length > 0));
+        // allow for hints to be an array of object
+        .filter((h) => (typeof h === 'string' && h.length > 0) || (typeof h === 'object')));
       this.data.recognizer.hints = [...setOfHints];
       if (!this.data.recognizer.hintsBoost && hintsBoost) this.data.recognizer.hintsBoost = hintsBoost;
       this.logger.debug({hints: this.data.recognizer.hints, hintsBoost: this.data.recognizer.hintsBoost},
@@ -342,6 +343,13 @@ class TaskGather extends SttTask {
       this._killAudio(cs);
       this.emit('dtmf', evt);
     }
+    if (this.isContinuousAsr && evt.dtmf === this.asrDtmfTerminationDigit && this._bufferedTranscripts.length > 0) {
+      this.logger.info(`continuousAsr triggered with dtmf ${this.asrDtmfTerminationDigit}`);
+      this._clearAsrTimer();
+      this._clearTimer();
+      this._startFinalAsrTimer();
+      return;
+    }
     if (evt.dtmf === this.finishOnKey && this.input.includes('digits')) {
       resolved = true;
       this._resolve('dtmf-terminator-key');
@@ -364,13 +372,6 @@ class TaskGather extends SttTask {
         this._resolve('dtmf-num-digits');
       }
     }
-    else if (this.isContinuousAsr && evt.dtmf === this.asrDtmfTerminationDigit) {
-      this.logger.info(`continuousAsr triggered with dtmf ${this.asrDtmfTerminationDigit}`);
-      this._clearAsrTimer();
-      this._clearTimer();
-      this._startFinalAsrTimer();
-      return;
-    }
     if (!resolved && this.interDigitTimeout > 0 && this.digitBuffer.length >= this.minDigits) {
       /* start interDigitTimer */
       const ms = this.interDigitTimeout * 1000;
@@ -846,7 +847,10 @@ class TaskGather extends SttTask {
         this._startAsrTimer();
 
         /* some STT engines will keep listening after a final response, so no need to restart */
-        if (!['soniox', 'aws', 'microsoft', 'deepgram'].includes(this.vendor)) this._startTranscribing(ep);
+        if (!['soniox', 'aws', 'microsoft', 'deepgram', 'speechmatics'].includes(this.vendor) &&
+          !this.vendor.startsWith('custom')) {
+          this._startTranscribing(ep);
+        }
       }
       else {
         /* this was removed to fix https://github.com/jambonz/jambonz-feature-server/issues/783 */

lib/tasks/llm/index.js

@@ -1,6 +1,7 @@
 const Task = require('../task');
 const {TaskPreconditions} = require('../../utils/constants');
 const TaskLlmOpenAI_S2S = require('./llms/openai_s2s');
+const TaskLlmVoiceAgent_S2S = require('./llms/voice_agent_s2s');
 
 class TaskLlm extends Task {
   constructor(logger, opts) {
@@ -44,6 +45,12 @@ class TaskLlm extends Task {
           llm = new TaskLlmOpenAI_S2S(this.logger, this.data, this);
         }
         break;
+
+      case 'voiceagent':
+      case 'deepgram':
+        llm = new TaskLlmVoiceAgent_S2S(this.logger, this.data, this);
+        break;
+
       default:
         throw new Error(`Unsupported vendor ${this.vendor} for LLM`);
     }

lib/tasks/llm/llms/voice_agent_s2s.js

@@ -0,0 +1,313 @@
+const Task = require('../../task');
+const TaskName = 'Llm_VoiceAgent_s2s';
+const {LlmEvents_VoiceAgent} = require('../../../utils/constants');
+const ClientEvent = 'client.event';
+const SessionDelete = 'session.delete';
+
+const va_server_events = [
+  'Error',
+  'Welcome',
+  'SettingsApplied',
+  'ConversationText',
+  'UserStartedSpeaking',
+  'EndOfThought',
+  'AgentThinking',
+  'FunctionCallRequest',
+  'FunctionCalling',
+  'AgentStartedSpeaking',
+  'AgentAudioDone',
+];
+
+const expandWildcards = (events) => {
+  // no-op for deepgram
+  return events;
+};
+
+class TaskLlmVoiceAgent_S2S extends Task {
+  constructor(logger, opts, parentTask) {
+    super(logger, opts, parentTask);
+    this.parent = parentTask;
+
+    this.vendor = this.parent.vendor;
+    this.model = this.parent.model;
+    this.auth = this.parent.auth;
+    this.connectionOptions = this.parent.connectOptions;
+
+    const {apiKey} = this.auth || {};
+    if (!apiKey) throw new Error('auth.apiKey is required for VoiceAgent S2S');
+
+    this.apiKey = apiKey;
+    this.authType = 'bearer';
+    this.actionHook = this.data.actionHook;
+    this.eventHook = this.data.eventHook;
+    this.toolHook = this.data.toolHook;
+    const {settingsConfiguration} = this.data.llmOptions;
+
+    if (typeof settingsConfiguration !== 'object') {
+      throw new Error('llmOptions with an initial settingsConfiguration is required for VoiceAgent S2S');
+    }
+
+    // eslint-disable-next-line no-unused-vars
+    const {audio, ...rest} = settingsConfiguration;
+    const cfg = this.settingsConfiguration = rest;
+
+    if (!cfg.agent) throw new Error('llmOptions.settingsConfiguration.agent is required for VoiceAgent S2S');
+    if (!cfg.agent.think) {
+      throw new Error('llmOptions.settingsConfiguration.agent.think is required for VoiceAgent S2S');
+    }
+    if (!cfg.agent.think.model) {
+      throw new Error('llmOptions.settingsConfiguration.agent.think.model is required for VoiceAgent S2S');
+    }
+    if (!cfg.agent.think.provider?.type) {
+      throw new Error('llmOptions.settingsConfiguration.agent.think.provider.type is required for VoiceAgent S2S');
+    }
+
+    this.results = {
+      completionReason: 'normal conversation end'
+    };
+
+    /**
+     * only one of these will have items,
+     * if includeEvents, then these are the events to include
+     * if excludeEvents, then these are the events to exclude
+     */
+    this.includeEvents = [];
+    this.excludeEvents = [];
+
+    /* default to all events if user did not specify */
+    this._populateEvents(this.data.events || va_server_events);
+
+    this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
+    this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
+  }
+
+  get name() { return TaskName; }
+
+  get host() {
+    const {host} = this.connectionOptions || {};
+    return host || 'agent.deepgram.com';
+  }
+
+  get path() {
+    const {path} = this.connectionOptions || {};
+    if (path) return path;
+
+    return '/agent';
+  }
+
+  async _api(ep, args) {
+    const res = await ep.api('uuid_voice_agent_s2s', `^^|${args.join('|')}`);
+    if (!res.body?.startsWith('+OK')) {
+      throw new Error({args}, `Error calling uuid_voice_agent_s2s: ${JSON.stringify(res.body)}`);
+    }
+  }
+
+  async exec(cs, {ep}) {
+    await super.exec(cs);
+
+    await this._startListening(cs, ep);
+
+    await this.awaitTaskDone();
+
+    /* note: the parent llm verb started the span, which is why this is necessary */
+    await this.parent.performAction(this.results);
+
+    this._unregisterHandlers();
+  }
+
+  async kill(cs) {
+    super.kill(cs);
+
+    this._api(cs.ep, [cs.ep.uuid, SessionDelete])
+      .catch((err) => this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:kill - error deleting session'));
+
+    this.notifyTaskDone();
+  }
+
+  /**
+   * Send function call response to the VoiceAgent server
+   */
+  async processToolOutput(ep, tool_call_id, data) {
+    try {
+      const {data:response} = data;
+      this.logger.debug({tool_call_id, response}, 'TaskLlmVoiceAgent_S2S:processToolOutput');
+
+      if (!response.type || response.type !== 'FunctionCallResponse') {
+        this.logger.info({response},
+          'TaskLlmVoiceAgent_S2S:processToolOutput - invalid tool output, must be FunctionCallResponse');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(response)]);
+      }
+    } catch (err) {
+      this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:processToolOutput');
+    }
+  }
+
+  /**
+   * Send a session.update to the VoiceAgent server
+   * Note: creating and deleting conversation items also supported as well as interrupting the assistant
+   */
+  async processLlmUpdate(ep, data, _callSid) {
+    try {
+      this.logger.debug({data, _callSid}, 'TaskLlmVoiceAgent_S2S:processLlmUpdate');
+
+      if (!data.type || ![
+        'UpdateInstructions',
+        'UpdateSpeak',
+        'InjectAgentMessage',
+      ].includes(data.type)) {
+        this.logger.info({data}, 'TaskLlmVoiceAgent_S2S:processLlmUpdate - invalid mid-call request');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+      }
+    } catch (err) {
+      this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:processLlmUpdate');
+    }
+  }
+
+  async _startListening(cs, ep) {
+    this._registerHandlers(ep);
+
+    try {
+      const args = [ep.uuid, 'session.create', this.host, this.path, this.authType, this.apiKey];
+      await this._api(ep, args);
+    } catch (err) {
+      this.logger.error({err}, `TaskLlmVoiceAgent_S2S:_startListening: ${JSON.stringify(err)}`);
+      this.notifyTaskDone();
+    }
+  }
+
+  async _sendClientEvent(ep, obj) {
+    let ok = true;
+    this.logger.debug({obj}, 'TaskLlmVoiceAgent_S2S:_sendClientEvent');
+    try {
+      const args = [ep.uuid, ClientEvent, JSON.stringify(obj)];
+      await this._api(ep, args);
+    } catch (err) {
+      ok = false;
+      this.logger.error({err}, 'TaskLlmVoiceAgent_S2S:_sendClientEvent - Error');
+    }
+    return ok;
+  }
+
+  async _sendInitialMessage(ep) {
+    if (!await this._sendClientEvent(ep, this.settingsConfiguration)) {
+      this.notifyTaskDone();
+    }
+  }
+
+  _registerHandlers(ep) {
+    this.addCustomEventListener(ep, LlmEvents_VoiceAgent.Connect, this._onConnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_VoiceAgent.ConnectFailure, this._onConnectFailure.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_VoiceAgent.Disconnect, this._onDisconnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_VoiceAgent.ServerEvent, this._onServerEvent.bind(this, ep));
+  }
+
+  _unregisterHandlers() {
+    this.removeCustomEventListeners();
+  }
+
+  _onError(_ep, evt) {
+    this.logger.info({evt}, 'TaskLlmVoiceAgent_S2S:_onError');
+    this.notifyTaskDone();
+  }
+
+  _onConnect(ep) {
+    this.logger.debug('TaskLlmVoiceAgent_S2S:_onConnect');
+    this._sendInitialMessage(ep);
+  }
+  _onConnectFailure(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmVoiceAgent_S2S:_onConnectFailure');
+    this.results = {completionReason: 'connection failure'};
+    this.notifyTaskDone();
+  }
+  _onDisconnect(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmVoiceAgent_S2S:_onConnectFailure');
+    this.results = {completionReason: 'disconnect from remote end'};
+    this.notifyTaskDone();
+  }
+  async _onServerEvent(_ep, evt) {
+    let endConversation = false;
+    const type = evt.type;
+    this.logger.info({evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent');
+
+    /* check for failures, such as rate limit exceeded, that should terminate the conversation */
+    if (type === 'response.done' && evt.response.status === 'failed') {
+      endConversation = true;
+      this.results = {
+        completionReason: 'server failure',
+        error: evt.response.status_details?.error
+      };
+    }
+
+    /* server errors of some sort */
+    else if (type === 'error') {
+      endConversation = true;
+      this.results = {
+        completionReason: 'server error',
+        error: evt.error
+      };
+    }
+
+    /* tool calls */
+    else if (type === 'FunctionCallRequest') {
+      this.logger.debug({evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - function_call');
+      if (!this.toolHook) {
+        this.logger.warn({evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - no toolHook defined!');
+      }
+      else {
+        const {function_name:name, function_call_id:call_id} = evt;
+        const args = evt.input;
+
+        try {
+          await this.parent.sendToolHook(call_id, {name, args});
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmVoiceAgent - error calling function');
+          this.results = {
+            completionReason: 'client error calling function',
+            error: err
+          };
+          endConversation = true;
+        }
+      }
+    }
+
+    /* check whether we should notify on this event */
+    if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
+      this.parent.sendEventHook(evt)
+        .catch((err) => this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - error sending event hook'));
+    }
+
+    if (endConversation) {
+      this.logger.info({results: this.results},
+        'TaskLlmVoiceAgent_S2S:_onServerEvent - ending conversation due to error');
+      this.notifyTaskDone();
+    }
+  }
+
+  _populateEvents(events) {
+    if (events.includes('all')) {
+      /* work by excluding specific events */
+      const exclude = events
+        .filter((evt) => evt.startsWith('-'))
+        .map((evt) => evt.slice(1));
+      if (exclude.length === 0) this.includeEvents = va_server_events;
+      else this.excludeEvents = expandWildcards(exclude);
+    }
+    else {
+      /* work by including specific events */
+      const include = events
+        .filter((evt) => !evt.startsWith('-'));
+      this.includeEvents = expandWildcards(include);
+    }
+
+    this.logger.debug({
+      includeEvents: this.includeEvents,
+      excludeEvents: this.excludeEvents
+    }, 'TaskLlmVoiceAgent_S2S:_populateEvents');
+  }
+}
+
+module.exports = TaskLlmVoiceAgent_S2S;

lib/tasks/play.js

@@ -1,5 +1,6 @@
 const Task = require('./task');
 const {TaskName, TaskPreconditions} = require('../utils/constants');
+const { PlayFileNotFoundError } = require('../utils/error');
 
 class TaskPlay extends Task {
   constructor(logger, opts) {
@@ -66,8 +67,20 @@ class TaskPlay extends Task {
         }
       }
     } catch (err) {
-      if (timeout) clearTimeout(timeout);
-      this.logger.info(err, `TaskPlay:exec - error playing ${this.url}`);
+      this.logger.info(`TaskPlay:exec - error playing ${this.url}: ${err.message}`);
+      this.playComplete = true;
+      if (err.message === 'File Not Found') {
+        const {writeAlerts, AlertType} = cs.srf.locals;
+        await this.performAction({status: 'fail', reason: 'playFailed'}, !(this.parentTask || cs.isConfirmCallSession));
+        this.emit('playDone');
+        writeAlerts({
+          account_sid: cs.accountSid,
+          alert_type: AlertType.PLAY_FILENOTFOUND,
+          url: this.url,
+          target_sid: cs.callSid
+        });
+        throw new PlayFileNotFoundError(this.url);
+      }
     }
     this.emit('playDone');
   }

lib/tasks/rest_dial.js

@@ -77,11 +77,13 @@ class TaskRestDial extends Task {
           synthesizer: {
             vendor: cs.speechSynthesisVendor,
             language: cs.speechSynthesisLanguage,
-            voice: cs.speechSynthesisVoice
+            voice: cs.speechSynthesisVoice,
+            label: cs.speechSynthesisLabel,
           },
           recognizer: {
             vendor:  cs.speechRecognizerVendor,
-            language: cs.speechRecognizerLanguage
+            language: cs.speechRecognizerLanguage,
+            label: cs.speechRecognizerLabel,
           }
         }
       };

lib/tasks/sip_refer.js

@@ -12,6 +12,7 @@ class TaskSipRefer extends Task {
 
     this.referTo = this.data.referTo;
     this.referredBy = this.data.referredBy;
+    this.referredByDisplayName = this.data.referredByDisplayName;
     this.headers = this.data.headers || {};
     this.eventHook = this.data.eventHook;
   }
@@ -94,7 +95,10 @@ class TaskSipRefer extends Task {
         }
         if (status >= 200) {
           this.referSpan.setAttributes({'refer.finalNotify': status});
-          await this.performAction({refer_status: 202, final_referred_call_status: status});
+          await this.performAction({refer_status: 202, final_referred_call_status: status})
+            .catch((err) => {
+              this.logger.error(err, 'TaskSipRefer:exec - error performing action finalNotify');
+            });
           this.notifyTaskDone();
         }
       }
@@ -102,7 +106,7 @@ class TaskSipRefer extends Task {
   }
 
   _normalizeReferHeaders(cs, dlg) {
-    let {referTo, referredBy} = this;
+    let {referTo, referredBy, referredByDisplayName} = this;
 
     /* get IP address of the SBC to use as hostname if needed */
     const {host} = parseUri(dlg.remote.uri);
@@ -117,9 +121,12 @@ class TaskSipRefer extends Task {
       referredBy = cs.req?.callingNumber || dlg.local.uri;
       this.logger.info({referredBy}, 'setting referredby');
     }
+    if (!referredByDisplayName) {
+      referredByDisplayName = cs.req?.callingName;
+    }
     if (!referredBy.startsWith('<') && !referredBy.startsWith('sip') && !referredBy.startsWith('"')) {
       /* they may have only provided a phone number/user */
-      referredBy = `sip:${referredBy}@${host}`;
+      referredBy = `${referredByDisplayName ? `"${referredByDisplayName}"` : ''}<sip:${referredBy}@${host}>`;
     }
     return {referTo, referredBy};
   }

lib/tasks/stt-task.js

@@ -219,7 +219,8 @@ class SttTask extends Task {
           roleArn
         });
       this.logger.debug({roleArn}, `(roleArn) got aws access token ${servedFromCache ? 'from cache' : ''}`);
-      credentials = {...credentials, accessKeyId, secretAccessKey, sessionToken};
+      // from role ARN, we will get SessionToken, but feature server use it as securityToken.
+      credentials = {...credentials, accessKeyId, secretAccessKey, securityToken: sessionToken};
     }
     else if (vendor === 'verbio' && credentials.client_id && credentials.client_secret) {
       const {access_token, servedFromCache} = await getVerbioAccessToken(credentials);
@@ -229,9 +230,13 @@ class SttTask extends Task {
     }
     else if (vendor == 'aws' && !JAMBONES_AWS_TRANSCRIBE_USE_GRPC) {
       /* get AWS access token */
-      const {accessKeyId, secretAccessKey, securityToken, region } = credentials;
+      const {speech_credential_sid, accessKeyId, secretAccessKey, securityToken, region } = credentials;
       if (!securityToken) {
-        const { servedFromCache, ...newCredentials} = await getAwsAuthToken({accessKeyId, secretAccessKey, region});
+        const { servedFromCache, ...newCredentials} = await getAwsAuthToken({
+          speech_credential_sid,
+          accessKeyId,
+          secretAccessKey,
+          region});
         this.logger.debug({newCredentials}, `got aws security token ${servedFromCache ? 'from cache' : ''}`);
         credentials = {...newCredentials, region};
       }

lib/tasks/transcribe.js

@@ -27,6 +27,7 @@ class TaskTranscribe extends SttTask {
     super(logger, opts, parentTask);
 
     this.transcriptionHook = this.data.transcriptionHook;
+    this.translationHook = this.data.translationHook;
     this.earlyMedia = this.data.earlyMedia === true || (parentTask && parentTask.earlyMedia);
 
     if (this.data.recognizer) {
@@ -302,7 +303,9 @@ class TaskTranscribe extends SttTask {
       case 'speechmatics':
         this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
         this.addCustomEventListener(
-          ep, SpeechmaticsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+          ep, SpeechmaticsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep, channel));
+        this.addCustomEventListener(
+          ep, SpeechmaticsTranscriptionEvents.Translation, this._onTranslation.bind(this, cs, ep, channel));
         this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Info,
           this._onSpeechmaticsInfo.bind(this, cs, ep));
         this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.RecognitionStarted,
@@ -441,7 +444,7 @@ class TaskTranscribe extends SttTask {
         this._startAsrTimer(channel);
 
         /* some STT engines will keep listening after a final response, so no need to restart */
-        if (!['soniox', 'aws', 'microsoft', 'deepgram', 'google']
+        if (!['soniox', 'aws', 'microsoft', 'deepgram', 'google', 'speechmatics']
           .includes(this.vendor)) this._startTranscribing(cs, ep, channel);
       }
       else {
@@ -466,7 +469,7 @@ class TaskTranscribe extends SttTask {
         this._resolve(channel, evt);
 
         /* some STT engines will keep listening after a final response, so no need to restart */
-        if (!['soniox', 'aws', 'microsoft', 'deepgram', 'google'].includes(this.vendor) &&
+        if (!['soniox', 'aws', 'microsoft', 'deepgram', 'google', 'speechmatics'].includes(this.vendor) &&
           !this.vendor.startsWith('custom:')) {
           this.logger.debug('TaskTranscribe:_onTranscription - restarting transcribe');
           this._startTranscribing(cs, ep, channel);
@@ -492,6 +495,47 @@ class TaskTranscribe extends SttTask {
     }
   }
 
+  async _onTranslation(_cs, _ep, channel, evt, _fsEvent) {
+    this.logger.debug({evt}, 'TaskTranscribe:_onTranslation');
+    if (this.translationHook && evt.results?.length > 0) {
+      try {
+        const b3 = this.getTracingPropagation();
+        const httpHeaders = b3 && {b3};
+        const payload = {
+          ...this.cs.callInfo,
+          ...httpHeaders,
+          translation: {
+            channel,
+            language: evt.language,
+            translation: evt.results[0].content
+          }
+        };
+
+        this.logger.debug({payload}, 'sending translationHook');
+        const json = await this.cs.requestor.request('verb:hook', this.translationHook, payload);
+        this.logger.info({json}, 'completed translationHook');
+        if (json && Array.isArray(json) && !this.parentTask) {
+          const makeTask = require('./make_task');
+          const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
+          if (tasks && tasks.length > 0) {
+            this.logger.info({tasks: tasks}, `${this.name} replacing application with ${tasks.length} tasks`);
+            this.cs.replaceApplication(tasks);
+          }
+        }
+      } catch (err) {
+        this.logger.info(err, 'TranscribeTask:_onTranslation error');
+      }
+      if (this.parentTask) {
+        this.parentTask.emit('translation', evt);
+      }
+    }
+    if (this.killed) {
+      this.logger.debug('TaskTranscribe:_onTranslation exiting after receiving final transcription');
+      this._clearTimer();
+      this.notifyTaskDone();
+    }
+  }
+
   async _resolve(channel, evt) {
     if (evt.is_final) {
       /* we've got a final transcript, so end the otel child span for this channel */
@@ -671,7 +715,7 @@ class TaskTranscribe extends SttTask {
     this.logger.debug({evt}, 'TaskGather:_onSpeechmaticsInfo');
   }
 
-  async _onSpeechmaticsErrror(cs, _ep, evt) {
+  async _onSpeechmaticsError(cs, _ep, evt) {
     // eslint-disable-next-line no-unused-vars
     const {message, ...e} = evt;
     this._onVendorError(cs, _ep, {error: JSON.stringify(e)});

lib/utils/action-hook-delay.js

@@ -126,7 +126,12 @@ class ActionHookDelayProcessor extends Emitter {
     try {
       this._taskInProgress = makeTask(this.logger, t[0]);
       this._taskInProgress.disableTracing = true;
-      this._taskInProgress.exec(this.cs, {ep: this.ep});
+      this._taskInProgress.exec(this.cs, {ep: this.ep}).catch((err) => {
+        this.logger.info(`ActionHookDelayProcessor#_onNoResponseTimer: error playing file: ${err.message}`);
+        this._taskInProgress = null;
+        this.ep.removeAllListeners('playback-start');
+        this.ep.removeAllListeners('playback-stop');
+      });
     } catch (err) {
       this.logger.info(err, 'ActionHookDelayProcessor#_onNoResponseTimer: error starting action');
       this._taskInProgress = null;

lib/utils/constants.json

@@ -129,6 +129,7 @@
   },
   "SpeechmaticsTranscriptionEvents": {
     "Transcription": "speechmatics_transcribe::transcription",
+    "Translation": "speechmatics_transcribe::translation",
     "Info": "speechmatics_transcribe::info",
     "RecognitionStarted": "speechmatics_transcribe::recognition_started",
     "ConnectFailure": "speechmatics_transcribe::connect_failed",
@@ -174,6 +175,13 @@
     "Disconnect": "openai_s2s::disconnect",
     "ServerEvent": "openai_s2s::server_event"
   },
+  "LlmEvents_VoiceAgent": {
+    "Error": "error",
+    "Connect": "voice_agent_s2s::connect",
+    "ConnectFailure": "voice_agent_s2s::connect_failed",
+    "Disconnect": "voice_agent_s2s::disconnect",
+    "ServerEvent": "voice_agent_s2s::server_event"
+  },
   "QueueResults": {
     "Bridged": "bridged",
     "Error": "error",
@@ -220,7 +228,18 @@
     "ToneTimeout": "amd_tone_timeout",
     "Stopped": "amd_stopped"
   },
+  "MediaPath": {
+    "NoMedia": "no-media",
+    "PartialMedia": "partial-media",
+    "FullMedia": "full-media"
+  },
   "MAX_SIMRINGS": 10,
   "BONG_TONE": "tone_stream://v=-7;%(100,0,941.0,1477.0);v=-7;>=2;+=.1;%(1400,0,350,440)",
-  "FS_UUID_SET_NAME": "fsUUIDs"
+  "FS_UUID_SET_NAME": "fsUUIDs",
+  "SystemState" : {
+    "Online": "ONLINE",
+    "Offline": "OFFLINE",
+    "GracefulShutdownInProgress":"SHUTDOWN_IN_PROGRESS"
+  },
+  "FEATURE_SERVER" : "feature-server"
 }

lib/utils/db-utils.js

@@ -215,11 +215,23 @@ module.exports = (logger, srf) => {
     }
   };
 
+  const lookupVoipCarrierBySid = async(sid) => {
+    const pp = pool.promise();
+    try {
+      const [r] = await pp.query('SELECT * FROM voip_carriers WHERE voip_carrier_sid = ?', [sid]);
+      return r;
+
+    } catch (err) {
+      logger.error({err}, `lookupVoipCarrierBySid: Error ${sid}`);
+    }
+  };
+
   return {
     lookupAccountDetails,
     updateSpeechCredentialLastUsed,
     lookupCarrier,
     lookupCarrierByPhoneNumber,
-    lookupGoogleCustomVoice
+    lookupGoogleCustomVoice,
+    lookupVoipCarrierBySid
   };
 };

lib/utils/error.js

@@ -1,9 +1,24 @@
-class SpeechCredentialError extends Error {
+class NonFatalTaskError extends Error {
   constructor(msg) {
     super(msg);
   }
 }
 
+class SpeechCredentialError extends NonFatalTaskError {
+  constructor(msg) {
+    super(msg);
+  }
+}
+
+class PlayFileNotFoundError extends NonFatalTaskError {
+  constructor(url) {
+    super('File not found');
+    this.url = url;
+  }
+}
+
 module.exports = {
-  SpeechCredentialError
+  SpeechCredentialError,
+  NonFatalTaskError,
+  PlayFileNotFoundError
 };

lib/utils/install-srf-locals.js

@@ -199,7 +199,8 @@ function installSrfLocals(srf, logger) {
   } = require('@jambonz/speech-utils')({}, logger);
   const {
     writeAlerts,
-    AlertType
+    AlertType,
+    writeSystemAlerts
   } = require('@jambonz/time-series')(logger, {
     host: JAMBONES_TIME_SERIES_HOST,
     commitSize: 50,
@@ -269,7 +270,8 @@ function installSrfLocals(srf, logger) {
     getFreeswitch,
     stats: stats,
     writeAlerts,
-    AlertType
+    AlertType,
+    writeSystemAlerts
   };
 
   if (localIp) {

lib/utils/place-outdial.js

@@ -1,5 +1,5 @@
 const Emitter = require('events');
-const {CallStatus} = require('./constants');
+const {CallStatus, MediaPath} = require('./constants');
 const SipError = require('drachtio-srf').SipError;
 const {TaskPreconditions, CallDirection} = require('../utils/constants');
 const CallInfo = require('../session/call-info');
@@ -381,7 +381,8 @@ class SingleDialer extends Emitter {
         callInfo: this.callInfo,
         accountInfo: this.accountInfo,
         tasks,
-        rootSpan: this.rootSpan
+        rootSpan: this.rootSpan,
+        req: this.req
       });
       await cs.exec();
 
@@ -455,21 +456,26 @@ class SingleDialer extends Emitter {
     return cs;
   }
 
-  async releaseMediaToSBC(remoteSdp, localSdp) {
+  async releaseMediaToSBC(remoteSdp, localSdp, releaseMediaEntirely) {
     assert(this.dlg && this.dlg.connected && this.ep && typeof remoteSdp === 'string');
     const sdp = stripCodecs(this.logger, remoteSdp, localSdp) || remoteSdp;
     await this.dlg.modify(sdp, {
       headers: {
-        'X-Reason': 'release-media'
+        'X-Reason': releaseMediaEntirely ? 'release-media-entirely' : 'release-media'
       }
     });
-    this.ep.destroy()
-      .then(() => this.ep = null)
-      .catch((err) => this.logger.error({err}, 'SingleDialer:releaseMediaToSBC: Error destroying endpoint'));
+    try {
+      await this.ep.destroy();
+    } catch (err) {
+      this.logger.error({err}, 'SingleDialer:releaseMediaToSBC: Error destroying endpoint');
+    }
+    this.ep = null;
   }
 
-  async reAnchorMedia() {
+  async reAnchorMedia(currentMediaRoute = MediaPath.PartialMedia) {
     assert(this.dlg && this.dlg.connected && !this.ep);
+
+    this.logger.debug('SingleDialer:reAnchorMedia: re-anchoring media after partial media');
     this.ep = await this.ms.createEndpoint({remoteSdp: this.dlg.remote.sdp});
     this._configMsEndpoint();
     await this.dlg.modify(this.ep.local.sdp, {
@@ -477,6 +483,11 @@ class SingleDialer extends Emitter {
         'X-Reason': 'anchor-media'
       }
     });
+
+    if (currentMediaRoute === MediaPath.NoMedia) {
+      this.logger.debug('SingleDialer:reAnchorMedia: repoint endpoint after no media');
+      await this.ep.modify(this.dlg.remote.sdp);
+    }
   }
 
   _notifyCallStatusChange({callStatus, sipStatus, sipReason, duration}) {

lib/utils/sbc-pinger.js

@@ -46,12 +46,24 @@ module.exports = (logger) => {
             const {srf} = require('../..');
             srf.locals.publicIp = publicIp;
           })
-          .on(LifeCycleEvents.ScaleIn, () => {
+          .on(LifeCycleEvents.ScaleIn, async() => {
             logger.info('AWS scale-in notification: begin drying up calls');
             dryUpCalls = true;
             lifecycleEmitter.operationalState = LifeCycleEvents.ScaleIn;
 
             const {srf} = require('../..');
+            const {writeSystemAlerts} = srf.locals;
+            if (writeSystemAlerts) {
+              const {SystemState, FEATURE_SERVER} = require('./constants');
+              await writeSystemAlerts({
+                system_component: FEATURE_SERVER,
+                state : SystemState.GracefulShutdownInProgress,
+                fields : {
+                  detail: `feature-server with process_id ${process.pid} shutdown in progress`,
+                  host: srf.locals?.ipv4
+                }
+              });
+            }
             pingProxies(srf);
 
             // if we have zero calls, we can complete the scale-in right

lib/utils/transcription-utils.js

@@ -109,6 +109,8 @@ const stickyVars = {
     'SPEECHMATICS_HOST',
     'SPEECHMATICS_PATH',
     'SPEECHMATICS_SPEECH_HINTS',
+    'SPEECHMATICS_TRANSLATION_LANGUAGES',
+    'SPEECHMATICS_TRANSLATION_PARTIALS'
   ]
 };
 
@@ -183,7 +185,10 @@ const selectDefaultGoogleModel = (task, language, version) => {
     (useV2 ? 'long' : 'latest_long');
 };
 const consolidateTranscripts = (bufferedTranscripts, channel, language, vendor) => {
-  if (bufferedTranscripts.length === 1) return bufferedTranscripts[0];
+  if (bufferedTranscripts.length === 1) {
+    bufferedTranscripts[0].is_final = true;
+    return bufferedTranscripts[0];
+  }
   let totalConfidence = 0;
   const finalTranscript = bufferedTranscripts.reduce((acc, evt) => {
     totalConfidence += evt.alternatives[0].confidence;
@@ -937,11 +942,18 @@ module.exports = (logger) => {
       };
     }
     else if ('speechmatics' === vendor) {
+      const {speechmaticsOptions = {}} = rOpts;
       opts = {
         ...opts,
         ...(sttCredentials.api_key) && {SPEECHMATICS_API_KEY: sttCredentials.api_key},
         ...(sttCredentials.speechmatics_stt_uri) && {SPEECHMATICS_HOST: sttCredentials.speechmatics_stt_uri},
         ...(rOpts.hints?.length > 0 && {SPEECHMATICS_SPEECH_HINTS: rOpts.hints.join(',')}),
+        ...(speechmaticsOptions.translation_config &&
+          {
+            SPEECHMATICS_TRANSLATION_LANGUAGES: speechmaticsOptions.translation_config.target_languages.join(','),
+            SPEECHMATICS_TRANSLATION_PARTIALS: speechmaticsOptions.translation_config.enable_partials ? 1 : 0
+          }
+        ),
       };
     }
     else if (vendor.startsWith('custom:')) {

package-lock.json

@@ -17,8 +17,8 @@
         "@jambonz/realtimedb-helpers": "^0.8.8",
         "@jambonz/speech-utils": "^0.1.22",
         "@jambonz/stats-collector": "^0.1.10",
-        "@jambonz/time-series": "^0.2.9",
-        "@jambonz/verb-specifications": "^0.0.83",
+        "@jambonz/time-series": "^0.2.12",
+        "@jambonz/verb-specifications": "^0.0.90",
         "@opentelemetry/api": "^1.8.0",
         "@opentelemetry/exporter-jaeger": "^1.23.0",
         "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
@@ -31,7 +31,7 @@
         "bent": "^7.3.12",
         "debug": "^4.3.4",
         "deepcopy": "^2.1.0",
-        "drachtio-fsmrf": "^3.0.45",
+        "drachtio-fsmrf": "^3.0.46",
         "drachtio-srf": "^4.5.35",
         "express": "^4.19.2",
         "express-validator": "^7.0.1",
@@ -1572,18 +1572,20 @@
       }
     },
     "node_modules/@jambonz/time-series": {
-      "version": "0.2.9",
-      "resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.9.tgz",
-      "integrity": "sha512-+/Oal0mjjV4iQ8q0ymtvVJP+GqgGpYUb2bc/FD/xDxOzKtl340l9yoM3oczREJg5IvEkpExz6NogJzUiCSeVnQ==",
+      "version": "0.2.12",
+      "resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.12.tgz",
+      "integrity": "sha512-EYw5f1QasblWrP2K/NabpJYkQm8XOCP1fem8luO8c7Jm8YTBwI+Ge3zB7rpU8ruoVdbrTot/pcihhTqzq4IYqA==",
+      "license": "MIT",
       "dependencies": {
         "debug": "^4.3.1",
         "influx": "^5.9.3"
       }
     },
     "node_modules/@jambonz/verb-specifications": {
-      "version": "0.0.83",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.83.tgz",
-      "integrity": "sha512-3m1o3gnWw1yEwNfkZ6IIyUhdUqsQ3aWBNoWJHswe4udvVbEIxPHi+8jKGTJVF2vxddzwfOWp7RmMCV9RmKWzkw==",
+      "version": "0.0.90",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.90.tgz",
+      "integrity": "sha512-ix2XByDTSroWlEMMY26ba2mbON4Yh3911Agm44GeV3ygmJMGABwCv28KqItOVBW0Ro5uSM+cQ0vEacPVnmAldA==",
+      "license": "MIT",
       "dependencies": {
         "debug": "^4.3.4",
         "pino": "^8.8.0"
@@ -3877,9 +3879,10 @@
       }
     },
     "node_modules/drachtio-fsmrf": {
-      "version": "3.0.45",
-      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.45.tgz",
-      "integrity": "sha512-YsrevTwGvg9v0OQXB4gZedlmzegB83fyd3NX3X2x7NXsKa5jO6TZCvZVHtBf/jR/ELE3B0aVLpxHw2YviRMIuQ==",
+      "version": "3.0.46",
+      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.46.tgz",
+      "integrity": "sha512-ScgyOnsOL45feuKKquT2Gij/jDRdjVha2TnQ6/Me2/M/C+29c9W7cdYlt3UZB3GyodazBukwIy5RrZf1iuXBGw==",
+      "license": "MIT",
       "dependencies": {
         "camel-case": "^4.1.2",
         "debug": "^2.6.9",
@@ -10742,18 +10745,18 @@
       }
     },
     "@jambonz/time-series": {
-      "version": "0.2.9",
-      "resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.9.tgz",
-      "integrity": "sha512-+/Oal0mjjV4iQ8q0ymtvVJP+GqgGpYUb2bc/FD/xDxOzKtl340l9yoM3oczREJg5IvEkpExz6NogJzUiCSeVnQ==",
+      "version": "0.2.12",
+      "resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.12.tgz",
+      "integrity": "sha512-EYw5f1QasblWrP2K/NabpJYkQm8XOCP1fem8luO8c7Jm8YTBwI+Ge3zB7rpU8ruoVdbrTot/pcihhTqzq4IYqA==",
       "requires": {
         "debug": "^4.3.1",
         "influx": "^5.9.3"
       }
     },
     "@jambonz/verb-specifications": {
-      "version": "0.0.83",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.83.tgz",
-      "integrity": "sha512-3m1o3gnWw1yEwNfkZ6IIyUhdUqsQ3aWBNoWJHswe4udvVbEIxPHi+8jKGTJVF2vxddzwfOWp7RmMCV9RmKWzkw==",
+      "version": "0.0.90",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.90.tgz",
+      "integrity": "sha512-ix2XByDTSroWlEMMY26ba2mbON4Yh3911Agm44GeV3ygmJMGABwCv28KqItOVBW0Ro5uSM+cQ0vEacPVnmAldA==",
       "requires": {
         "debug": "^4.3.4",
         "pino": "^8.8.0"
@@ -12464,9 +12467,9 @@
       }
     },
     "drachtio-fsmrf": {
-      "version": "3.0.45",
-      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.45.tgz",
-      "integrity": "sha512-YsrevTwGvg9v0OQXB4gZedlmzegB83fyd3NX3X2x7NXsKa5jO6TZCvZVHtBf/jR/ELE3B0aVLpxHw2YviRMIuQ==",
+      "version": "3.0.46",
+      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.46.tgz",
+      "integrity": "sha512-ScgyOnsOL45feuKKquT2Gij/jDRdjVha2TnQ6/Me2/M/C+29c9W7cdYlt3UZB3GyodazBukwIy5RrZf1iuXBGw==",
       "requires": {
         "camel-case": "^4.1.2",
         "debug": "^2.6.9",

package.json

@@ -33,8 +33,8 @@
     "@jambonz/realtimedb-helpers": "^0.8.8",
     "@jambonz/speech-utils": "^0.1.22",
     "@jambonz/stats-collector": "^0.1.10",
-    "@jambonz/time-series": "^0.2.9",
-    "@jambonz/verb-specifications": "^0.0.83",
+    "@jambonz/time-series": "^0.2.12",
+    "@jambonz/verb-specifications": "^0.0.90",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/exporter-jaeger": "^1.23.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
@@ -47,7 +47,7 @@
     "bent": "^7.3.12",
     "debug": "^4.3.4",
     "deepcopy": "^2.1.0",
-    "drachtio-fsmrf": "^3.0.45",
+    "drachtio-fsmrf": "^3.0.46",
     "drachtio-srf": "^4.5.35",
     "express": "^4.19.2",
     "express-validator": "^7.0.1",

test/db/jambones-sql.sql

@@ -351,6 +351,8 @@ speech_credential_sid CHAR(36) NOT NULL,
 model VARCHAR(512) NOT NULL,
 reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
 name VARCHAR(64) NOT NULL,
+voice_cloning_key MEDIUMTEXT,
+use_voice_cloning_key BOOLEAN DEFAULT false,
 PRIMARY KEY (google_custom_voice_sid)
 );