#1088 ignore UtteranceEnd if we have unprocessed words (#1089)

* #1088 ignore UtteranceEnd if we have unprocessed words

* wip

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
+FROM --platform=linux/amd64 node:20-alpine as base
 
 RUN apk --update --no-cache add --virtual .builds-deps build-base python3
 

lib/config.js

@@ -137,6 +137,8 @@ const JAMBONES_EAGERLY_PRE_CACHE_AUDIO = parseInt(process.env.JAMBONES_EAGERLY_P
 
 const JAMBONES_USE_FREESWITCH_TIMER_FD = process.env.JAMBONES_USE_FREESWITCH_TIMER_FD;
 const JAMBONES_DIAL_SBC_FOR_REGISTERED_USER = process.env.JAMBONES_DIAL_SBC_FOR_REGISTERED_USER || false;
+const JAMBONES_MEDIA_TIMEOUT_MS = process.env.JAMBONES_MEDIA_TIMEOUT_MS || 0;
+const JAMBONES_MEDIA_HOLD_TIMEOUT_MS = process.env.JAMBONES_MEDIA_HOLD_TIMEOUT_MS || 0;
 
 module.exports = {
   JAMBONES_MYSQL_HOST,
@@ -223,5 +225,7 @@ module.exports = {
   JAMBONZ_DISABLE_DIAL_PAI_HEADER,
   JAMBONES_DISABLE_DIRECT_P2P_CALL,
   JAMBONES_USE_FREESWITCH_TIMER_FD,
-  JAMBONES_DIAL_SBC_FOR_REGISTERED_USER
+  JAMBONES_DIAL_SBC_FOR_REGISTERED_USER,
+  JAMBONES_MEDIA_TIMEOUT_MS,
+  JAMBONES_MEDIA_HOLD_TIMEOUT_MS
 };

lib/http-routes/api/create-call.js

@@ -100,6 +100,7 @@ router.post('/',
         ...(req.body?.application_sid && {'X-Application-Sid': req.body.application_sid}),
         ...(restDial.fromHost && {'X-Preferred-From-Host': restDial.fromHost}),
         ...(record_all_calls && {'X-Record-All-Calls': recordOutputFormat}),
+        ...(target.proxy && {'X-SIP-Proxy': target.proxy}),
         ...target.headers
       };
 

lib/middleware.js

@@ -1,5 +1,5 @@
 const uuidv4 = require('uuid-random');
-const {CallDirection, AllowedSipRecVerbs} = require('./utils/constants');
+const {CallDirection, AllowedSipRecVerbs, WS_CLOSE_CODES} = require('./utils/constants');
 const {parseSiprecPayload} = require('./utils/siprec-utils');
 const CallInfo = require('./session/call-info');
 const HttpRequestor = require('./utils/http-requestor');
@@ -460,7 +460,7 @@ module.exports = function(srf, logger) {
       }).catch((err) => this.logger.info({err}, 'Error generating alert for parsing application'));
       logger.info({err}, `Error retrieving or parsing application: ${err?.message}`);
       res.send(480, {headers: {'X-Reason': err?.message || 'unknown'}});
-      app.requestor.close();
+      app.requestor.close(WS_CLOSE_CODES.GoingAway);
     }
   }
 

lib/session/call-session.js

@@ -28,7 +28,9 @@ const {
   JAMBONES_INJECT_CONTENT,
   JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
   AWS_REGION,
-  JAMBONES_USE_FREESWITCH_TIMER_FD
+  JAMBONES_USE_FREESWITCH_TIMER_FD,
+  JAMBONES_MEDIA_TIMEOUT_MS,
+  JAMBONES_MEDIA_HOLD_TIMEOUT_MS
 } = require('../config');
 const bent = require('bent');
 const BackgroundTaskManager = require('../utils/background-task-manager');
@@ -570,7 +572,18 @@ class CallSession extends Emitter {
         this._actionHookDelayProcessor = new ActionHookDelayProcessor(this.logger, opts, this, this.ep);
         this._actionHookDelayProcessor.on('giveup', () => {
           this.logger.info('CallSession: ActionHookDelayProcessor: giveup event - hanging up call');
-          this._jambonzHangup();
+          const {writeAlerts} = this.srf.locals;
+          try {
+            writeAlerts({
+              alert_type: 'bot-action-delay-giveup',
+              account_sid: this.accountSid,
+              message: 'Call terminated due to bot action delay timeout',
+              target_sid: this.callSid
+            });
+          } catch (err) {
+            this.logger.error({err}, 'Error writing bot-action-delay-giveup alert');
+          }
+          this._jambonzHangup('bot-action-delay-giveup');
           if (this.wakeupResolver) {
             this.logger.debug('CallSession: Giveup timer expired - waking up');
             this.wakeupResolver({reason: 'noResponseGiveUp'});
@@ -853,6 +866,8 @@ class CallSession extends Emitter {
     }
   }
   clearTtsStream() {
+    this.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'user_interruption'})
+      .catch((err) => this.logger.info({err}, 'CallSession:clearTtsStream - Error sending user_interruption'));
     this.ttsStreamingBuffer?.clear();
   }
 
@@ -1069,6 +1084,12 @@ class CallSession extends Emitter {
             api_key: credential.api_key
           };
         }
+        else if ('voxist' === vendor) {
+          return {
+            speech_credential_sid: credential.speech_credential_sid,
+            api_key: credential.api_key
+          };
+        }
         else if ('whisper' === vendor) {
           return {
             api_key: credential.api_key,
@@ -1094,7 +1115,8 @@ class CallSession extends Emitter {
             speech_credential_sid: credential.speech_credential_sid,
             auth_token: credential.auth_token,
             custom_stt_url: credential.custom_stt_url,
-            custom_tts_url: credential.custom_tts_url
+            custom_tts_url: credential.custom_tts_url,
+            custom_tts_streaming_url: credential.custom_tts_streaming_url
           };
         }
       }
@@ -1272,6 +1294,11 @@ class CallSession extends Emitter {
       this.wakeupResolver({reason: 'session ended'});
       this.wakeupResolver = null;
     }
+
+    if (this._maxCallDurationTimer) {
+      clearTimeout(this._maxCallDurationTimer);
+      this._maxCallDurationTimer = null;
+    }
   }
 
   /**
@@ -1447,7 +1474,7 @@ class CallSession extends Emitter {
     if (!listenTask) {
       return this.logger.info('CallSession:_lccListenStatus - invalid listen_status: Dial does not have a listen');
     }
-    listenTask.updateListen(opts.listen_status);
+    listenTask.updateListen(opts.listen_status || opts.stream_status);
   }
 
   /**
@@ -1589,7 +1616,7 @@ Duration=${duration} `
     // this whole thing requires us to be in a Dial verb
     const task = this.currentTask;
     if (!task || ![TaskName.Dial, TaskName.Listen].includes(task.name)) {
-      return this.logger.info('CallSession:_lccWhisper - invalid command since we are not in a dial or listen');
+      return this.logger.info('CallSession:_lccWhisper - invalid command since we are not in a dial or stream/listen');
     }
 
     // allow user to provide a url object, a url string, an array of tasks, or a single task
@@ -1772,62 +1799,66 @@ Duration=${duration} `
   async updateCall(opts, callSid) {
     this.logger.debug(opts, 'CallSession:updateCall');
 
-    if (opts.call_status) {
-      return this._lccCallStatus(opts);
-    }
-    if (opts.call_hook || opts.child_call_hook) {
-      return await this._lccCallHook(opts);
-    }
-    if (opts.listen_status) {
-      await this._lccListenStatus(opts);
-    }
-    if (opts.transcribe_status) {
-      await this._lccTranscribeStatus(opts);
-    }
-    else if (opts.mute_status) {
-      await this._lccMuteStatus(opts.mute_status === 'mute', callSid);
-    }
-    else if (opts.conf_hold_status) {
-      await this._lccConfHoldStatus(opts);
-    }
-    else if (opts.conf_mute_status) {
-      await this._lccConfMuteStatus(opts);
-    }
-    else if (opts.sip_request) {
-      const res = await this._lccSipRequest(opts, callSid);
-      return {status: res.status, reason: res.reason};
-    } else if (opts.dtmf) {
-      await this._lccDtmf(opts, callSid);
-    }
-    else if (opts.record) {
-      await this.notifyRecordOptions(opts.record);
-    }
-    else if (opts.tag) {
-      return this._lccTag(opts);
-    }
-    else if (opts.conferenceParticipantAction) {
-      return this._lccConferenceParticipantAction(opts.conferenceParticipantAction);
-    }
-    else if (opts.dub) {
-      return this._lccDub(opts.dub, callSid);
-    }
-    else if (opts.boostAudioSignal) {
-      return this._lccBoostAudioSignal(opts, callSid);
-    }
-    else if (opts.media_path) {
-      return this._lccMediaPath(opts.media_path, callSid);
-    }
-    else if (opts.llm_tool_output) {
-      return this._lccToolOutput(opts.tool_call_id, opts.llm_tool_output, callSid);
-    }
-    else if (opts.llm_update) {
-      return this._lccLlmUpdate(opts.llm_update, callSid);
-    }
+    try {
+      if (opts.call_status) {
+        return this._lccCallStatus(opts);
+      }
+      if (opts.call_hook || opts.child_call_hook) {
+        return await this._lccCallHook(opts);
+      }
+      if (opts.listen_status || opts.stream_status) {
+        await this._lccListenStatus(opts);
+      }
+      if (opts.transcribe_status) {
+        await this._lccTranscribeStatus(opts);
+      }
+      else if (opts.mute_status) {
+        await this._lccMuteStatus(opts.mute_status === 'mute', callSid);
+      }
+      else if (opts.conf_hold_status) {
+        await this._lccConfHoldStatus(opts);
+      }
+      else if (opts.conf_mute_status) {
+        await this._lccConfMuteStatus(opts);
+      }
+      else if (opts.sip_request) {
+        const res = await this._lccSipRequest(opts, callSid);
+        return {status: res.status, reason: res.reason};
+      } else if (opts.dtmf) {
+        await this._lccDtmf(opts, callSid);
+      }
+      else if (opts.record) {
+        await this.notifyRecordOptions(opts.record);
+      }
+      else if (opts.tag) {
+        return this._lccTag(opts);
+      }
+      else if (opts.conferenceParticipantAction) {
+        return this._lccConferenceParticipantAction(opts.conferenceParticipantAction);
+      }
+      else if (opts.dub) {
+        return this._lccDub(opts.dub, callSid);
+      }
+      else if (opts.boostAudioSignal) {
+        return this._lccBoostAudioSignal(opts, callSid);
+      }
+      else if (opts.media_path) {
+        return this._lccMediaPath(opts.media_path, callSid);
+      }
+      else if (opts.llm_tool_output) {
+        return this._lccToolOutput(opts.tool_call_id, opts.llm_tool_output, callSid);
+      }
+      else if (opts.llm_update) {
+        return this._lccLlmUpdate(opts.llm_update, callSid);
+      }
 
-    // whisper may be the only thing we are asked to do, or it may that
-    // we are doing a whisper after having muted, paused recording etc..
-    if (opts.whisper) {
-      return this._lccWhisper(opts, callSid);
+      // whisper may be the only thing we are asked to do, or it may that
+      // we are doing a whisper after having muted, paused recording etc..
+      if (opts.whisper) {
+        return this._lccWhisper(opts, callSid);
+      }
+    } catch (err) {
+      this.logger.info({err, opts, callSid}, 'CallSession:updateCall - error updating call');
     }
   }
 
@@ -2088,6 +2119,7 @@ Duration=${duration} `
         break;
 
       case 'listen:status':
+      case 'stream:status':
         this._lccListenStatus(data);
         break;
 
@@ -2462,7 +2494,7 @@ Duration=${duration} `
           res.send(200, {body: this.ep.local.sdp});
         }
         else {
-          if (this.currentTask.name === TaskName.Dial && this.currentTask.isOnHoldEnabled) {
+          if (this.currentTask && this.currentTask.name === TaskName.Dial && this.currentTask.isOnHoldEnabled) {
             this.logger.info('onholdMusic reINVITE after media has been released');
             await this.currentTask.handleReinviteAfterMediaReleased(req, res);
           } else {
@@ -2796,15 +2828,25 @@ Duration=${duration} `
 
   _configMsEndpoint() {
     this._enableInbandDtmfIfRequired(this.ep);
+    this.ep.once('destroy', this._handleMediaTimeout.bind(this));
     const opts = {
       ...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
-      ...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'})
+      ...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'}),
+      ...(JAMBONES_MEDIA_TIMEOUT_MS && {media_timeout: JAMBONES_MEDIA_TIMEOUT_MS}),
+      ...(JAMBONES_MEDIA_HOLD_TIMEOUT_MS && {media_hold_timeout: JAMBONES_MEDIA_HOLD_TIMEOUT_MS})
     };
     if (Object.keys(opts).length > 0) {
       this.ep.set(opts);
     }
   }
 
+  async _handleMediaTimeout(evt) {
+    if (evt.reason === 'MEDIA_TIMEOUT' && !this.callGone) {
+      this.logger.info('CallSession:_handleMediaTimeout: received MEDIA_TIMEOUT, hangup the call');
+      this._jambonzHangup('Media Timeout');
+    }
+  }
+
   async _enableInbandDtmfIfRequired(ep) {
     if (ep.inbandDtmfEnabled) return;
     // only enable inband dtmf detection if voip carrier dtmf_type === tones
@@ -2929,6 +2971,27 @@ Duration=${duration} `
     }
     this.logger.info({vendor}, 'CallSession:_onTtsStreamingConnectFailure - tts streaming connect failure');
   }
+
+  async startMaxCallDurationTimer(timeLimit) {
+    if (!this._maxCallDurationTimer && timeLimit > 0) {
+      this.timeLimit = timeLimit;
+      this._maxCallDurationTimer = setTimeout(this._onMaxCallDuration.bind(this), timeLimit * 1000);
+      this.logger.debug(`CallSession:startMaxCallDurationTimer - started max call duration timer for ${timeLimit}s`);
+    }
+  }
+
+  /**
+   * _onMaxCallDuration - called when the call has reached the maximum duration
+   */
+  _onMaxCallDuration() {
+    this.logger.info(`callSession:_onMaxCallDuration tearing down call as it has reached ${this.timeLimit}s`);
+    if (!this.dlg) {
+      this.logger.debug('CallSession:_onMaxCallDuration - no dialog, call already gone');
+      return;
+    }
+    this._jambonzHangup('Max Call Duration');
+    this._maxCallDurationTimer = null;
+  }
 }
 
 module.exports = CallSession;

lib/session/inbound-call-session.js

@@ -70,8 +70,12 @@ class InboundCallSession extends CallSession {
     this._hangup('caller');
   }
 
-  _jambonzHangup() {
-    this.dlg?.destroy();
+  _jambonzHangup(reason) {
+    this.dlg?.destroy({
+      headers: {
+        ...(reason && {'X-Reason': reason})
+      }
+    });
     // kill current task or wakeup the call session.
     this._callReleased();
   }

lib/tasks/config.js

@@ -187,18 +187,20 @@ class TaskConfig extends Task {
         : cs.speechRecognizerVendor;
       cs.speechRecognizerLabel = this.recognizer.label === 'default'
         ? cs.speechRecognizerLabel : this.recognizer.label;
-      cs.speechRecognizerLanguage = this.recognizer.language !== 'default'
+      cs.speechRecognizerLanguage = this.recognizer.language !== undefined && this.recognizer.language !== 'default'
         ? this.recognizer.language
         : cs.speechRecognizerLanguage;
 
       //fallback
-      cs.fallbackSpeechRecognizerVendor = this.recognizer.fallbackVendor !== 'default'
+      cs.fallbackSpeechRecognizerVendor = this.recognizer.fallbackVendor !== undefined &&
+        this.recognizer.fallbackVendor !== 'default'
         ? this.recognizer.fallbackVendor
         : cs.fallbackSpeechRecognizerVendor;
       cs.fallbackSpeechRecognizerLabel = this.recognizer.fallbackLabel === 'default' ?
         cs.fallbackSpeechRecognizerLabel :
         this.recognizer.fallbackLabel;
-      cs.fallbackSpeechRecognizerLanguage = this.recognizer.fallbackLanguage !== 'default'
+      cs.fallbackSpeechRecognizerLanguage = this.recognizer.fallbackLanguage !== undefined &&
+        this.recognizer.fallbackLanguage !== 'default'
         ? this.recognizer.fallbackLanguage
         : cs.fallbackSpeechRecognizerLanguage;
 

lib/tasks/dial.js

@@ -121,8 +121,9 @@ class TaskDial extends Task {
       }
     }
 
-    if (this.data.listen) {
-      this.listenTask = makeTask(logger, {'listen': this.data.listen}, this);
+    const listenData = this.data.listen || this.data.stream;
+    if (listenData) {
+      this.listenTask = makeTask(logger, {'listen': listenData }, this);
     }
     if (this.data.transcribe) {
       this.transcribeTask = makeTask(logger, {'transcribe' : this.data.transcribe}, this);
@@ -229,10 +230,10 @@ class TaskDial extends Task {
               try {
                 await this.epOther.play(this.dialMusic);
               } catch (err) {
-                this.logger.error(err, `TaskDial:exec error playing ${this.dialMusic}`);
+                this.logger.error(err, `TaskDial:exec error playing dialMusic ${this.dialMusic}`);
                 await sleepFor(1000);
               }
-            } while (!this.killed || !this.bridged);
+            } while (!this.killed && !this.bridged && this._mediaPath === MediaPath.FullMedia);
           })();
         }
       }
@@ -273,7 +274,9 @@ class TaskDial extends Task {
     this._removeDtmfDetection(this.dlg);
     await this._killOutdials();
     if (this.sd) {
-      this.sd.kill();
+      const byeReasonHeader = this.killReason === KillReason.MediaTimeout ? 'Media Timeout' : undefined;
+      this.sd.kill(byeReasonHeader);
+      this.sd.ep?.removeListener('destroy', this._handleMediaTimeout.bind(this));
       this.sd.removeAllListeners();
       this.sd = null;
     }
@@ -519,7 +522,7 @@ class TaskDial extends Task {
     const {req, callInfo, direction, srf} = cs;
     const {getSBC} = srf.locals;
     const {lookupTeamsByAccount, lookupAccountBySid} = srf.locals.dbHelpers;
-    const {lookupCarrier, lookupCarrierByPhoneNumber} = dbUtils(this.logger, cs.srf);
+    const {lookupCarrier, lookupCarrierByPhoneNumber, lookupVoipCarrierBySid} = dbUtils(this.logger, cs.srf);
     let sbcAddress = this.proxy || getSBC();
     const teamsInfo = {};
     let fqdn;
@@ -538,6 +541,8 @@ class TaskDial extends Task {
       ...this.headers
     };
 
+    // default to inband dtmf if not specified
+    this.inbandDtmfEnabled = cs.inbandDtmfEnabled;
     // get calling user from From header
     const parsedFrom = req.getParsedHeader('from');
     const fromUri = parseUri(parsedFrom.uri);
@@ -615,10 +620,17 @@ class TaskDial extends Task {
           const str = this.callerId || req.callingNumber || '';
           const callingNumber = str.startsWith('+') ? str.substring(1) : str;
           const voip_carrier_sid = await lookupCarrierByPhoneNumber(cs.accountSid, callingNumber);
+          const req_voip_carrier_sid = req.has('X-Voip-Carrier-Sid') ? req.get('X-Voip-Carrier-Sid') : null;
           if (voip_carrier_sid) {
             this.logger.info(
               `Dial:_attemptCalls: selected voip_carrier_sid ${voip_carrier_sid} for callingNumber: ${callingNumber}`);
             opts.headers['X-Requested-Carrier-Sid'] = voip_carrier_sid;
+
+            // Checking if outbound carrier is different from inbound carrier and has dtmf type tones
+            if (voip_carrier_sid !== req_voip_carrier_sid) {
+              const [voipCarrier] = await lookupVoipCarrierBySid(voip_carrier_sid);
+              this.inbandDtmfEnabled = voipCarrier?.dtmf_type === 'tones';
+            }
           }
         }
 
@@ -862,7 +874,7 @@ class TaskDial extends Task {
     if (cs.sipRequestWithinDialogHook) this._initSipIndialogRequestListener(cs, this.dlg);
 
     if (this.transcribeTask) this.transcribeTask.exec(cs, {ep: this.epOther, ep2:this.ep});
-    if (this.listenTask) this.listenTask.exec(cs, {ep: this.epOther});
+    if (this.listenTask) this.listenTask.exec(cs, {ep: this.listenTask.channel === 2 ? this.ep : this.epOther});
     if (this.startAmd) {
       try {
         this.startAmd(cs, this.ep, this, this.data.amd);
@@ -887,6 +899,14 @@ class TaskDial extends Task {
     if (this.canReleaseMedia || this.shouldExitMediaPathEntirely) {
       setTimeout(this._releaseMedia.bind(this, cs, sd, this.shouldExitMediaPathEntirely), 200);
     }
+
+    this.sd.ep.once('destroy', this._handleMediaTimeout.bind(this));
+  }
+
+  _handleMediaTimeout(evt) {
+    if (evt.reason === 'MEDIA_TIMEOUT' && this.sd && this.bridged) {
+      this.kill(this.cs, KillReason.MediaTimeout);
+    }
   }
 
   _bridgeEarlyMedia(sd) {

lib/tasks/gather.js

@@ -11,6 +11,7 @@ const {
   NvidiaTranscriptionEvents,
   JambonzTranscriptionEvents,
   AssemblyAiTranscriptionEvents,
+  VoxistTranscriptionEvents,
   VadDetection,
   VerbioTranscriptionEvents,
   SpeechmaticsTranscriptionEvents
@@ -24,6 +25,7 @@ const makeTask = require('./make_task');
 const assert = require('assert');
 const SttTask = require('./stt-task');
 const { SpeechCredentialError } = require('../utils/error');
+const SPEECHMATICS_DEFAULT_ASR_TIMEOUT = 1200;
 
 class TaskGather extends SttTask {
   constructor(logger, opts, parentTask) {
@@ -162,6 +164,16 @@ class TaskGather extends SttTask {
       this.logger.debug({hints: this.data.recognizer.hints, hintsBoost: this.data.recognizer.hintsBoost},
         'Gather:exec - applying global sttHints');
     }
+
+    // specials case for speechmatics: they dont do endpointing so we need to enable continuous ASR
+    if (this.vendor === 'speechmatics' && !this.isContinuousAsr) {
+      const maxDelay = this.recognizer?.speechmaticsOptions?.transcription_config?.max_delay;
+      if (maxDelay) this.asrTimeout = Math.min(SPEECHMATICS_DEFAULT_ASR_TIMEOUT, maxDelay * 1000);
+      else this.asrTimeout = SPEECHMATICS_DEFAULT_ASR_TIMEOUT;
+      this.isContinuousAsr = true;
+      this.logger.debug(`Gather:exec - auto-enabling continuous ASR for speechmatics w/ timeout ${this.asrTimeout}`);
+    }
+
     if (!this.isContinuousAsr && cs.isContinuousAsr) {
       this.isContinuousAsr = true;
       this.asrTimeout = cs.asrTimeout * 1000;
@@ -513,6 +525,17 @@ class TaskGather extends SttTask {
           this._onVendorConnectFailure.bind(this, cs, ep));
         break;
 
+      case 'voxist':
+        this.bugname = `${this.bugname_prefix}voxist_transcribe`;
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.Transcription,
+          this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(
+          ep, VoxistTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.ConnectFailure,
+          this._onVendorConnectFailure.bind(this, cs, ep));
+        break;
+
       case 'speechmatics':
         this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
         this.addCustomEventListener(
@@ -613,6 +636,13 @@ class TaskGather extends SttTask {
     this._asrTimer = setTimeout(() => {
       this.logger.debug('_startAsrTimer - asr timer went off');
       const evt = this.consolidateTranscripts(this._bufferedTranscripts, 1, this.language, this.vendor);
+
+      /* special case for speechmatics - keep listening if we dont have any transcripts */
+      if (this.vendor === 'speechmatics' && this._bufferedTranscripts.length === 0) {
+        this.logger.debug('Gather:_startAsrTimer - speechmatics, no transcripts yet, keep listening');
+        this._startAsrTimer();
+        return;
+      }
       this._resolve(this._bufferedTranscripts.length > 0 ? 'speech' : 'timeout', evt);
     }, this.asrTimeout);
     this.logger.debug(`_startAsrTimer: set for ${this.asrTimeout}ms`);
@@ -755,10 +785,16 @@ class TaskGather extends SttTask {
         this.logger.debug('Gather:_onTranscription - got UtteranceEnd event from deepgram but no buffered transcripts');
       }
       else {
-        this.logger.debug('Gather:_onTranscription - got UtteranceEnd event from deepgram, return buffered transcript');
-        evt = this.consolidateTranscripts(this._bufferedTranscripts, 1, this.language, this.vendor);
-        this._bufferedTranscripts = [];
-        this._resolve('speech', evt);
+        const utteranceTime = evt.last_word_end;
+        if (utteranceTime && this._dgTimeOfLastUnprocessedWord && utteranceTime < this._dgTimeOfLastUnprocessedWord) {
+          this.logger.debug('Gather:_onTranscription - got UtteranceEnd with unprocessed words, continue listening');
+        }
+        else {
+          this.logger.debug('Gather:_onTranscription - got UtteranceEnd from deepgram, return buffered transcript');
+          evt = this.consolidateTranscripts(this._bufferedTranscripts, 1, this.language, this.vendor);
+          this._bufferedTranscripts = [];
+          this._resolve('speech', evt);
+        }
       }
       return;
     }
@@ -769,7 +805,7 @@ class TaskGather extends SttTask {
 
     evt = this.normalizeTranscription(evt, this.vendor, 1, this.language,
       this.shortUtterance, this.data.recognizer.punctuation);
-    this.logger.debug({evt, bugname, finished, vendor: this.vendor}, 'Gather:_onTranscription normalized transcript');
+    //this.logger.debug({evt, bugname, finished, vendor: this.vendor}, 'Gather:_onTranscription normalized transcript');
 
     if (evt.alternatives.length === 0) {
       this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, continue listening');
@@ -777,8 +813,6 @@ class TaskGather extends SttTask {
     }
     const confidence = evt.alternatives[0].confidence;
     const minConfidence = this.data.recognizer?.minConfidence;
-    this.logger.debug({evt},
-      `TaskGather:_onTranscription - confidence (${confidence}), minConfidence (${minConfidence})`);
     if (confidence && minConfidence && confidence < minConfidence) {
       this.logger.info({evt},
         'TaskGather:_onTranscription - Transcript confidence ' +
@@ -832,7 +866,7 @@ class TaskGather extends SttTask {
         const t = evt.alternatives[0].transcript;
         if (t) {
           /* remove trailing punctuation */
-          if (/[,;:\.!\?]$/.test(t)) {
+          if (this.vendor !== 'speechmatics' && /[,;:\.!\?]$/.test(t)) {
             this.logger.debug('TaskGather:_onTranscription - removing trailing punctuation');
             evt.alternatives[0].transcript = t.slice(0, -1);
           }
@@ -894,8 +928,21 @@ class TaskGather extends SttTask {
         if (originalEvent.is_final && evt.alternatives[0].transcript !== '') {
           this.logger.debug({evt}, 'Gather:_onTranscription - buffering a completed (partial) deepgram transcript');
           this._bufferedTranscripts.push(evt);
+          this._dgTimeOfLastUnprocessedWord = null;
+        }
+        if (evt.alternatives[0].transcript === '') {
+          emptyTranscript = true;
+        }
+        else if (!originalEvent.is_final) {
+          /* Deepgram: we have unprocessed words-save last word end time so we can later compare to UtteranceEnd */
+          const words = originalEvent.channel.alternatives[0].words;
+          if (words?.length > 0) {
+            this._dgTimeOfLastUnprocessedWord = words.slice(-1)[0].end;
+            this.logger.debug(
+              `TaskGather:_onTranscription - saving word end time: ${this._dgTimeOfLastUnprocessedWord}`);
+          }
+
         }
-        if (evt.alternatives[0].transcript === '') emptyTranscript = true;
       }
 
       if (!emptyTranscript) {
@@ -1165,7 +1212,7 @@ class TaskGather extends SttTask {
         if (this.parentTask) this.parentTask.emit('stt-low-confidence', evt);
         else {
           this.emit('stt-low-confidence', evt);
-          returnedVerbs = await this.performAction({reason: 'stt-low-confidence'});
+          returnedVerbs = await this.performAction({speech:evt, reason: 'stt-low-confidence'});
         }
       }
     } catch (err) {  /*already logged error*/ }

lib/tasks/listen.js

@@ -17,7 +17,7 @@ class TaskListen extends Task {
 
     [
       'action', 'auth', 'method', 'url', 'finishOnKey', 'maxLength', 'metadata', 'mixType', 'passDtmf', 'playBeep',
-      'sampleRate', 'timeout', 'transcribe', 'wsAuth', 'disableBidirectionalAudio'
+      'sampleRate', 'timeout', 'transcribe', 'wsAuth', 'disableBidirectionalAudio', 'channel'
     ].forEach((k) => this[k] = this.data[k]);
 
     this.mixType = this.mixType || 'mono';

lib/tasks/llm/index.js

@@ -2,6 +2,8 @@ const Task = require('../task');
 const {TaskPreconditions} = require('../../utils/constants');
 const TaskLlmOpenAI_S2S = require('./llms/openai_s2s');
 const TaskLlmVoiceAgent_S2S = require('./llms/voice_agent_s2s');
+const TaskLlmUltravox_S2S = require('./llms/ultravox_s2s');
+const TaskLlmElevenlabs_S2S = require('./llms/elevenlabs_s2s');
 
 class TaskLlm extends Task {
   constructor(logger, opts) {
@@ -41,9 +43,7 @@ class TaskLlm extends Task {
     switch (this.vendor) {
       case 'openai':
       case 'microsoft':
-        if (this.model.startsWith('gpt-4o-realtime')) {
-          llm = new TaskLlmOpenAI_S2S(this.logger, this.data, this);
-        }
+        llm = new TaskLlmOpenAI_S2S(this.logger, this.data, this);
         break;
 
       case 'voiceagent':
@@ -51,6 +51,14 @@ class TaskLlm extends Task {
         llm = new TaskLlmVoiceAgent_S2S(this.logger, this.data, this);
         break;
 
+      case 'ultravox':
+        llm = new TaskLlmUltravox_S2S(this.logger, this.data, this);
+        break;
+
+      case 'elevenlabs':
+        llm = new TaskLlmElevenlabs_S2S(this.logger, this.data, this);
+        break;
+
       default:
         throw new Error(`Unsupported vendor ${this.vendor} for LLM`);
     }

lib/tasks/llm/llms/elevenlabs_s2s.js

@@ -0,0 +1,302 @@
+const Task = require('../../task');
+const TaskName = 'Llm_Elevenlabs_s2s';
+const {LlmEvents_Elevenlabs} = require('../../../utils/constants');
+const {request} = require('undici');
+const ClientEvent = 'client.event';
+const SessionDelete = 'session.delete';
+
+const elevenlabs_server_events = [
+  'conversation_initiation_metadata',
+  'user_transcript',
+  'agent_response',
+  'client_tool_call'
+];
+
+const expandWildcards = (events) => {
+  const expandedEvents = [];
+
+  events.forEach((evt) => {
+    if (evt.endsWith('.*')) {
+      const prefix = evt.slice(0, -2); // Remove the wildcard ".*"
+      const matchingEvents = elevenlabs_server_events.filter((e) => e.startsWith(prefix));
+      expandedEvents.push(...matchingEvents);
+    } else {
+      expandedEvents.push(evt);
+    }
+  });
+
+  return expandedEvents;
+};
+
+class TaskLlmElevenlabs_S2S extends Task {
+  constructor(logger, opts, parentTask) {
+    super(logger, opts, parentTask);
+    this.parent = parentTask;
+
+    this.vendor = this.parent.vendor;
+    this.auth = this.parent.auth;
+
+    const {agent_id, api_key} = this.auth || {};
+    if (!agent_id) throw new Error('auth.agent_id is required for Elevenlabs S2S');
+
+    this.agent_id = agent_id;
+    this.api_key = api_key;
+    this.actionHook = this.data.actionHook;
+    this.eventHook = this.data.eventHook;
+    this.toolHook = this.data.toolHook;
+    const {
+      conversation_initiation_client_data,
+      input_sample_rate = 16000,
+      output_sample_rate = 16000
+    } = this.data.llmOptions;
+    this.conversation_initiation_client_data = conversation_initiation_client_data;
+    this.input_sample_rate = input_sample_rate;
+    this.output_sample_rate = output_sample_rate;
+    this.results = {
+      completionReason: 'normal conversation end'
+    };
+
+    /**
+     * only one of these will have items,
+     * if includeEvents, then these are the events to include
+     * if excludeEvents, then these are the events to exclude
+     */
+    this.includeEvents = [];
+    this.excludeEvents = [];
+
+    /* default to all events if user did not specify */
+    this._populateEvents(this.data.events || elevenlabs_server_events);
+
+    this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
+    this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
+  }
+
+  get name() { return TaskName; }
+
+  async getSignedUrl() {
+    if (!this.api_key) {
+      return {
+        host: 'api.elevenlabs.io',
+        path: `/v1/convai/conversation?agent_id=${this.agent_id}`,
+      };
+    }
+
+    const {statusCode, body} = await request(
+      `https://api.elevenlabs.io/v1/convai/conversation/get_signed_url?agent_id=${this.agent_id}`, {
+        method: 'GET',
+        headers: {
+          'xi-api-key': this.api_key
+        },
+      }
+    );
+    const data = await body.json();
+    if (statusCode !== 200 || !data?.signed_url) {
+      this.logger.error({statusCode, data}, 'Elevenlabs Error registering call');
+      throw new Error(`Elevenlabs  Error registering call: ${data.message}`);
+    }
+
+    const url = new URL(data.signed_url);
+    return {
+      host: url.hostname,
+      path: url.pathname + url.search,
+    };
+  }
+
+  async _api(ep, args) {
+    const res = await ep.api('uuid_elevenlabs_s2s', `^^|${args.join('|')}`);
+    if (!res.body?.startsWith('+OK')) {
+      throw new Error({args}, `Error calling uuid_elevenlabs_s2s: ${res.body}`);
+    }
+  }
+
+  async exec(cs, {ep}) {
+    await super.exec(cs);
+    await this._startListening(cs, ep);
+
+    await this.awaitTaskDone();
+
+    /* note: the parent llm verb started the span, which is why this is necessary */
+    await this.parent.performAction(this.results);
+
+    this._unregisterHandlers();
+  }
+
+  async kill(cs) {
+    super.kill(cs);
+
+    this._api(cs.ep, [cs.ep.uuid, SessionDelete])
+      .catch((err) => this.logger.info({err}, 'TaskLlmElevenlabs_S2S:kill - error deleting session'));
+
+    this.notifyTaskDone();
+  }
+
+  /**
+   * Send function call output to the Elevenlabs server in the form of conversation.item.create
+   * per https://elevenlabs.io/docs/conversational-ai/api-reference/conversational-ai/websocket
+   */
+  async processToolOutput(ep, tool_call_id, rawData) {
+    try {
+      const {data} = rawData;
+      this.logger.debug({tool_call_id, data}, 'TaskLlmElevenlabs_S2S:processToolOutput');
+
+      if (!data.type || data.type !== 'client_tool_result') {
+        this.logger.info({data},
+          'TaskLlmElevenlabs_S2S:processToolOutput - invalid tool output, must be client_tool_result');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+      }
+    } catch (err) {
+      this.logger.info({err}, 'TaskLlmElevenlabs_S2S:processToolOutput');
+    }
+  }
+
+  /**
+   * Send a session.update to the Elevenlabs server
+   * Note: creating and deleting conversation items also supported as well as interrupting the assistant
+   */
+  async processLlmUpdate(ep, data, _callSid) {
+    this.logger.debug({data, _callSid}, 'TaskLlmElevenlabs_S2S:processLlmUpdate, ignored');
+  }
+
+  async _startListening(cs, ep) {
+    this._registerHandlers(ep);
+
+    try {
+      const {host, path} = await this.getSignedUrl();
+      const args = [ep.uuid, 'session.create', this.input_sample_rate, this.output_sample_rate, host, path];
+      await this._api(ep, args);
+    } catch (err) {
+      this.logger.error({err}, 'TaskLlmElevenlabs_S2S:_startListening');
+      this.notifyTaskDone();
+    }
+  }
+
+  async _sendClientEvent(ep, obj) {
+    let ok = true;
+    this.logger.debug({obj}, 'TaskLlmElevenlabs_S2S:_sendClientEvent');
+    try {
+      const args = [ep.uuid, ClientEvent, JSON.stringify(obj)];
+      await this._api(ep, args);
+    } catch (err) {
+      ok = false;
+      this.logger.error({err}, 'TaskLlmElevenlabs_S2S:_sendClientEvent - Error');
+    }
+    return ok;
+  }
+
+  async _sendInitialMessage(ep) {
+    if (this.conversation_initiation_client_data) {
+      if (!await this._sendClientEvent(ep, {
+        type: 'conversation_initiation_client_data',
+        conversation_initiation_client_data: this.conversation_initiation_client_data
+      })) {
+        this.notifyTaskDone();
+      }
+    }
+  }
+
+  _registerHandlers(ep) {
+    this.addCustomEventListener(ep, LlmEvents_Elevenlabs.Connect, this._onConnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Elevenlabs.ConnectFailure, this._onConnectFailure.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Elevenlabs.Disconnect, this._onDisconnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Elevenlabs.ServerEvent, this._onServerEvent.bind(this, ep));
+  }
+
+  _unregisterHandlers() {
+    this.removeCustomEventListeners();
+  }
+
+  _onError(ep, evt) {
+    this.logger.info({evt}, 'TaskLlmElevenlabs_S2S:_onError');
+    this.notifyTaskDone();
+  }
+
+  _onConnect(ep) {
+    this.logger.debug('TaskLlmElevenlabs_S2S:_onConnect');
+    this._sendInitialMessage(ep);
+  }
+  _onConnectFailure(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmElevenlabs_S2S:_onConnectFailure');
+    this.results = {completionReason: 'connection failure'};
+    this.notifyTaskDone();
+  }
+  _onDisconnect(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmElevenlabs_S2S:_onConnectFailure');
+    this.results = {completionReason: 'disconnect from remote end'};
+    this.notifyTaskDone();
+  }
+  async _onServerEvent(ep, evt) {
+    let endConversation = false;
+    const type = evt.type;
+    this.logger.info({evt}, 'TaskLlmElevenlabs_S2S:_onServerEvent');
+
+    if (type === 'error') {
+      endConversation = true;
+      this.results = {
+        completionReason: 'server error',
+        error: evt.error
+      };
+    }
+
+    /* tool calls */
+    else if (type === 'client_tool_call') {
+      this.logger.debug({evt}, 'TaskLlmElevenlabs_S2S:_onServerEvent - function_call');
+      if (!this.toolHook) {
+        this.logger.warn({evt}, 'TaskLlmElevenlabs_S2S:_onServerEvent - no toolHook defined!');
+      }
+      else {
+        const {client_tool_call} = evt;
+        const {tool_name: name, tool_call_id: call_id, parameters: args} = client_tool_call;
+
+        try {
+          await this.parent.sendToolHook(call_id, {name, args});
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmElevenlabs_S2S - error calling function');
+          this.results = {
+            completionReason: 'client error calling function',
+            error: err
+          };
+          endConversation = true;
+        }
+      }
+    }
+
+    /* check whether we should notify on this event */
+    if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
+      this.parent.sendEventHook(evt)
+        .catch((err) => this.logger.info({err},
+          'TaskLlmElevenlabs_S2S:_onServerEvent - error sending event hook'));
+    }
+
+    if (endConversation) {
+      this.logger.info({results: this.results},
+        'TaskLlmElevenlabs_S2S:_onServerEvent - ending conversation due to error');
+      this.notifyTaskDone();
+    }
+  }
+
+  _populateEvents(events) {
+    if (events.includes('all')) {
+      /* work by excluding specific events */
+      const exclude = events
+        .filter((evt) => evt.startsWith('-'))
+        .map((evt) => evt.slice(1));
+      if (exclude.length === 0) this.includeEvents = elevenlabs_server_events;
+      else this.excludeEvents = expandWildcards(exclude);
+    }
+    else {
+      /* work by including specific events */
+      const include = events
+        .filter((evt) => !evt.startsWith('-'));
+      this.includeEvents = expandWildcards(include);
+    }
+
+    this.logger.debug({
+      includeEvents: this.includeEvents,
+      excludeEvents: this.excludeEvents
+    }, 'TaskLlmElevenlabs_S2S:_populateEvents');
+  }
+}
+
+module.exports = TaskLlmElevenlabs_S2S;

lib/tasks/llm/llms/openai_s2s.js

@@ -120,7 +120,7 @@ class TaskLlmOpenAI_S2S extends Task {
 
     switch (this.vendor) {
       case 'openai':
-        return 'v1/realtime?model=${this.model}';
+        return `v1/realtime?model=${this.model}`;
       case 'microsoft':
         return `openai/realtime?api-version=2024-10-01-preview&deployment=${this.model}`;
     }

lib/tasks/llm/llms/ultravox_s2s.js

@@ -0,0 +1,245 @@
+const Task = require('../../task');
+const TaskName = 'Llm_Ultravox_s2s';
+const {request} = require('undici');
+const {LlmEvents_Ultravox} = require('../../../utils/constants');
+
+const ultravox_server_events = [
+  'pong',
+  'state',
+  'transcript',
+  'conversationText',
+  'clientToolInvocation',
+  'playbackClearBuffer',
+];
+
+const ClientEvent = 'client.event';
+
+const expandWildcards = (events) => {
+  // no-op for deepgram
+  return events;
+};
+
+const SessionDelete = 'session.delete';
+
+class TaskLlmUltravox_S2S extends Task {
+  constructor(logger, opts, parentTask) {
+    super(logger, opts, parentTask);
+    this.parent = parentTask;
+
+    this.vendor = this.parent.vendor;
+    this.model = this.parent.model || 'fixie-ai/ultravox';
+    this.auth = this.parent.auth;
+    this.connectionOptions = this.parent.connectOptions;
+
+    const {apiKey} = this.auth || {};
+    if (!apiKey) throw new Error('auth.apiKey is required for Vendor: Ultravox');
+    this.apiKey = apiKey;
+    this.actionHook = this.data.actionHook;
+    this.eventHook = this.data.eventHook;
+    this.toolHook = this.data.toolHook;
+
+    /**
+     * only one of these will have items,
+     * if includeEvents, then these are the events to include
+     * if excludeEvents, then these are the events to exclude
+     */
+    this.includeEvents = [];
+    this.excludeEvents = [];
+
+    /* default to all events if user did not specify */
+    this._populateEvents(this.data.events || ultravox_server_events);
+
+    this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
+    this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
+  }
+
+  get name() { return TaskName; }
+
+  async _api(ep, args) {
+    const res = await ep.api('uuid_ultravox_s2s', `^^|${args.join('|')}`);
+    if (!res.body?.startsWith('+OK')) {
+      throw new Error(`Error calling uuid_ultravox_s2s: ${JSON.stringify(res.body)}`);
+    }
+  }
+
+  async createCall() {
+    const payload = {
+      ...this.data.llmOptions,
+      model: this.model,
+      medium: {
+        ...(this.data.llmOptions.medium || {}),
+        serverWebSocket: {
+          inputSampleRate: 8000,
+          outputSampleRate: 8000,
+        }
+      }
+    };
+    const {statusCode, body} = await request('https://api.ultravox.ai/api/calls', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'X-API-Key': this.apiKey
+      },
+      body: JSON.stringify(payload)
+    });
+    const data = await body.json();
+    if (statusCode !== 201 || !data?.joinUrl) {
+      this.logger.error({statusCode, data}, 'Ultravox Error registering call');
+      throw new Error(`Ultravox  Error registering call: ${data.message}`);
+    }
+    this.logger.info({joinUrl: data.joinUrl}, 'Ultravox Call registered');
+    return data.joinUrl;
+  }
+
+  _unregisterHandlers() {
+    this.removeCustomEventListeners();
+  }
+
+  _registerHandlers(ep) {
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.Connect, this._onConnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.ConnectFailure, this._onConnectFailure.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.Disconnect, this._onDisconnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.ServerEvent, this._onServerEvent.bind(this, ep));
+  }
+
+  async _startListening(cs, ep) {
+    this._registerHandlers(ep);
+
+    const joinUrl = await this.createCall();
+    // split the joinUrl into host and path
+    const {host, pathname, search} = new URL(joinUrl);
+
+    try {
+      const args = [ep.uuid, 'session.create', host, pathname + search];
+      await this._api(ep, args);
+    } catch (err) {
+      this.logger.error({err}, 'TaskLlmUltraVox_S2S:_startListening');
+      this.notifyTaskDone();
+    }
+  }
+
+  async exec(cs, {ep}) {
+    await super.exec(cs);
+
+    await this._startListening(cs, ep);
+
+    await this.awaitTaskDone();
+
+    /* note: the parent llm verb started the span, which is why this is necessary */
+    await this.parent.performAction(this.results);
+
+    this._unregisterHandlers();
+  }
+
+  async kill(cs) {
+    super.kill(cs);
+
+    this._api(cs.ep, [cs.ep.uuid, SessionDelete])
+      .catch((err) => this.logger.info({err}, 'TaskLlmUltravox_S2S:kill - error deleting session'));
+
+    this.notifyTaskDone();
+  }
+
+  _onConnect(ep) {
+    this.logger.debug('TaskLlmUltravox_S2S:_onConnect');
+  }
+  _onConnectFailure(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmUltravox_S2S:_onConnectFailure');
+    this.results = {completionReason: 'connection failure'};
+    this.notifyTaskDone();
+  }
+  _onDisconnect(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmUltravox_S2S:_onConnectFailure');
+    this.results = {completionReason: 'disconnect from remote end'};
+    this.notifyTaskDone();
+  }
+
+  async _onServerEvent(_ep, evt) {
+    let endConversation = false;
+    const type = evt.type;
+    this.logger.info({evt}, 'TaskLlmUltravox_S2S:_onServerEvent');
+
+    /* server errors of some sort */
+    if (type === 'error') {
+      endConversation = true;
+      this.results = {
+        completionReason: 'server error',
+        error: evt.error
+      };
+    }
+
+    /* tool calls */
+    else if (type === 'client_tool_invocation') {
+      this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent - function_call');
+      if (!this.toolHook) {
+        this.logger.warn({evt}, 'TaskLlmUltravox_S2S:_onServerEvent - no toolHook defined!');
+      }
+      else {
+        const {toolName: name, invocationId: call_id, parameters: args} = evt;
+
+        try {
+          await this.parent.sendToolHook(call_id, {name, args});
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmUltravox_S2S - error calling function');
+          this.results = {
+            completionReason: 'client error calling function',
+            error: err
+          };
+          endConversation = true;
+        }
+      }
+    }
+
+    /* check whether we should notify on this event */
+    if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
+      this.parent.sendEventHook(evt)
+        .catch((err) => this.logger.info({err}, 'TaskLlmUltravox_S2S:_onServerEvent - error sending event hook'));
+    }
+
+    if (endConversation) {
+      this.logger.info({results: this.results},
+        'TaskLlmUltravox_S2S:_onServerEvent - ending conversation due to error');
+      this.notifyTaskDone();
+    }
+  }
+
+  async processToolOutput(ep, tool_call_id, data) {
+    try {
+      this.logger.debug({tool_call_id, data}, 'TaskLlmUltravox_S2S:processToolOutput');
+
+      if (!data.type || data.type !== 'client_tool_result') {
+        this.logger.info({data},
+          'TaskLlmUltravox_S2S:processToolOutput - invalid tool output, must be client_tool_result');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+      }
+    } catch (err) {
+      this.logger.info({err}, 'TaskLlmUltravox_S2S:processToolOutput');
+    }
+  }
+
+  _populateEvents(events) {
+    if (events.includes('all')) {
+      /* work by excluding specific events */
+      const exclude = events
+        .filter((evt) => evt.startsWith('-'))
+        .map((evt) => evt.slice(1));
+      if (exclude.length === 0) this.includeEvents = ultravox_server_events;
+      else this.excludeEvents = expandWildcards(exclude);
+    }
+    else {
+      /* work by including specific events */
+      const include = events
+        .filter((evt) => !evt.startsWith('-'));
+      this.includeEvents = expandWildcards(include);
+    }
+
+    this.logger.debug({
+      includeEvents: this.includeEvents,
+      excludeEvents: this.excludeEvents
+    }, 'TaskLlmUltravox_S2S:_populateEvents');
+  }
+}
+
+module.exports = TaskLlmUltravox_S2S;

lib/tasks/llm/llms/voice_agent_s2s.js

@@ -98,7 +98,7 @@ class TaskLlmVoiceAgent_S2S extends Task {
   async _api(ep, args) {
     const res = await ep.api('uuid_voice_agent_s2s', `^^|${args.join('|')}`);
     if (!res.body?.startsWith('+OK')) {
-      throw new Error({args}, `Error calling uuid_voice_agent_s2s: ${JSON.stringify(res.body)}`);
+      throw new Error(`Error calling uuid_voice_agent_s2s: ${JSON.stringify(res.body)}`);
     }
   }
 

lib/tasks/make_task.js

@@ -84,6 +84,7 @@ function makeTask(logger, obj, parent) {
       const TaskTranscribe = require('./transcribe');
       return new TaskTranscribe(logger, data, parent);
     case TaskName.Listen:
+    case TaskName.Stream:
       const TaskListen = require('./listen');
       return new TaskListen(logger, data, parent);
     case TaskName.Redirect:

lib/tasks/redirect.js

@@ -1,5 +1,6 @@
 const Task = require('./task');
 const {TaskName} = require('../utils/constants');
+const WsRequestor = require('../utils/ws-requestor');
 
 /**
  * Redirects to a new application
@@ -13,6 +14,17 @@ class TaskRedirect extends Task {
 
   async exec(cs) {
     await super.exec(cs);
+
+    if (cs.requestor instanceof WsRequestor && cs.application.requestor._isAbsoluteUrl(this.actionHook)) {
+      this.logger.info(`Task:performAction redirecting to ${this.actionHook}, requires new ws connection`);
+      try {
+        this.cs.requestor.close();
+        const requestor = new WsRequestor(this.logger, cs.accountSid, {url: this.actionHook}, this.webhook_secret) ;
+        this.cs.application.requestor = requestor;
+      } catch (err) {
+        this.logger.info(err, `Task:performAction error redirecting to ${this.actionHook}`);
+      }
+    }
     await this.performAction();
   }
 }

lib/tasks/rest_dial.js

@@ -12,6 +12,7 @@ class TaskRestDial extends Task {
 
     this.from = this.data.from;
     this.callerName = this.data.callerName;
+    this.timeLimit = this.data.timeLimit;
     this.fromHost = this.data.fromHost;
     this.to = this.data.to;
     this.call_hook = this.data.call_hook;
@@ -66,6 +67,9 @@ class TaskRestDial extends Task {
     const cs = this.callSession;
     cs.setDialog(dlg);
     cs.referHook = this.referHook;
+    if (this.timeLimit) {
+      cs.startMaxCallDurationTimer(this.timeLimit);
+    }
     this.logger.debug('TaskRestDial:_onConnect - call connected');
     if (this.sipRequestWithinDialogHook) this._initSipRequestWithinDialogHandler(cs, dlg);
     try {

lib/tasks/say.js

@@ -213,7 +213,7 @@ class TaskSay extends TtsTask {
           ep.once('playback-start', (evt) => {
             this.logger.debug({evt}, 'Say got playback-start');
             if (this.otelSpan) {
-              this._addStreamingTtsAttributes(this.otelSpan, evt);
+              this._addStreamingTtsAttributes(this.otelSpan, evt, vendor);
               this.otelSpan.end();
               this.otelSpan = null;
               if (evt.variable_tts_cache_filename) {
@@ -240,6 +240,7 @@ class TaskSay extends TtsTask {
                 language,
                 voice,
                 engine,
+                model: this.model || this.model_id,
                 text
               }).catch((err) => this.logger.info({err}, 'Error adding file to cache'));
             }
@@ -307,7 +308,7 @@ class TaskSay extends TtsTask {
     this.notifyTaskDone();
   }
 
-  _addStreamingTtsAttributes(span, evt) {
+  _addStreamingTtsAttributes(span, evt, vendor) {
     const attrs = {'tts.cached': false};
     for (const [key, value] of Object.entries(evt)) {
       if (key.startsWith('variable_tts_')) {
@@ -321,6 +322,9 @@ class TaskSay extends TtsTask {
           .replace('elevenlabs_', 'elevenlabs.');
         if (spanMapping[newKey]) newKey = spanMapping[newKey];
         attrs[newKey] = value;
+        if (key === 'variable_tts_time_to_first_byte_ms' && value) {
+          this.cs.srf.locals.stats.histogram('tts.response_time', value, [`vendor:${vendor}`]);
+        }
       }
     }
     delete attrs['cache_filename']; //no value in adding this to the span

lib/tasks/transcribe.js

@@ -13,6 +13,7 @@ const {
   JambonzTranscriptionEvents,
   TranscribeStatus,
   AssemblyAiTranscriptionEvents,
+  VoxistTranscriptionEvents,
   VerbioTranscriptionEvents,
   SpeechmaticsTranscriptionEvents
 } = require('../utils/constants.json');
@@ -300,6 +301,17 @@ class TaskTranscribe extends SttTask {
           this._onVendorConnectFailure.bind(this, cs, ep, channel));
         break;
 
+      case 'voxist':
+        this.bugname = `${this.bugname_prefix}voxist_transcribe`;
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.Transcription,
+          this._onTranscription.bind(this, cs, ep, channel));
+        this.addCustomEventListener(ep,
+          VoxistTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.ConnectFailure,
+          this._onVendorConnectFailure.bind(this, cs, ep, channel));
+        break;
+
       case 'speechmatics':
         this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
         this.addCustomEventListener(

lib/tasks/tts-task.js

@@ -59,7 +59,7 @@ class TtsTask extends Task {
   }
 
   async setTtsStreamingChannelVars(vendor, language, voice, credentials, ep) {
-    const {api_key, model_id} = credentials;
+    const {api_key, model_id, custom_tts_streaming_url, auth_token} = credentials;
     const {stability, similarity_boost, use_speaker_boost, style} = this.options;
     let obj;
 
@@ -94,8 +94,36 @@ class TtsTask extends Task {
           ...(style && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_STYLE: style})
         };
         break;
+      case 'rimelabs':
+        const {
+          pauseBetweenBrackets, phonemizeBetweenBrackets, inlineSpeedAlpha, speedAlpha, reduceLatency
+        } = this.options;
+        obj = {
+          RIMELABS_API_KEY: api_key,
+          RIMELABS_TTS_STREAMING_MODEL_ID: model_id,
+          RIMELABS_TTS_STREAMING_VOICE_ID: voice,
+          RIMELABS_TTS_STREAMING_LANGUAGE: language || 'en',
+          ...(pauseBetweenBrackets && {RIMELABS_TTS_STREAMING_PAUSE_BETWEEN_BRACKETS: pauseBetweenBrackets}),
+          ...(phonemizeBetweenBrackets &&
+            {RIMELABS_TTS_STREAMING_PHONEMIZE_BETWEEN_BRACKETS: phonemizeBetweenBrackets}),
+          ...(inlineSpeedAlpha && {RIMELABS_TTS_STREAMING_INLINE_SPEED_ALPHA: inlineSpeedAlpha}),
+          ...(speedAlpha && {RIMELABS_TTS_STREAMING_SPEED_ALPHA: speedAlpha}),
+          ...(reduceLatency && {RIMELABS_TTS_STREAMING_REDUCE_LATENCY: reduceLatency})
+        };
+        break;
       default:
-        throw new Error(`vendor ${vendor} is not supported for tts streaming yet`);
+        if (vendor.startsWith('custom:')) {
+          const use_tls = custom_tts_streaming_url.startsWith('wss://');
+          obj = {
+            CUSTOM_TTS_STREAMING_HOST: custom_tts_streaming_url.replace(/^(ws|wss):\/\//, ''),
+            CUSTOM_TTS_STREAMING_API_KEY: auth_token,
+            CUSTOM_TTS_STREAMING_VOICE_ID: voice,
+            CUSTOM_TTS_STREAMING_LANGUAGE: language || 'en',
+            CUSTOM_TTS_STREAMING_USE_TLS: use_tls
+          };
+        } else {
+          throw new Error(`vendor ${vendor} is not supported for tts streaming yet`);
+        }
     }
     this.logger.info({vendor, credentials, obj}, 'setTtsStreamingChannelVars');
 
@@ -115,16 +143,16 @@ class TtsTask extends Task {
         `No text-to-speech service credentials for ${vendor} with labels: ${label} have been configured`);
     }
     /* parse Nuance voices into name and model */
-    let model;
     if (vendor === 'nuance' && voice) {
       const arr = /([A-Za-z-]*)\s+-\s+(enhanced|standard)/.exec(voice);
       if (arr) {
         voice = arr[1];
-        model = arr[2];
+        this.model = arr[2];
       }
     } else if (vendor === 'deepgram') {
-      model = voice;
+      this.model = voice;
     }
+    this.model_id = credentials.model_id;
 
     /* allow for microsoft custom region voice and api_key to be specified as an override */
     if (vendor === 'microsoft' && this.options.deploymentId) {
@@ -187,7 +215,8 @@ class TtsTask extends Task {
     // If vendor is changed from the previous one, then reset the cache_speech_handles flag
     //cs.currentTtsVendor = vendor;
 
-    if (!preCache && !this._disableTracing) this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
+    if (!preCache && !this._disableTracing)
+      this.logger.info({vendor, language, voice, model: this.model}, 'TaskSay:exec');
     try {
       if (!credentials) {
         writeAlerts({
@@ -222,7 +251,7 @@ class TtsTask extends Task {
             language,
             voice,
             engine,
-            model,
+            model: this.model,
             salt,
             credentials,
             options: this.options,

lib/utils/background-task-manager.js

@@ -103,6 +103,7 @@ class BackgroundTaskManager extends Emitter {
   async _initBargeIn(opts) {
     let task;
     try {
+      const copy = JSON.parse(JSON.stringify(opts));
       const t = normalizeJambones(this.logger, [opts]);
       task = makeTask(this.logger, t[0]);
       task
@@ -121,7 +122,7 @@ class BackgroundTaskManager extends Emitter {
           if (task.sticky && !this.cs.callGone && !this.cs._stopping) {
             this.logger.info('BackgroundTaskManager:_initBargeIn: restarting background bargeIn');
             this._bargeInHandled = false;
-            this.newTask('bargeIn', opts, true);
+            this.newTask('bargeIn', copy, true);
           }
           return;
         })

lib/utils/constants.json

@@ -28,6 +28,7 @@
     "SipRedirect": "sip:redirect",
     "Say": "say",
     "SayLegacy": "say:legacy",
+    "Stream": "stream",
     "Tag": "tag",
     "Transcribe": "transcribe"
   },
@@ -148,6 +149,12 @@
     "ConnectFailure": "assemblyai_transcribe::connect_failed",
     "Connect": "assemblyai_transcribe::connect"
   },
+  "VoxistTranscriptionEvents": {
+    "Transcription": "voxist_transcribe::transcription",
+    "Error": "voxist_transcribe::error",
+    "ConnectFailure": "voxist_transcribe::connect_failed",
+    "Connect": "voxist_transcribe::connect"
+  },
   "VadDetection": {
     "Detection": "vad_detect:detection"
   },
@@ -175,6 +182,13 @@
     "Disconnect": "openai_s2s::disconnect",
     "ServerEvent": "openai_s2s::server_event"
   },
+  "LlmEvents_Elevenlabs": {
+    "Error": "error",
+    "Connect": "elevenlabs_s2s::connect",
+    "ConnectFailure": "elevenlabs_s2s::connect_failed",
+    "Disconnect": "elevenlabs_s2s::disconnect",
+    "ServerEvent": "elevenlabs_s2s::server_event"
+  },
   "LlmEvents_VoiceAgent": {
     "Error": "error",
     "Connect": "voice_agent_s2s::connect",
@@ -182,6 +196,13 @@
     "Disconnect": "voice_agent_s2s::disconnect",
     "ServerEvent": "voice_agent_s2s::server_event"
   },
+  "LlmEvents_Ultravox": {
+    "Error": "error",
+    "Connect": "ultravox_s2s::connect",
+    "ConnectFailure": "ultravox_s2s::connect_failed",
+    "Disconnect": "ultravox_s2s::disconnect",
+    "ServerEvent": "ultravox_s2s::server_event"
+  },
   "QueueResults": {
     "Bridged": "bridged",
     "Error": "error",
@@ -196,7 +217,8 @@
   },
   "KillReason": {
     "Hangup": "hangup",
-    "Replaced": "replaced"
+    "Replaced": "replaced",
+    "MediaTimeout": "media_timeout"
   },
   "HookMsgTypes": [
     "session:new",
@@ -250,6 +272,16 @@
     "ConnectFailure": "elevenlabs_tts_streaming::connect_failed",
     "Connect": "elevenlabs_tts_streaming::connect"
   },
+  "RimelabsTtsStreamingEvents": {
+    "Empty": "rimelabs_tts_streaming::empty",
+    "ConnectFailure": "rimelabs_tts_streaming::connect_failed",
+    "Connect": "rimelabs_tts_streaming::connect"
+  },
+  "CustomTtsStreamingEvents": {
+    "Empty": "custom_tts_streaming::empty",
+    "ConnectFailure": "custom_tts_streaming::connect_failed",
+    "Connect": "custom_tts_streaming::connect"
+  },
   "TtsStreamingEvents": {
     "Empty": "tts_streaming::empty",
     "Pause": "tts_streaming::pause",
@@ -270,5 +302,9 @@
     "Offline": "OFFLINE",
     "GracefulShutdownInProgress":"SHUTDOWN_IN_PROGRESS"
   },
-  "FEATURE_SERVER" : "feature-server"
+  "FEATURE_SERVER" : "feature-server",
+  "WS_CLOSE_CODES": {
+    "NormalClosure": 1000,
+    "GoingAway": 1001
+  }
 }

lib/utils/db-utils.js

@@ -122,6 +122,10 @@ const speechMapper = (cred) => {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
     }
+    else if ('voxist' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = o.api_key;
+    }
     else if ('whisper' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
@@ -143,6 +147,7 @@ const speechMapper = (cred) => {
       obj.auth_token = o.auth_token;
       obj.custom_stt_url = o.custom_stt_url;
       obj.custom_tts_url = o.custom_tts_url;
+      obj.custom_tts_streaming_url = o.custom_tts_streaming_url;
     }
   } catch (err) {
     console.log(err);

lib/utils/error.js

@@ -17,8 +17,17 @@ class PlayFileNotFoundError extends NonFatalTaskError {
   }
 }
 
+class HTTPResponseError extends Error {
+  constructor(statusCode) {
+    super('Unexpected HTTP Response');
+    delete this.stack;
+    this.statusCode = statusCode;
+  }
+}
+
 module.exports = {
   SpeechCredentialError,
   NonFatalTaskError,
-  PlayFileNotFoundError
+  PlayFileNotFoundError,
+  HTTPResponseError
 };

lib/utils/http-requestor.js

@@ -16,6 +16,7 @@ const {
   NODE_ENV,
   HTTP_USER_AGENT_HEADER,
 } = require('../config');
+const {HTTPResponseError} = require('./error');
 
 const toBase64 = (str) => Buffer.from(str || '', 'utf8').toString('base64');
 
@@ -190,8 +191,7 @@ class HttpRequestor extends BaseRequestor {
         followRedirects: false
       });
       if (![200, 202, 204].includes(statusCode)) {
-        const err = new Error();
-        err.statusCode = statusCode;
+        const err = new HTTPResponseError(statusCode);
         throw err;
       }
       if (headers['content-type']?.includes('application/json')) {

lib/utils/place-outdial.js

@@ -17,7 +17,9 @@ const HttpRequestor = require('./http-requestor');
 const WsRequestor = require('./ws-requestor');
 const {makeOpusFirst} = require('./sdp-utils');
 const {
-  JAMBONES_USE_FREESWITCH_TIMER_FD
+  JAMBONES_USE_FREESWITCH_TIMER_FD,
+  JAMBONES_MEDIA_TIMEOUT_MS,
+  JAMBONES_MEDIA_HOLD_TIMEOUT_MS
 } = require('../config');
 
 class SingleDialer extends Emitter {
@@ -317,14 +319,19 @@ class SingleDialer extends Emitter {
   /**
    * kill the call in progress or the stable dialog, whichever we have
    */
-  async kill() {
+  async kill(Reason) {
     this.killed = true;
     if (this.inviteInProgress) await this.inviteInProgress.cancel();
     else if (this.dlg && this.dlg.connected) {
       const duration = moment().diff(this.dlg.connectTime, 'seconds');
       this.logger.debug('SingleDialer:kill hanging up called party');
       this.emit('callStatusChange', {callStatus: CallStatus.Completed, duration});
-      this.dlg.destroy();
+      const headers = {
+        ...(Reason && {'X-Reason': Reason})
+      };
+      this.dlg.destroy({
+        headers
+      });
     }
     if (this.ep) {
       this.logger.debug(`SingleDialer:kill - deleting endpoint ${this.ep.uuid}`);
@@ -335,11 +342,22 @@ class SingleDialer extends Emitter {
   _configMsEndpoint() {
     const opts = {
       ...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
-      ...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'})
+      ...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'}),
+      ...(JAMBONES_MEDIA_TIMEOUT_MS && {media_timeout: JAMBONES_MEDIA_TIMEOUT_MS}),
+      ...(JAMBONES_MEDIA_HOLD_TIMEOUT_MS && {media_hold_timeout: JAMBONES_MEDIA_HOLD_TIMEOUT_MS})
     };
     if (Object.keys(opts).length > 0) {
       this.ep.set(opts);
     }
+    if (this.dialTask?.inbandDtmfEnabled && !this.ep.inbandDtmfEnabled) {
+      // https://developer.signalwire.com/freeswitch/FreeSWITCH-Explained/Modules/mod-dptools/6587132/#0-about
+      try {
+        this.ep.execute('start_dtmf');
+        this.ep.inbandDtmfEnabled = true;
+      } catch (err) {
+        this.logger.info(err, 'place-outdial:_configMsEndpoint - error enable inband DTMF');
+      }
+    }
   }
 
   /**

lib/utils/transcription-utils.js

@@ -30,6 +30,7 @@ const stickyVars = {
     'DEEPGRAM_SPEECH_TIER',
     'DEEPGRAM_SPEECH_MODEL',
     'DEEPGRAM_SPEECH_ENABLE_SMART_FORMAT',
+    'DEEPGRAM_SPEECH_ENABLE_NO_DELAY',
     'DEEPGRAM_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION',
     'DEEPGRAM_SPEECH_PROFANITY_FILTER',
     'DEEPGRAM_SPEECH_REDACT',
@@ -43,7 +44,9 @@ const stickyVars = {
     'DEEPGRAM_SPEECH_UTTERANCE_END_MS',
     'DEEPGRAM_SPEECH_VAD_TURNOFF',
     'DEEPGRAM_SPEECH_TAG',
-    'DEEPGRAM_SPEECH_MODEL_VERSION'
+    'DEEPGRAM_SPEECH_MODEL_VERSION',
+    'DEEPGRAM_SPEECH_FILLER_WORDS',
+    'DEEPGRAM_SPEECH_KEYTERMS',
   ],
   aws: [
     'AWS_VOCABULARY_NAME',
@@ -104,6 +107,9 @@ const stickyVars = {
     'ASSEMBLYAI_API_KEY',
     'ASSEMBLYAI_WORD_BOOST'
   ],
+  voxist: [
+    'VOXIST_API_KEY',
+  ],
   speechmatics: [
     'SPEECHMATICS_API_KEY',
     'SPEECHMATICS_HOST',
@@ -208,7 +214,7 @@ const consolidateTranscripts = (bufferedTranscripts, channel, language, vendor)
     const lastChar = acc.alternatives[0].transcript.slice(-1);
     const firstChar = newTranscript.charAt(0);
 
-    if (lastChar.match(/\d/) && firstChar.match(/\d/)) {
+    if (vendor === 'speechmatics' || (lastChar.match(/\d/) && firstChar.match(/\d/))) {
       acc.alternatives[0].transcript += newTranscript;
     } else {
       acc.alternatives[0].transcript += ` ${newTranscript}`;
@@ -516,6 +522,25 @@ const normalizeAssemblyAi = (evt, channel, language) => {
   };
 };
 
+const normalizeVoxist = (evt, channel, language) => {
+  const copy = JSON.parse(JSON.stringify(evt));
+  return {
+    language_code: language,
+    channel_tag: channel,
+    is_final: evt.type === 'final',
+    alternatives: [
+      {
+        confidence: 1.00,
+        transcript: evt.text,
+      }
+    ],
+    vendor: {
+      name: 'voxist',
+      evt: copy
+    }
+  };
+};
+
 const normalizeSpeechmatics = (evt, channel, language) => {
   const copy = JSON.parse(JSON.stringify(evt));
   const is_final = evt.message === 'AddTranscript';
@@ -566,6 +591,8 @@ module.exports = (logger) => {
         return normalizeCobalt(evt, channel, language);
       case 'assemblyai':
         return normalizeAssemblyAi(evt, channel, language, shortUtterance);
+      case 'voxist':
+        return normalizeVoxist(evt, channel, language);
       case 'verbio':
         return normalizeVerbio(evt, channel, language);
       case 'speechmatics':
@@ -704,6 +731,8 @@ module.exports = (logger) => {
         //azureSttEndpointId overrides sttCredentials.custom_stt_endpoint
         ...(rOpts.azureSttEndpointId &&
           {AZURE_SERVICE_ENDPOINT_ID: rOpts.azureSttEndpointId}),
+        ...(azureOptions.speechRecognitionMode &&
+          {AZURE_RECOGNITION_MODE: azureOptions.speechRecognitionMode}),
       };
     }
     else if ('nuance' === vendor) {
@@ -777,6 +806,8 @@ module.exports = (logger) => {
           {DEEPGRAM_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION: 1},
         ...(deepgramOptions.smartFormatting) &&
           {DEEPGRAM_SPEECH_ENABLE_SMART_FORMAT: 1},
+        ...(deepgramOptions.noDelay) &&
+          {DEEPGRAM_SPEECH_ENABLE_NO_DELAY: 1},
         ...(deepgramOptions.profanityFilter) &&
           {DEEPGRAM_SPEECH_PROFANITY_FILTER: 1},
         ...(deepgramOptions.redact) &&
@@ -812,7 +843,11 @@ module.exports = (logger) => {
         ...(deepgramOptions.tag) &&
           {DEEPGRAM_SPEECH_TAG: deepgramOptions.tag},
         ...(deepgramOptions.version) &&
-          {DEEPGRAM_SPEECH_MODEL_VERSION: deepgramOptions.version}
+          {DEEPGRAM_SPEECH_MODEL_VERSION: deepgramOptions.version},
+        ...(deepgramOptions.fillerWords) &&
+          {DEEPGRAM_SPEECH_FILLER_WORDS: deepgramOptions.fillerWords},
+        ...((Array.isArray(deepgramOptions.keyterms) && deepgramOptions.keyterms.length > 0) &&
+          {DEEPGRAM_SPEECH_KEYTERMS: deepgramOptions.keyterms.join(',')})
       };
     }
     else if ('soniox' === vendor) {
@@ -921,6 +956,13 @@ module.exports = (logger) => {
           {ASSEMBLYAI_WORD_BOOST: JSON.stringify(rOpts.hints)})
       };
     }
+    else if ('voxist' === vendor) {
+      opts = {
+        ...opts,
+        ...(sttCredentials.api_key) &&
+          {VOXIST_API_KEY: sttCredentials.api_key},
+      };
+    }
     else if ('verbio' === vendor) {
       const {verbioOptions = {}} = rOpts;
       opts = {
@@ -954,6 +996,36 @@ module.exports = (logger) => {
             SPEECHMATICS_TRANSLATION_PARTIALS: speechmaticsOptions.translation_config.enable_partials ? 1 : 0
           }
         ),
+        ...(speechmaticsOptions.transcription_config?.domain &&
+          {SPEECHMATICS_DOMAIN: speechmaticsOptions.transcription_config.domain}),
+        ...{SPEECHMATICS_MAX_DELAY: speechmaticsOptions.transcription_config?.max_delay || 0.7},
+        ...{SPEECHMATICS_MAX_DELAY_MODE: speechmaticsOptions.transcription_config?.max_delay_mode || 'flexible'},
+        ...(speechmaticsOptions.transcription_config?.diarization &&
+            {SPEECHMATICS_DIARIZATION: speechmaticsOptions.transcription_config.diarization}),
+        ...(speechmaticsOptions.transcription_config?.speaker_diarization_config?.speaker_sensitivity &&
+          {SPEECHMATICS_DIARIZATION_SPEAKER_SENSITIVITY:
+            speechmaticsOptions.transcription_config.speaker_diarization_config.speaker_sensitivity}),
+        ...(speechmaticsOptions.transcription_config?.speaker_diarization_config?.max_speakers &&
+          {SPEECHMATICS_DIARIZATION_MAX_SPEAKERS:
+            speechmaticsOptions.transcription_config.speaker_diarization_config.max_speakers}),
+        ...(speechmaticsOptions.transcription_config?.output_locale &&
+          {SPEECHMATICS_OUTPUT_LOCALE: speechmaticsOptions.transcription_config.output_locale}),
+        ...(speechmaticsOptions.transcription_config?.punctuation_overrides?.permitted_marks &&
+          {SPEECHMATICS_PUNCTUATION_ALLOWED:
+            speechmaticsOptions.transcription_config.punctuation_overrides.permitted_marks.join(',')}),
+        ...(speechmaticsOptions.transcription_config?.punctuation_overrides?.sensitivity &&
+          {SPEECHMATICS_PUNCTUATION_SENSITIVITY:
+            speechmaticsOptions.transcription_config?.punctuation_overrides?.sensitivity}),
+        ...(speechmaticsOptions.transcription_config?.operating_point &&
+          {SPEECHMATICS_OPERATING_POINT: speechmaticsOptions.transcription_config.operating_point}),
+        ...(speechmaticsOptions.transcription_config?.enable_entities &&
+          {SPEECHMATICS_ENABLE_ENTTIES: speechmaticsOptions.transcription_config.enable_entities}),
+        ...(speechmaticsOptions.transcription_config?.audio_filtering_config?.volume_threshold &&
+          {SPEECHMATICS_VOLUME_THRESHOLD:
+            speechmaticsOptions.transcription_config.audio_filtering_config.volume_threshold}),
+        ...(speechmaticsOptions.transcription_config?.transcript_filtering_config?.remove_disfluencies &&
+          {SPEECHMATICS_REMOVE_DISFLUENCIES:
+            speechmaticsOptions.transcription_config.transcript_filtering_config.remove_disfluencies})
       };
     }
     else if (vendor.startsWith('custom:')) {

lib/utils/tts-streaming-buffer.js

@@ -4,63 +4,50 @@ const {
   TtsStreamingEvents,
   TtsStreamingConnectionStatus
 } = require('../utils/constants');
-const FEED_INTERVAL = 2000;
+
 const MAX_CHUNK_SIZE = 1800;
-const HIGH_WATER_BUFFER_SIZE = 5000;
-const LOW_WATER_BUFFER_SIZE = 1000;
-const MIN_INITIAL_WORDS = 4;
+const HIGH_WATER_BUFFER_SIZE = 1000;
+const LOW_WATER_BUFFER_SIZE = 200;
+const TIMEOUT_RETRY_MSECS = 3000;
 
-const findSentenceBoundary = (text, limit) => {
-  const sentenceEndRegex = /[.!?](?=\s|$)/g;
-  let lastSentenceBoundary = -1;
-  let match;
 
-  while ((match = sentenceEndRegex.exec(text)) && match.index < limit) {
-    /* Ensure it's not a decimal point (e.g., "3.14") */
-    if (match.index === 0 || !/\d$/.test(text[match.index - 1])) {
-      lastSentenceBoundary = match.index + 1; // Include the punctuation
-    }
-  }
-  return lastSentenceBoundary;
-};
-
-const findWordBoundary = (text, limit) => {
-  const wordBoundaryRegex = /\s+/g;
-  let lastWordBoundary = -1;
-  let match;
-
-  while ((match = wordBoundaryRegex.exec(text)) && match.index < limit) {
-    lastWordBoundary = match.index;
-  }
-  return lastWordBoundary;
-};
+const isWhitespace = (str) => /^\s*$/.test(str);
 
+/**
+ * Each queue item is an object:
+ *   - { type: 'text', value: '…' } for text tokens.
+ *   - { type: 'flush' } for a flush command.
+ */
 class TtsStreamingBuffer extends Emitter {
   constructor(cs) {
     super();
     this.cs = cs;
     this.logger = cs.logger;
 
-    this.tokens = '';
+    // Use an array to hold our structured items.
+    this.queue = [];
+    // Track total number of characters in text items.
+    this.bufferedLength = 0;
     this.eventHandlers = [];
     this._isFull = false;
     this._connectionStatus = TtsStreamingConnectionStatus.NotConnected;
-    this._flushPending = false;
-    this._countSendsInThisTurn = 0;
+    this.timer = null;
+    // Record the last time the text buffer was updated.
+    this.lastUpdateTime = 0;
   }
 
   get isEmpty() {
-    return this.tokens.length === 0;
+    return this.queue.length === 0;
+  }
+
+  get size() {
+    return this.bufferedLength;
   }
 
   get isFull() {
     return this._isFull;
   }
 
-  get size() {
-    return this.tokens.length;
-  }
-
   get ep() {
     return this.cs?.ep;
   }
@@ -68,7 +55,8 @@ class TtsStreamingBuffer extends Emitter {
   async start() {
     assert.ok(
       this._connectionStatus === TtsStreamingConnectionStatus.NotConnected,
-      'TtsStreamingBuffer:start already started, or has failed');
+      'TtsStreamingBuffer:start already started, or has failed'
+    );
 
     this.vendor = this.cs.getTsStreamingVendor();
     if (!this.vendor) {
@@ -81,9 +69,9 @@ class TtsStreamingBuffer extends Emitter {
     this._connectionStatus = TtsStreamingConnectionStatus.Connecting;
     try {
       if (this.eventHandlers.length === 0) this._initHandlers(this.ep);
-      await  this._api(this.ep, [this.ep.uuid, 'connect']);
+      await this._api(this.ep, [this.ep.uuid, 'connect']);
     } catch (err) {
-      this.logger.info({err}, 'TtsStreamingBuffer:start Error connecting to TTS streaming');
+      this.logger.info({ err }, 'TtsStreamingBuffer:start Error connecting to TTS streaming');
       this._connectionStatus = TtsStreamingConnectionStatus.Failed;
     }
   }
@@ -93,211 +81,319 @@ class TtsStreamingBuffer extends Emitter {
     this.removeCustomEventListeners();
     if (this.ep) {
       this._api(this.ep, [this.ep.uuid, 'close'])
-        .catch((err) => this.logger.info({err}, 'TtsStreamingBuffer:kill Error closing TTS streaming'));
+        .catch((err) =>
+          this.logger.info({ err }, 'TtsStreamingBuffer:stop Error closing TTS streaming')
+        );
     }
     this.timer = null;
-    this.tokens = '';
+    this.queue = [];
+    this.bufferedLength = 0;
     this._connectionStatus = TtsStreamingConnectionStatus.NotConnected;
   }
 
   /**
-   * Add tokens to the buffer and start feeding them to the endpoint if necessary.
+   * Buffer new text tokens.
    */
   async bufferTokens(tokens) {
-
     if (this._connectionStatus === TtsStreamingConnectionStatus.Failed) {
       this.logger.info('TtsStreamingBuffer:bufferTokens TTS streaming connection failed, rejecting request');
-      return {status: 'failed', reason: `connection to ${this.vendor} failed`};
+      return { status: 'failed', reason: `connection to ${this.vendor} failed` };
+    }
+
+    if (0 === this.bufferedLength && isWhitespace(tokens)) {
+      this.logger.debug({tokens}, 'TtsStreamingBuffer:bufferTokens discarded whitespace tokens');
+      return { status: 'ok' };
     }
 
     const displayedTokens = tokens.length <= 40 ? tokens : tokens.substring(0, 40);
     const totalLength = tokens.length;
 
-    /* if we crossed the high water mark, reject the request */
-    if (this.tokens.length + totalLength > HIGH_WATER_BUFFER_SIZE) {
+    if (this.bufferedLength + totalLength > HIGH_WATER_BUFFER_SIZE) {
       this.logger.info(
-        `TtsStreamingBuffer:bufferTokensTTS buffer is full, rejecting request to buffer ${totalLength} tokens`);
-
+        `TtsStreamingBuffer throttling: buffer is full, rejecting request to buffer ${totalLength} tokens`
+      );
       if (!this._isFull) {
         this._isFull = true;
         this.emit(TtsStreamingEvents.Pause);
       }
-      return {status: 'failed', reason: 'full'};
+      return { status: 'failed', reason: 'full' };
     }
 
     this.logger.debug(
-      `TtsStreamingBuffer:bufferTokens "${displayedTokens}" (length: ${totalLength}), starting? ${this.isEmpty}`
+      `TtsStreamingBuffer:bufferTokens "${displayedTokens}" (length: ${totalLength})`
     );
-    this.tokens += (tokens || '');
+    this.queue.push({ type: 'text', value: tokens });
+    this.bufferedLength += totalLength;
+    // Update the last update time each time new text is buffered.
+    this.lastUpdateTime = Date.now();
 
-    const  leftoverTokens = await this._feedTokens();
-
-    /* do we need to start a timer to periodically feed tokens to the endpoint? */
-    if (this.isEmpty && leftoverTokens > 0) {
-      assert(!this.timer);
-      this.timer = setInterval(async() => {
-        const remaining = await this._feedTokens();
-        if (remaining === 0) {
-          clearInterval(this.timer);
-          this.timer = null;
-        }
-      }, FEED_INTERVAL);
-    }
-
-    return {status: 'ok'};
+    await this._feedQueue();
+    return { status: 'ok' };
   }
 
+  /**
+   * Insert a flush command. If no text is queued, flush immediately.
+   * Otherwise, append a flush marker so that all text preceding it will be sent
+   * (regardless of sentence boundaries) before the flush is issued.
+   */
   flush() {
-    this.logger.debug('TtsStreamingBuffer:flush');
     if (this._connectionStatus === TtsStreamingConnectionStatus.Connecting) {
       this.logger.debug('TtsStreamingBuffer:flush TTS stream is not quite ready - wait for connect');
-      this._flushPending = true;
+      if (this.queue.length === 0 || this.queue[this.queue.length - 1].type !== 'flush') {
+        this.queue.push({ type: 'flush' });
+      }
       return;
     }
     else if (this._connectionStatus === TtsStreamingConnectionStatus.Connected) {
-      this._countSendsInThisTurn = 0;
-      this._api(this.ep, [this.ep.uuid, 'flush'])
-        .catch((err) => this.logger.info({err},
-          `TtsStreamingBuffer:flush Error flushing TTS streaming: ${JSON.stringify(err)}`));
+      if (this.isEmpty) {
+        this._doFlush();
+      }
+      else {
+        if (this.queue[this.queue.length - 1].type !== 'flush') {
+          this.queue.push({ type: 'flush' });
+          this.logger.debug('TtsStreamingBuffer:flush added flush marker to queue');
+        }
+      }
+    }
+    else {
+      this.logger.debug(
+        `TtsStreamingBuffer:flush TTS stream is not connected, status: ${this._connectionStatus}`
+      );
     }
   }
 
   clear() {
     this.logger.debug('TtsStreamingBuffer:clear');
-
     if (this._connectionStatus !== TtsStreamingConnectionStatus.Connected) return;
     clearTimeout(this.timer);
-    this._api(this.ep, [this.ep.uuid, 'clear'])
-      .catch((err) => this.logger.info({err}, 'TtsStreamingBuffer:clear Error clearing TTS streaming'));
-    this.tokens = '';
+    this._api(this.ep, [this.ep.uuid, 'clear']).catch((err) =>
+      this.logger.info({ err }, 'TtsStreamingBuffer:clear Error clearing TTS streaming')
+    );
+    this.queue = [];
+    this.bufferedLength = 0;
     this.timer = null;
     this._isFull = false;
   }
 
   /**
-   * Send the next chunk of tokens to the endpoint (max 2000 chars)
-   * Return the number of tokens left in the buffer.
+   * Process the queue in two phases.
+   *
+   * Phase 1: Look for flush markers. When a flush marker is found (even if not at the very front),
+   *   send all text tokens that came before it immediately (ignoring sentence boundaries)
+   *   and then send the flush command. Repeat until there are no flush markers left.
+   *
+   * Phase 2: With the remaining queue (now containing only text items), accumulate text
+   *   up to MAX_CHUNK_SIZE and use sentence-boundary logic to determine a chunk.
+   *   Then, remove the exact tokens (or portions thereof) that were consumed.
    */
-  async _feedTokens() {
-    this.logger.debug({tokens: this.tokens}, '_feedTokens');
-
+  async _feedQueue(handlingTimeout = false) {
+    this.logger.debug({ queue: this.queue }, 'TtsStreamingBuffer:_feedQueue');
     try {
-      if (!this.cs.isTtsStreamOpen || !this.ep || !this.tokens) {
-        this.logger.debug('TTS stream is not open or no tokens to send');
-        return this.tokens?.length || 0;
+      if (!this.cs.isTtsStreamOpen || !this.ep) {
+        this.logger.debug('TtsStreamingBuffer:_feedQueue TTS stream is not open or no endpoint available');
+        return;
+      }
+      if (
+        this._connectionStatus === TtsStreamingConnectionStatus.NotConnected ||
+        this._connectionStatus === TtsStreamingConnectionStatus.Failed
+      ) {
+        this.logger.debug('TtsStreamingBuffer:_feedQueue TTS stream is not connected');
+        return;
       }
 
-      if (this._connectionStatus === TtsStreamingConnectionStatus.NotConnected ||
-        this._connectionStatus === TtsStreamingConnectionStatus.Failed) {
-        this.logger.debug('TtsStreamingBuffer:_feedTokens TTS stream is not connected');
-        return this.tokens.length;
+      // --- Phase 1: Process flush markers ---
+      // Process any flush marker that isn’t in the very first position.
+      let flushIndex = this.queue.findIndex((item, idx) => item.type === 'flush' && idx > 0);
+      while (flushIndex !== -1) {
+        let flushText = '';
+        // Accumulate all text tokens preceding the flush marker.
+        for (let i = 0; i < flushIndex; i++) {
+          if (this.queue[i].type === 'text') {
+            flushText += this.queue[i].value;
+          }
+        }
+        // Remove those text items.
+        for (let i = 0; i < flushIndex; i++) {
+          const item = this.queue.shift();
+          if (item.type === 'text') {
+            this.bufferedLength -= item.value.length;
+          }
+        }
+        // Remove the flush marker (now at the front).
+        if (this.queue.length > 0 && this.queue[0].type === 'flush') {
+          this.queue.shift();
+        }
+        // Immediately send all accumulated text (ignoring sentence boundaries).
+        if (flushText.length > 0) {
+          const modifiedFlushText = flushText.replace(/\n\n/g, '\n \n');
+          try {
+            await this._api(this.ep, [this.ep.uuid, 'send', modifiedFlushText]);
+          } catch (err) {
+            this.logger.info({ err, flushText }, 'TtsStreamingBuffer:_feedQueue Error sending TTS chunk');
+          }
+        }
+        // Send the flush command.
+        await this._doFlush();
+
+        flushIndex = this.queue.findIndex((item, idx) => item.type === 'flush' && idx > 0);
       }
 
-      if (this._connectionStatus === TtsStreamingConnectionStatus.Connecting) {
-        this.logger.debug('TtsStreamingBuffer:_feedTokens TTS stream is not ready, waiting for connect');
-        return this.tokens.length;
+      // If a flush marker is at the very front, process it.
+      while (this.queue.length > 0 && this.queue[0].type === 'flush') {
+        this.queue.shift();
+        await this._doFlush();
       }
 
-      /**
-       * Rules:
-       * 1. If this is our first send, we must have at least N words
-       * 2. Otherwise, must EITHER have N words OR be the ending of a sentence
-       *
-       * When sending, send the max size possible, capped at a limit to avoid overwhelming the server.
-       */
+      // --- Phase 2: Process remaining text tokens ---
+      if (this.queue.length === 0) {
+        this._removeTimer();
+        return;
+      }
 
-      /* must have at least N words, or be the ending of a sentence */
-      const words = this.tokens.split(' ').length;
-      if (words < MIN_INITIAL_WORDS) {
-        const endsWithPunctuation = /[.!?]$/.test(this.tokens);
-        if (!endsWithPunctuation || this._countSendsInThisTurn === 0) {
-          this.logger.debug(`TtsStreamingBuffer:_feedTokens: only ${words} words to send, waiting for more`);
-          return this.tokens.length;
+      // Accumulate contiguous text tokens (from the front) up to MAX_CHUNK_SIZE.
+      let combinedText = '';
+      for (const item of this.queue) {
+        if (item.type !== 'text') break;
+        combinedText += item.value;
+        if (combinedText.length >= MAX_CHUNK_SIZE) break;
+      }
+      if (combinedText.length === 0) {
+        this._removeTimer();
+        return;
+      }
+
+      const limit = Math.min(MAX_CHUNK_SIZE, combinedText.length);
+      let chunkEnd = findSentenceBoundary(combinedText, limit);
+      if (chunkEnd <= 0) {
+        if (handlingTimeout) {
+          chunkEnd = findWordBoundary(combinedText, limit);
+          if (chunkEnd <= 0) {
+            this._setTimerIfNeeded();
+            return;
+          }
+        } else {
+          this._setTimerIfNeeded();
+          return;
         }
       }
+      const chunk = combinedText.slice(0, chunkEnd);
 
-      const limit = Math.min(MAX_CHUNK_SIZE, this.tokens.length);
-      let chunkEnd = findSentenceBoundary(this.tokens, limit);
-
-      if (chunkEnd === -1) {
-        this.logger.debug('TtsStreamingBuffer:_feedTokens: no sentence boundary found, look for word boundary');
-        chunkEnd = findWordBoundary(this.tokens, limit);
+      // Now we iterate over the queue items
+      // and deduct their lengths until we've accounted for chunkEnd characters.
+      let remaining = chunkEnd;
+      let tokensProcessed = 0;
+      for (let i = 0; i < this.queue.length; i++) {
+        const token = this.queue[i];
+        if (token.type !== 'text') break;
+        if (remaining >= token.value.length) {
+          remaining -= token.value.length;
+          tokensProcessed = i + 1;
+        } else {
+          // Partially consumed token: update its value to remove the consumed part.
+          token.value = token.value.slice(remaining);
+          tokensProcessed = i;
+          remaining = 0;
+          break;
+        }
       }
+      // Remove the fully consumed tokens from the front of the queue.
+      this.queue.splice(0, tokensProcessed);
+      this.bufferedLength -= chunkEnd;
 
-      if (chunkEnd === -1) {
-        chunkEnd = limit;
-      }
-
-      const chunk = this.tokens.slice(0, chunkEnd);
-      this.tokens = this.tokens.slice(chunkEnd); // Remove sent chunk
-
-      /* freeswitch looks for sequence of 2 newlines to determine end of message, so insert a space */
       const modifiedChunk = chunk.replace(/\n\n/g, '\n \n');
+      this.logger.debug(`TtsStreamingBuffer:_feedQueue sending chunk to tts: ${modifiedChunk}`);
 
-      if (modifiedChunk.length > 0) {
-        try {
-          this._countSendsInThisTurn++;
-          this.logger.debug({tokens: modifiedChunk},
-            `TtsStreamingBuffer:_feedTokens: sending tokens, in send#${this._countSendsInThisTurn}`);
-          await this._api(this.ep, [this.ep.uuid, 'send', modifiedChunk]);
-        } catch (err) {
-          this.logger.info({err}, 'TtsStreamingBuffer:_feedTokens Error sending TTS chunk');
-        }
-
-        this.logger.debug(`TtsStreamingBuffer:_feedTokens: sent ${chunk.length}, remaining: ${this.tokens.length}`);
-
-        if (this.isFull && this.tokens.length <= LOW_WATER_BUFFER_SIZE) {
-          this.logger.info('TtsStreamingBuffer:_feedTokens TTS streaming buffer is no longer full');
-          this._isFull = false;
-          this.emit(TtsStreamingEvents.Resume);
-        }
+      try {
+        await this._api(this.ep, [this.ep.uuid, 'send', modifiedChunk]);
+      } catch (err) {
+        this.logger.info({ err, chunk }, 'TtsStreamingBuffer:_feedQueue Error sending TTS chunk');
       }
+
+      if (this._isFull && this.bufferedLength <= LOW_WATER_BUFFER_SIZE) {
+        this.logger.info('TtsStreamingBuffer throttling: buffer is no longer full - resuming');
+        this._isFull = false;
+        this.emit(TtsStreamingEvents.Resume);
+      }
+
+      return this._feedQueue();
     } catch (err) {
-      this.logger.info({err}, 'TtsStreamingBuffer:_feedTokens Error sending TTS chunk');
-      this.tokens = '';
+      this.logger.info({ err }, 'TtsStreamingBuffer:_feedQueue Error sending TTS chunk');
+      this.queue = [];
+      this.bufferedLength = 0;
     }
-
-    if (0 === this.tokens.length && this.timer) {
-      clearTimeout(this.timer);
-      this.timer = null;
-    }
-
-    return this.tokens.length;
   }
 
   async _api(ep, args) {
-    const apiCmd = `uuid_${this.vendor}_tts_streaming`;
+    const apiCmd = `uuid_${this.vendor.startsWith('custom:') ? 'custom' : this.vendor}_tts_streaming`;
     const res = await ep.api(apiCmd, `^^|${args.join('|')}`);
     if (!res.body?.startsWith('+OK')) {
-      throw new Error({args}, `Error calling ${apiCmd}: ${res.body}`);
+      this.logger.info({ args }, `Error calling ${apiCmd}: ${res.body}`);
+      throw new Error(`Error calling ${apiCmd}: ${res.body}`);
+    }
+  }
+
+  _doFlush() {
+    return this._api(this.ep, [this.ep.uuid, 'flush'])
+      .then(() => this.logger.debug('TtsStreamingBuffer:_doFlush sent flush command'))
+      .catch((err) =>
+        this.logger.info(
+          { err },
+          `TtsStreamingBuffer:_doFlush Error flushing TTS streaming: ${JSON.stringify(err)}`
+        )
+      );
+  }
+
+  async _onConnect(vendor) {
+    this.logger.info(`TtsStreamingBuffer:_onConnect streaming tts connection made to ${vendor} successful`);
+    this._connectionStatus = TtsStreamingConnectionStatus.Connected;
+    if (this.queue.length > 0) {
+      await this._feedQueue();
     }
   }
 
   _onConnectFailure(vendor) {
-    this.logger.info(`streaming tts connection failed to ${vendor}`);
+    this.logger.info(`TtsStreamingBuffer:_onConnectFailure streaming tts connection failed to ${vendor}`);
     this._connectionStatus = TtsStreamingConnectionStatus.Failed;
-    this.tokens = '';
-    this.emit(TtsStreamingEvents.ConnectFailure, {vendor});
+    this.queue = [];
+    this.bufferedLength = 0;
+    this.emit(TtsStreamingEvents.ConnectFailure, { vendor });
   }
 
-  async _onConnect(vendor) {
-    this.logger.info(`streaming tts connection made to ${vendor}`);
-    this._connectionStatus = TtsStreamingConnectionStatus.Connected;
-    if (this.tokens.length > 0) {
-      await this._feedTokens();
+  _setTimerIfNeeded() {
+    if (this.bufferedLength > 0 && !this.timer) {
+      this.logger.debug({queue: this.queue},
+        `TtsStreamingBuffer:_setTimerIfNeeded setting timer because ${this.bufferedLength} buffered`);
+      this.timer = setTimeout(this._onTimeout.bind(this), TIMEOUT_RETRY_MSECS);
     }
-    if (this._flushPending) {
-      this.flush();
-      this._flushPending = false;
+  }
+
+  _removeTimer() {
+    if (this.timer) {
+      this.logger.debug('TtsStreamingBuffer:_removeTimer clearing timer');
+      clearTimeout(this.timer);
+      this.timer = null;
     }
   }
 
+  _onTimeout() {
+    this.logger.debug('TtsStreamingBuffer:_onTimeout Timeout waiting for sentence boundary');
+    // Check if new text has been added since the timer was set.
+    const now = Date.now();
+    if (now - this.lastUpdateTime < TIMEOUT_RETRY_MSECS) {
+      this.logger.debug('TtsStreamingBuffer:_onTimeout New text received recently; postponing flush.');
+      this._setTimerIfNeeded();
+      return;
+    }
+    this.timer = null;
+    this._feedQueue(true);
+  }
+
   _onTtsEmpty(vendor) {
-    this.emit(TtsStreamingEvents.Empty, {vendor});
+    this.emit(TtsStreamingEvents.Empty, { vendor });
   }
 
   addCustomEventListener(ep, event, handler) {
-    this.eventHandlers.push({ep, event, handler});
+    this.eventHandlers.push({ ep, event, handler });
     ep.addCustomEventListener(event, handler);
   }
 
@@ -307,10 +403,11 @@ class TtsStreamingBuffer extends Emitter {
 
   _initHandlers(ep) {
     [
-      // DH: add other vendors here as modules are added
       'deepgram',
       'cartesia',
-      'elevenlabs'
+      'elevenlabs',
+      'rimelabs',
+      'custom'
     ].forEach((vendor) => {
       const eventClassName = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}TtsStreamingEvents`;
       const eventClass = require('../utils/constants')[eventClassName];
@@ -323,4 +420,33 @@ class TtsStreamingBuffer extends Emitter {
   }
 }
 
+const findSentenceBoundary = (text, limit) => {
+  // Look for punctuation or double newline that signals sentence end.
+  const sentenceEndRegex = /[.!?](?=\s|$)|\n\n/g;
+  let lastSentenceBoundary = -1;
+  let match;
+  while ((match = sentenceEndRegex.exec(text)) && match.index < limit) {
+    const precedingText = text.slice(0, match.index).trim();
+    if (precedingText.length > 0) {
+      if (
+        match[0] === '\n\n' ||
+        (match.index === 0 || !/\d$/.test(text[match.index - 1]))
+      ) {
+        lastSentenceBoundary = match.index + (match[0] === '\n\n' ? 2 : 1);
+      }
+    }
+  }
+  return lastSentenceBoundary;
+};
+
+const findWordBoundary = (text, limit) => {
+  const wordBoundaryRegex = /\s+/g;
+  let lastWordBoundary = -1;
+  let match;
+  while ((match = wordBoundaryRegex.exec(text)) && match.index < limit) {
+    lastWordBoundary = match.index;
+  }
+  return lastWordBoundary;
+};
+
 module.exports = TtsStreamingBuffer;

lib/utils/ws-requestor.js

@@ -1,7 +1,7 @@
 const assert = require('assert');
 const BaseRequestor = require('./base-requestor');
 const short = require('short-uuid');
-const {HookMsgTypes} = require('./constants.json');
+const {HookMsgTypes, WS_CLOSE_CODES} = require('./constants.json');
 const Websocket = require('ws');
 const snakeCaseKeys = require('./snakecase-keys');
 const {
@@ -261,13 +261,13 @@ class WsRequestor extends BaseRequestor {
     }
   }
 
-  close() {
+  close(code = WS_CLOSE_CODES.NormalClosure) {
     this.closedGracefully = true;
-    this.logger.debug('WsRequestor:close closing socket');
+    this.logger.debug(`WsRequestor:close closing socket with code ${code}`);
     this._stopPingTimer();
     try {
       if (this.ws) {
-        this.ws.close(1000);
+        this.ws.close(code);
         this.ws.removeAllListeners();
         this.ws = null;
       }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jambonz-feature-server",
-  "version": "0.9.2",
+  "version": "0.9.3",
   "main": "app.js",
   "engines": {
     "node": ">= 18.x"
@@ -31,9 +31,9 @@
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/mw-registrar": "^0.2.7",
     "@jambonz/realtimedb-helpers": "^0.8.8",
-    "@jambonz/speech-utils": "^0.2.1",
+    "@jambonz/speech-utils": "^0.2.3",
     "@jambonz/stats-collector": "^0.1.10",
-    "@jambonz/verb-specifications": "^0.0.90",
+    "@jambonz/verb-specifications": "^0.0.97",
     "@jambonz/time-series": "^0.2.13",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/exporter-jaeger": "^1.23.0",
@@ -47,8 +47,8 @@
     "bent": "^7.3.12",
     "debug": "^4.3.4",
     "deepcopy": "^2.1.0",
-    "drachtio-fsmrf": "^3.0.46",
-    "drachtio-srf": "^4.5.35",
+    "drachtio-fsmrf": "^4.0.1",
+    "drachtio-srf": "^5.0.1",
     "express": "^4.19.2",
     "express-validator": "^7.0.1",
     "moment": "^2.30.1",
@@ -56,7 +56,7 @@
     "pino": "^8.20.0",
     "polly-ssml-split": "^0.1.0",
     "proxyquire": "^2.1.3",
-    "sdp-transform": "^2.14.2",
+    "sdp-transform": "^2.15.0",
     "short-uuid": "^5.1.0",
     "sinon": "^17.0.1",
     "to-snake-case": "^1.0.0",

test/create-call-test.js

@@ -222,3 +222,62 @@ test('test create-call app_json', async(t) => {
     t.error(err);
   }
 });
+
+test('test create-call timeLimit', async(t) => {
+  clearModule.all();
+  const {srf, disconnect} = require('../app');
+
+  try {
+    await connect(srf);
+
+
+    // GIVEN
+    let from = 'create-call-app-json';
+    let account_sid = 'bb845d4b-83a9-4cde-a6e9-50f3743bab3f';
+
+    // Give UAS app time to come up
+    const p = sippUac('uas.xml', '172.38.0.10', from);
+    await waitFor(1000);
+
+    const startTime = Date.now();
+
+    const app_json = `[
+      {
+        "verb": "pause",
+        "length": 7
+      }
+    ]`;
+
+    const post = bent('http://127.0.0.1:3000/', 'POST', 'json', 201);
+    post('v1/createCall', {
+      'account_sid':account_sid,
+      "call_hook": {
+        "url": "http://127.0.0.1:3100/",
+        "method": "POST",
+        "username": "username",
+        "password": "password"
+      },
+      app_json,
+      "from": from,
+      "to": {
+        "type": "phone",
+        "number": "15583084809"
+      },
+      "timeLimit": 1,
+      "speech_recognizer_vendor": "google",
+      "speech_recognizer_language": "en"
+    });
+
+    //THEN
+    await p;
+    const endTime = Date.now();
+
+    t.ok(endTime - startTime < 2000, 'create-call: timeLimit is respected');
+
+    disconnect();
+  } catch (err) {
+    console.log(`error received: ${err}`);
+    disconnect();
+    t.error(err);
+  }
+});