Update background-task-manager.js

* initial changes for stream synonym to listen

* listen on B endpoint if nested listen in dial has channel === 2

lib/middleware.js

@@ -1,5 +1,5 @@
 const uuidv4 = require('uuid-random');
-const {CallDirection, AllowedSipRecVerbs} = require('./utils/constants');
+const {CallDirection, AllowedSipRecVerbs, WS_CLOSE_CODES} = require('./utils/constants');
 const {parseSiprecPayload} = require('./utils/siprec-utils');
 const CallInfo = require('./session/call-info');
 const HttpRequestor = require('./utils/http-requestor');
@@ -460,7 +460,7 @@ module.exports = function(srf, logger) {
       }).catch((err) => this.logger.info({err}, 'Error generating alert for parsing application'));
       logger.info({err}, `Error retrieving or parsing application: ${err?.message}`);
       res.send(480, {headers: {'X-Reason': err?.message || 'unknown'}});
-      app.requestor.close();
+      app.requestor.close(WS_CLOSE_CODES.GoingAway);
     }
   }
 

lib/session/call-session.js

@@ -1109,7 +1109,8 @@ class CallSession extends Emitter {
             speech_credential_sid: credential.speech_credential_sid,
             auth_token: credential.auth_token,
             custom_stt_url: credential.custom_stt_url,
-            custom_tts_url: credential.custom_tts_url
+            custom_tts_url: credential.custom_tts_url,
+            custom_tts_streaming_url: credential.custom_tts_streaming_url
           };
         }
       }
@@ -1467,7 +1468,7 @@ class CallSession extends Emitter {
     if (!listenTask) {
       return this.logger.info('CallSession:_lccListenStatus - invalid listen_status: Dial does not have a listen');
     }
-    listenTask.updateListen(opts.listen_status);
+    listenTask.updateListen(opts.listen_status || opts.stream_status);
   }
 
   /**
@@ -1609,7 +1610,7 @@ Duration=${duration} `
     // this whole thing requires us to be in a Dial verb
     const task = this.currentTask;
     if (!task || ![TaskName.Dial, TaskName.Listen].includes(task.name)) {
-      return this.logger.info('CallSession:_lccWhisper - invalid command since we are not in a dial or listen');
+      return this.logger.info('CallSession:_lccWhisper - invalid command since we are not in a dial or stream/listen');
     }
 
     // allow user to provide a url object, a url string, an array of tasks, or a single task
@@ -1798,7 +1799,7 @@ Duration=${duration} `
     if (opts.call_hook || opts.child_call_hook) {
       return await this._lccCallHook(opts);
     }
-    if (opts.listen_status) {
+    if (opts.listen_status || opts.stream_status) {
       await this._lccListenStatus(opts);
     }
     if (opts.transcribe_status) {
@@ -2108,6 +2109,7 @@ Duration=${duration} `
         break;
 
       case 'listen:status':
+      case 'stream:status':
         this._lccListenStatus(data);
         break;
 

lib/tasks/dial.js

@@ -121,8 +121,9 @@ class TaskDial extends Task {
       }
     }
 
-    if (this.data.listen) {
-      this.listenTask = makeTask(logger, {'listen': this.data.listen}, this);
+    const listenData = this.data.listen || this.data.stream;
+    if (listenData) {
+      this.listenTask = makeTask(logger, {'listen': listenData }, this);
     }
     if (this.data.transcribe) {
       this.transcribeTask = makeTask(logger, {'transcribe' : this.data.transcribe}, this);
@@ -873,7 +874,7 @@ class TaskDial extends Task {
     if (cs.sipRequestWithinDialogHook) this._initSipIndialogRequestListener(cs, this.dlg);
 
     if (this.transcribeTask) this.transcribeTask.exec(cs, {ep: this.epOther, ep2:this.ep});
-    if (this.listenTask) this.listenTask.exec(cs, {ep: this.epOther});
+    if (this.listenTask) this.listenTask.exec(cs, {ep: this.listenTask.channel === 2 ? this.ep : this.epOther});
     if (this.startAmd) {
       try {
         this.startAmd(cs, this.ep, this, this.data.amd);

lib/tasks/gather.js

@@ -24,6 +24,7 @@ const makeTask = require('./make_task');
 const assert = require('assert');
 const SttTask = require('./stt-task');
 const { SpeechCredentialError } = require('../utils/error');
+const SPEECHMATICS_DEFAULT_ASR_TIMEOUT = 1200;
 
 class TaskGather extends SttTask {
   constructor(logger, opts, parentTask) {
@@ -162,6 +163,16 @@ class TaskGather extends SttTask {
       this.logger.debug({hints: this.data.recognizer.hints, hintsBoost: this.data.recognizer.hintsBoost},
         'Gather:exec - applying global sttHints');
     }
+
+    // specials case for speechmatics: they dont do endpointing so we need to enable continuous ASR
+    if (this.vendor === 'speechmatics' && !this.isContinuousAsr) {
+      const maxDelay = this.recognizer?.speechmaticsOptions?.transcription_config?.max_delay;
+      if (maxDelay) this.asrTimeout = Math.min(SPEECHMATICS_DEFAULT_ASR_TIMEOUT, maxDelay * 1000);
+      else this.asrTimeout = SPEECHMATICS_DEFAULT_ASR_TIMEOUT;
+      this.isContinuousAsr = true;
+      this.logger.debug(`Gather:exec - auto-enabling continuous ASR for speechmatics w/ timeout ${this.asrTimeout}`);
+    }
+
     if (!this.isContinuousAsr && cs.isContinuousAsr) {
       this.isContinuousAsr = true;
       this.asrTimeout = cs.asrTimeout * 1000;
@@ -832,7 +843,7 @@ class TaskGather extends SttTask {
         const t = evt.alternatives[0].transcript;
         if (t) {
           /* remove trailing punctuation */
-          if (/[,;:\.!\?]$/.test(t)) {
+          if (this.vendor !== 'speechmatics' && /[,;:\.!\?]$/.test(t)) {
             this.logger.debug('TaskGather:_onTranscription - removing trailing punctuation');
             evt.alternatives[0].transcript = t.slice(0, -1);
           }

lib/tasks/listen.js

@@ -17,7 +17,7 @@ class TaskListen extends Task {
 
     [
       'action', 'auth', 'method', 'url', 'finishOnKey', 'maxLength', 'metadata', 'mixType', 'passDtmf', 'playBeep',
-      'sampleRate', 'timeout', 'transcribe', 'wsAuth', 'disableBidirectionalAudio'
+      'sampleRate', 'timeout', 'transcribe', 'wsAuth', 'disableBidirectionalAudio', 'channel'
     ].forEach((k) => this[k] = this.data[k]);
 
     this.mixType = this.mixType || 'mono';

lib/tasks/llm/index.js

@@ -2,6 +2,7 @@ const Task = require('../task');
 const {TaskPreconditions} = require('../../utils/constants');
 const TaskLlmOpenAI_S2S = require('./llms/openai_s2s');
 const TaskLlmVoiceAgent_S2S = require('./llms/voice_agent_s2s');
+const TaskLlmUltravox_S2S = require('./llms/ultravox_s2s');
 
 class TaskLlm extends Task {
   constructor(logger, opts) {
@@ -49,6 +50,10 @@ class TaskLlm extends Task {
         llm = new TaskLlmVoiceAgent_S2S(this.logger, this.data, this);
         break;
 
+      case 'ultravox':
+        llm = new TaskLlmUltravox_S2S(this.logger, this.data, this);
+        break;
+
       default:
         throw new Error(`Unsupported vendor ${this.vendor} for LLM`);
     }

lib/tasks/llm/llms/openai_s2s.js

@@ -120,7 +120,7 @@ class TaskLlmOpenAI_S2S extends Task {
 
     switch (this.vendor) {
       case 'openai':
-        return 'v1/realtime?model=${this.model}';
+        return `v1/realtime?model=${this.model}`;
       case 'microsoft':
         return `openai/realtime?api-version=2024-10-01-preview&deployment=${this.model}`;
     }

lib/tasks/llm/llms/ultravox_s2s.js

@@ -0,0 +1,245 @@
+const Task = require('../../task');
+const TaskName = 'Llm_Ultravox_s2s';
+const {request} = require('undici');
+const {LlmEvents_Ultravox} = require('../../../utils/constants');
+
+const ultravox_server_events = [
+  'pong',
+  'state',
+  'transcript',
+  'conversationText',
+  'clientToolInvocation',
+  'playbackClearBuffer',
+];
+
+const ClientEvent = 'client.event';
+
+const expandWildcards = (events) => {
+  // no-op for deepgram
+  return events;
+};
+
+const SessionDelete = 'session.delete';
+
+class TaskLlmUltravox_S2S extends Task {
+  constructor(logger, opts, parentTask) {
+    super(logger, opts, parentTask);
+    this.parent = parentTask;
+
+    this.vendor = this.parent.vendor;
+    this.model = this.parent.model || 'fixie-ai/ultravox';
+    this.auth = this.parent.auth;
+    this.connectionOptions = this.parent.connectOptions;
+
+    const {apiKey} = this.auth || {};
+    if (!apiKey) throw new Error('auth.apiKey is required for Vendor: Ultravox');
+    this.apiKey = apiKey;
+    this.actionHook = this.data.actionHook;
+    this.eventHook = this.data.eventHook;
+    this.toolHook = this.data.toolHook;
+
+    /**
+     * only one of these will have items,
+     * if includeEvents, then these are the events to include
+     * if excludeEvents, then these are the events to exclude
+     */
+    this.includeEvents = [];
+    this.excludeEvents = [];
+
+    /* default to all events if user did not specify */
+    this._populateEvents(this.data.events || ultravox_server_events);
+
+    this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
+    this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
+  }
+
+  get name() { return TaskName; }
+
+  async _api(ep, args) {
+    const res = await ep.api('uuid_ultravox_s2s', `^^|${args.join('|')}`);
+    if (!res.body?.startsWith('+OK')) {
+      throw new Error(`Error calling uuid_ultravox_s2s: ${JSON.stringify(res.body)}`);
+    }
+  }
+
+  async createCall() {
+    const payload = {
+      ...this.data.llmOptions,
+      model: this.model,
+      medium: {
+        ...(this.data.llmOptions.medium || {}),
+        serverWebSocket: {
+          inputSampleRate: 8000,
+          outputSampleRate: 8000,
+        }
+      }
+    };
+    const {statusCode, body} = await request('https://api.ultravox.ai/api/calls', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'X-API-Key': this.apiKey
+      },
+      body: JSON.stringify(payload)
+    });
+    const data = await body.json();
+    if (statusCode !== 201 || !data?.joinUrl) {
+      this.logger.error({statusCode, data}, 'Ultravox Error registering call');
+      throw new Error(`Ultravox  Error registering call: ${data.message}`);
+    }
+    this.logger.info({joinUrl: data.joinUrl}, 'Ultravox Call registered');
+    return data.joinUrl;
+  }
+
+  _unregisterHandlers() {
+    this.removeCustomEventListeners();
+  }
+
+  _registerHandlers(ep) {
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.Connect, this._onConnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.ConnectFailure, this._onConnectFailure.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.Disconnect, this._onDisconnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.ServerEvent, this._onServerEvent.bind(this, ep));
+  }
+
+  async _startListening(cs, ep) {
+    this._registerHandlers(ep);
+
+    const joinUrl = await this.createCall();
+    // split the joinUrl into host and path
+    const {host, pathname, search} = new URL(joinUrl);
+
+    try {
+      const args = [ep.uuid, 'session.create', host, pathname + search];
+      await this._api(ep, args);
+    } catch (err) {
+      this.logger.error({err}, 'TaskLlmUltraVox_S2S:_startListening');
+      this.notifyTaskDone();
+    }
+  }
+
+  async exec(cs, {ep}) {
+    await super.exec(cs);
+
+    await this._startListening(cs, ep);
+
+    await this.awaitTaskDone();
+
+    /* note: the parent llm verb started the span, which is why this is necessary */
+    await this.parent.performAction(this.results);
+
+    this._unregisterHandlers();
+  }
+
+  async kill(cs) {
+    super.kill(cs);
+
+    this._api(cs.ep, [cs.ep.uuid, SessionDelete])
+      .catch((err) => this.logger.info({err}, 'TaskLlmUltravox_S2S:kill - error deleting session'));
+
+    this.notifyTaskDone();
+  }
+
+  _onConnect(ep) {
+    this.logger.debug('TaskLlmUltravox_S2S:_onConnect');
+  }
+  _onConnectFailure(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmUltravox_S2S:_onConnectFailure');
+    this.results = {completionReason: 'connection failure'};
+    this.notifyTaskDone();
+  }
+  _onDisconnect(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmUltravox_S2S:_onConnectFailure');
+    this.results = {completionReason: 'disconnect from remote end'};
+    this.notifyTaskDone();
+  }
+
+  async _onServerEvent(_ep, evt) {
+    let endConversation = false;
+    const type = evt.type;
+    this.logger.info({evt}, 'TaskLlmUltravox_S2S:_onServerEvent');
+
+    /* server errors of some sort */
+    if (type === 'error') {
+      endConversation = true;
+      this.results = {
+        completionReason: 'server error',
+        error: evt.error
+      };
+    }
+
+    /* tool calls */
+    else if (type === 'client_tool_invocation') {
+      this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent - function_call');
+      if (!this.toolHook) {
+        this.logger.warn({evt}, 'TaskLlmUltravox_S2S:_onServerEvent - no toolHook defined!');
+      }
+      else {
+        const {toolName: name, invocationId: call_id, parameters: args} = evt;
+
+        try {
+          await this.parent.sendToolHook(call_id, {name, args});
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmUltravox_S2S - error calling function');
+          this.results = {
+            completionReason: 'client error calling function',
+            error: err
+          };
+          endConversation = true;
+        }
+      }
+    }
+
+    /* check whether we should notify on this event */
+    if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
+      this.parent.sendEventHook(evt)
+        .catch((err) => this.logger.info({err}, 'TaskLlmUltravox_S2S:_onServerEvent - error sending event hook'));
+    }
+
+    if (endConversation) {
+      this.logger.info({results: this.results},
+        'TaskLlmUltravox_S2S:_onServerEvent - ending conversation due to error');
+      this.notifyTaskDone();
+    }
+  }
+
+  async processToolOutput(ep, tool_call_id, data) {
+    try {
+      this.logger.debug({tool_call_id, data}, 'TaskLlmUltravox_S2S:processToolOutput');
+
+      if (!data.type || data.type !== 'client_tool_result') {
+        this.logger.info({data},
+          'TaskLlmUltravox_S2S:processToolOutput - invalid tool output, must be client_tool_result');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+      }
+    } catch (err) {
+      this.logger.info({err}, 'TaskLlmUltravox_S2S:processToolOutput');
+    }
+  }
+
+  _populateEvents(events) {
+    if (events.includes('all')) {
+      /* work by excluding specific events */
+      const exclude = events
+        .filter((evt) => evt.startsWith('-'))
+        .map((evt) => evt.slice(1));
+      if (exclude.length === 0) this.includeEvents = ultravox_server_events;
+      else this.excludeEvents = expandWildcards(exclude);
+    }
+    else {
+      /* work by including specific events */
+      const include = events
+        .filter((evt) => !evt.startsWith('-'));
+      this.includeEvents = expandWildcards(include);
+    }
+
+    this.logger.debug({
+      includeEvents: this.includeEvents,
+      excludeEvents: this.excludeEvents
+    }, 'TaskLlmUltravox_S2S:_populateEvents');
+  }
+}
+
+module.exports = TaskLlmUltravox_S2S;

lib/tasks/make_task.js

@@ -84,6 +84,7 @@ function makeTask(logger, obj, parent) {
       const TaskTranscribe = require('./transcribe');
       return new TaskTranscribe(logger, data, parent);
     case TaskName.Listen:
+    case TaskName.Stream:
       const TaskListen = require('./listen');
       return new TaskListen(logger, data, parent);
     case TaskName.Redirect:

lib/tasks/tts-task.js

@@ -59,7 +59,7 @@ class TtsTask extends Task {
   }
 
   async setTtsStreamingChannelVars(vendor, language, voice, credentials, ep) {
-    const {api_key, model_id} = credentials;
+    const {api_key, model_id, custom_tts_streaming_url, auth_token} = credentials;
     const {stability, similarity_boost, use_speaker_boost, style} = this.options;
     let obj;
 
@@ -94,8 +94,36 @@ class TtsTask extends Task {
           ...(style && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_STYLE: style})
         };
         break;
+      case 'rimelabs':
+        const {
+          pauseBetweenBrackets, phonemizeBetweenBrackets, inlineSpeedAlpha, speedAlpha, reduceLatency
+        } = this.options;
+        obj = {
+          RIMELABS_API_KEY: api_key,
+          RIMELABS_TTS_STREAMING_MODEL_ID: model_id,
+          RIMELABS_TTS_STREAMING_VOICE_ID: voice,
+          RIMELABS_TTS_STREAMING_LANGUAGE: language || 'en',
+          ...(pauseBetweenBrackets && {RIMELABS_TTS_STREAMING_PAUSE_BETWEEN_BRACKETS: pauseBetweenBrackets}),
+          ...(phonemizeBetweenBrackets &&
+            {RIMELABS_TTS_STREAMING_PHONEMIZE_BETWEEN_BRACKETS: phonemizeBetweenBrackets}),
+          ...(inlineSpeedAlpha && {RIMELABS_TTS_STREAMING_INLINE_SPEED_ALPHA: inlineSpeedAlpha}),
+          ...(speedAlpha && {RIMELABS_TTS_STREAMING_SPEED_ALPHA: speedAlpha}),
+          ...(reduceLatency && {RIMELABS_TTS_STREAMING_REDUCE_LATENCY: reduceLatency})
+        };
+        break;
       default:
-        throw new Error(`vendor ${vendor} is not supported for tts streaming yet`);
+        if (vendor.startsWith('custom:')) {
+          const use_tls = custom_tts_streaming_url.startsWith('wss://');
+          obj = {
+            CUSTOM_TTS_STREAMING_HOST: custom_tts_streaming_url.replace(/^(ws|wss):\/\//, ''),
+            CUSTOM_TTS_STREAMING_API_KEY: auth_token,
+            CUSTOM_TTS_STREAMING_VOICE_ID: voice,
+            CUSTOM_TTS_STREAMING_LANGUAGE: language || 'en',
+            CUSTOM_TTS_STREAMING_USE_TLS: use_tls
+          };
+        } else {
+          throw new Error(`vendor ${vendor} is not supported for tts streaming yet`);
+        }
     }
     this.logger.info({vendor, credentials, obj}, 'setTtsStreamingChannelVars');
 

lib/utils/background-task-manager.js

@@ -79,7 +79,7 @@ class BackgroundTaskManager extends Emitter {
   }
 
   // Initiate Listen
-  async _initListen(opts, bugname = 'jambonz-background-listen', ignoreCustomerData = false, type = 'listen') {
+  async _initListen(opts, bugname = 'jambonz-background-listen', ignoreCustomerData = true, type = 'listen') {
     let task;
     try {
       const t = normalizeJambones(this.logger, [opts]);

lib/utils/constants.json

@@ -28,6 +28,7 @@
     "SipRedirect": "sip:redirect",
     "Say": "say",
     "SayLegacy": "say:legacy",
+    "Stream": "stream",
     "Tag": "tag",
     "Transcribe": "transcribe"
   },
@@ -182,6 +183,13 @@
     "Disconnect": "voice_agent_s2s::disconnect",
     "ServerEvent": "voice_agent_s2s::server_event"
   },
+  "LlmEvents_Ultravox": {
+    "Error": "error",
+    "Connect": "ultravox_s2s::connect",
+    "ConnectFailure": "ultravox_s2s::connect_failed",
+    "Disconnect": "ultravox_s2s::disconnect",
+    "ServerEvent": "ultravox_s2s::server_event"
+  },
   "QueueResults": {
     "Bridged": "bridged",
     "Error": "error",
@@ -251,6 +259,16 @@
     "ConnectFailure": "elevenlabs_tts_streaming::connect_failed",
     "Connect": "elevenlabs_tts_streaming::connect"
   },
+  "RimelabsTtsStreamingEvents": {
+    "Empty": "rimelabs_tts_streaming::empty",
+    "ConnectFailure": "rimelabs_tts_streaming::connect_failed",
+    "Connect": "rimelabs_tts_streaming::connect"
+  },
+  "CustomTtsStreamingEvents": {
+    "Empty": "custom_tts_streaming::empty",
+    "ConnectFailure": "custom_tts_streaming::connect_failed",
+    "Connect": "custom_tts_streaming::connect"
+  },
   "TtsStreamingEvents": {
     "Empty": "tts_streaming::empty",
     "Pause": "tts_streaming::pause",
@@ -271,5 +289,9 @@
     "Offline": "OFFLINE",
     "GracefulShutdownInProgress":"SHUTDOWN_IN_PROGRESS"
   },
-  "FEATURE_SERVER" : "feature-server"
+  "FEATURE_SERVER" : "feature-server",
+  "WS_CLOSE_CODES": {
+    "NormalClosure": 1000,
+    "GoingAway": 1001
+  }
 }

lib/utils/db-utils.js

@@ -143,6 +143,7 @@ const speechMapper = (cred) => {
       obj.auth_token = o.auth_token;
       obj.custom_stt_url = o.custom_stt_url;
       obj.custom_tts_url = o.custom_tts_url;
+      obj.custom_tts_streaming_url = o.custom_tts_streaming_url;
     }
   } catch (err) {
     console.log(err);

lib/utils/error.js

@@ -17,8 +17,17 @@ class PlayFileNotFoundError extends NonFatalTaskError {
   }
 }
 
+class HTTPResponseError extends Error {
+  constructor(statusCode) {
+    super('Unexpected HTTP Response');
+    delete this.stack;
+    this.statusCode = statusCode;
+  }
+}
+
 module.exports = {
   SpeechCredentialError,
   NonFatalTaskError,
-  PlayFileNotFoundError
+  PlayFileNotFoundError,
+  HTTPResponseError
 };

lib/utils/http-requestor.js

@@ -16,6 +16,7 @@ const {
   NODE_ENV,
   HTTP_USER_AGENT_HEADER,
 } = require('../config');
+const {HTTPResponseError} = require('./error');
 
 const toBase64 = (str) => Buffer.from(str || '', 'utf8').toString('base64');
 
@@ -190,8 +191,7 @@ class HttpRequestor extends BaseRequestor {
         followRedirects: false
       });
       if (![200, 202, 204].includes(statusCode)) {
-        const err = new Error();
-        err.statusCode = statusCode;
+        const err = new HTTPResponseError(statusCode);
         throw err;
       }
       if (headers['content-type']?.includes('application/json')) {

lib/utils/transcription-utils.js

@@ -43,7 +43,8 @@ const stickyVars = {
     'DEEPGRAM_SPEECH_UTTERANCE_END_MS',
     'DEEPGRAM_SPEECH_VAD_TURNOFF',
     'DEEPGRAM_SPEECH_TAG',
-    'DEEPGRAM_SPEECH_MODEL_VERSION'
+    'DEEPGRAM_SPEECH_MODEL_VERSION',
+    'DEEPGRAM_SPEECH_FILLER_WORDS'
   ],
   aws: [
     'AWS_VOCABULARY_NAME',
@@ -208,7 +209,7 @@ const consolidateTranscripts = (bufferedTranscripts, channel, language, vendor)
     const lastChar = acc.alternatives[0].transcript.slice(-1);
     const firstChar = newTranscript.charAt(0);
 
-    if (lastChar.match(/\d/) && firstChar.match(/\d/)) {
+    if (vendor === 'speechmatics' || (lastChar.match(/\d/) && firstChar.match(/\d/))) {
       acc.alternatives[0].transcript += newTranscript;
     } else {
       acc.alternatives[0].transcript += ` ${newTranscript}`;
@@ -812,7 +813,9 @@ module.exports = (logger) => {
         ...(deepgramOptions.tag) &&
           {DEEPGRAM_SPEECH_TAG: deepgramOptions.tag},
         ...(deepgramOptions.version) &&
-          {DEEPGRAM_SPEECH_MODEL_VERSION: deepgramOptions.version}
+          {DEEPGRAM_SPEECH_MODEL_VERSION: deepgramOptions.version},
+        ...(deepgramOptions.fillerWords) &&
+          {DEEPGRAM_SPEECH_FILLER_WORDS: deepgramOptions.fillerWords}
       };
     }
     else if ('soniox' === vendor) {
@@ -954,6 +957,36 @@ module.exports = (logger) => {
             SPEECHMATICS_TRANSLATION_PARTIALS: speechmaticsOptions.translation_config.enable_partials ? 1 : 0
           }
         ),
+        ...(speechmaticsOptions.transcription_config?.domain &&
+          {SPEECHMATICS_DOMAIN: speechmaticsOptions.transcription_config.domain}),
+        ...{SPEECHMATICS_MAX_DELAY: speechmaticsOptions.transcription_config?.max_delay || 0.7},
+        ...{SPEECHMATICS_MAX_DELAY_MODE: speechmaticsOptions.transcription_config?.max_delay_mode || 'flexible'},
+        ...(speechmaticsOptions.transcription_config?.diarization &&
+            {SPEECHMATICS_DIARIZATION: speechmaticsOptions.transcription_config.diarization}),
+        ...(speechmaticsOptions.transcription_config?.speaker_diarization_config?.speaker_sensitivity &&
+          {SPEECHMATICS_DIARIZATION_SPEAKER_SENSITIVITY:
+            speechmaticsOptions.transcription_config.speaker_diarization_config.speaker_sensitivity}),
+        ...(speechmaticsOptions.transcription_config?.speaker_diarization_config?.max_speakers &&
+          {SPEECHMATICS_DIARIZATION_MAX_SPEAKERS:
+            speechmaticsOptions.transcription_config.speaker_diarization_config.max_speakers}),
+        ...(speechmaticsOptions.transcription_config?.output_locale &&
+          {SPEECHMATICS_OUTPUT_LOCALE: speechmaticsOptions.transcription_config.output_locale}),
+        ...(speechmaticsOptions.transcription_config?.punctuation_overrides?.permitted_marks &&
+          {SPEECHMATICS_PUNCTUATION_ALLOWED:
+            speechmaticsOptions.transcription_config.punctuation_overrides.permitted_marks.join(',')}),
+        ...(speechmaticsOptions.transcription_config?.punctuation_overrides?.sensitivity &&
+          {SPEECHMATICS_PUNCTUATION_SENSITIVITY:
+            speechmaticsOptions.transcription_config?.punctuation_overrides?.sensitivity}),
+        ...(speechmaticsOptions.transcription_config?.operating_point &&
+          {SPEECHMATICS_OPERATING_POINT: speechmaticsOptions.transcription_config.operating_point}),
+        ...(speechmaticsOptions.transcription_config?.enable_entities &&
+          {SPEECHMATICS_ENABLE_ENTTIES: speechmaticsOptions.transcription_config.enable_entities}),
+        ...(speechmaticsOptions.transcription_config?.audio_filtering_config?.volume_threshold &&
+          {SPEECHMATICS_VOLUME_THRESHOLD:
+            speechmaticsOptions.transcription_config.audio_filtering_config.volume_threshold}),
+        ...(speechmaticsOptions.transcription_config?.transcript_filtering_config?.remove_disfluencies &&
+          {SPEECHMATICS_REMOVE_DISFLUENCIES:
+            speechmaticsOptions.transcription_config.transcript_filtering_config.remove_disfluencies})
       };
     }
     else if (vendor.startsWith('custom:')) {

lib/utils/tts-streaming-buffer.js

@@ -214,7 +214,7 @@ class TtsStreamingBuffer extends Emitter {
   }
 
   async _api(ep, args) {
-    const apiCmd = `uuid_${this.vendor}_tts_streaming`;
+    const apiCmd = `uuid_${this.vendor.startsWith('custom:') ? 'custom' : this.vendor}_tts_streaming`;
     const res = await ep.api(apiCmd, `^^|${args.join('|')}`);
     if (!res.body?.startsWith('+OK')) {
       this.logger.info({args}, `Error calling ${apiCmd}: ${res.body}`);
@@ -277,7 +277,9 @@ class TtsStreamingBuffer extends Emitter {
       // DH: add other vendors here as modules are added
       'deepgram',
       'cartesia',
-      'elevenlabs'
+      'elevenlabs',
+      'rimelabs',
+      'custom'
     ].forEach((vendor) => {
       const eventClassName = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}TtsStreamingEvents`;
       const eventClass = require('../utils/constants')[eventClassName];

lib/utils/ws-requestor.js

@@ -1,7 +1,7 @@
 const assert = require('assert');
 const BaseRequestor = require('./base-requestor');
 const short = require('short-uuid');
-const {HookMsgTypes} = require('./constants.json');
+const {HookMsgTypes, WS_CLOSE_CODES} = require('./constants.json');
 const Websocket = require('ws');
 const snakeCaseKeys = require('./snakecase-keys');
 const {
@@ -261,13 +261,13 @@ class WsRequestor extends BaseRequestor {
     }
   }
 
-  close() {
+  close(code = WS_CLOSE_CODES.NormalClosure) {
     this.closedGracefully = true;
-    this.logger.debug('WsRequestor:close closing socket');
+    this.logger.debug(`WsRequestor:close closing socket with code ${code}`);
     this._stopPingTimer();
     try {
       if (this.ws) {
-        this.ws.close(1000);
+        this.ws.close(code);
         this.ws.removeAllListeners();
         this.ws = null;
       }

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "jambonz-feature-server",
-  "version": "0.9.2",
+  "version": "0.9.3",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "jambonz-feature-server",
-      "version": "0.9.2",
+      "version": "0.9.3",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-auto-scaling": "^3.549.0",
@@ -18,7 +18,7 @@
         "@jambonz/speech-utils": "^0.2.1",
         "@jambonz/stats-collector": "^0.1.10",
         "@jambonz/time-series": "^0.2.13",
-        "@jambonz/verb-specifications": "^0.0.91",
+        "@jambonz/verb-specifications": "^0.0.94",
         "@opentelemetry/api": "^1.8.0",
         "@opentelemetry/exporter-jaeger": "^1.23.0",
         "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
@@ -31,8 +31,8 @@
         "bent": "^7.3.12",
         "debug": "^4.3.4",
         "deepcopy": "^2.1.0",
-        "drachtio-fsmrf": "^3.0.46",
-        "drachtio-srf": "^4.5.35",
+        "drachtio-fsmrf": "^4.0.1",
+        "drachtio-srf": "^5.0.1",
         "express": "^4.19.2",
         "express-validator": "^7.0.1",
         "moment": "^2.30.1",
@@ -40,7 +40,7 @@
         "pino": "^8.20.0",
         "polly-ssml-split": "^0.1.0",
         "proxyquire": "^2.1.3",
-        "sdp-transform": "^2.14.2",
+        "sdp-transform": "^2.15.0",
         "short-uuid": "^5.1.0",
         "sinon": "^17.0.1",
         "to-snake-case": "^1.0.0",
@@ -1671,10 +1671,9 @@
       }
     },
     "node_modules/@jambonz/verb-specifications": {
-      "version": "0.0.91",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.91.tgz",
-      "integrity": "sha512-C9UIKytogOdUp5sGqfEetl82/8lDQ8zgEU5diQKpU4dZeA3M8qItqjJGSXexNh4PK9ECUYNorsPoRGNxGcr2IA==",
-      "license": "MIT",
+      "version": "0.0.94",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.94.tgz",
+      "integrity": "sha512-gFqZvbzM+us9T2CLkMaFSjlnclTGCMzuP9BD9fDPJNU36/CaIX3TO+3/EDCcIVhg4+b/smr6cqTHjaLQ5fZn4g==",
       "dependencies": {
         "debug": "^4.3.4",
         "pino": "^8.8.0"
@@ -3969,18 +3968,17 @@
       }
     },
     "node_modules/drachtio-fsmrf": {
-      "version": "3.0.46",
-      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.46.tgz",
-      "integrity": "sha512-ScgyOnsOL45feuKKquT2Gij/jDRdjVha2TnQ6/Me2/M/C+29c9W7cdYlt3UZB3GyodazBukwIy5RrZf1iuXBGw==",
-      "license": "MIT",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-4.0.1.tgz",
+      "integrity": "sha512-QjXHe1d0D7KJTmXAyiKr07TNKPIT6BAPYHqBU8lp51Ms7MgN/nNUCVUPvL3RRUA6FfU7gxaNtYy0LIfdskHVJQ==",
       "dependencies": {
         "camel-case": "^4.1.2",
         "debug": "^2.6.9",
         "delegates": "^0.1.0",
         "drachtio-modesl": "^1.2.9",
-        "drachtio-srf": "^4.5.38",
+        "drachtio-srf": "^5.0.1",
         "only": "^0.0.2",
-        "sdp-transform": "^2.14.1",
+        "sdp-transform": "^2.15.0",
         "snake-case": "^3.0.4",
         "uuid-random": "^1.3.2"
       },
@@ -4027,15 +4025,15 @@
       }
     },
     "node_modules/drachtio-srf": {
-      "version": "4.5.40",
-      "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-4.5.40.tgz",
-      "integrity": "sha512-rybIu6kcY8oyw98z9eYQ5lSOALdEINPVc82N3Updl78qgLJvo9qEQ/j+idawUfMlIuqJ0Ab38wkqNeg5Cn3jNA==",
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-5.0.1.tgz",
+      "integrity": "sha512-HMPn+xLm2TCYY6cJoRpTlhCRakyKgPD93lM1gg6nNIlO9gPSKlUjX3mxL4iaFKuh+UFMMTAoB26fj892A6Lr4Q==",
       "dependencies": {
         "debug": "^3.2.7",
         "delegates": "^0.1.0",
         "node-noop": "^0.0.1",
         "only": "^0.0.2",
-        "sdp-transform": "^2.14.1",
+        "sdp-transform": "^2.15.0",
         "short-uuid": "^4.2.2",
         "sip-methods": "^0.3.0",
         "sip-status": "^0.1.0",
@@ -8172,9 +8170,9 @@
       "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA=="
     },
     "node_modules/sdp-transform": {
-      "version": "2.14.2",
-      "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.14.2.tgz",
-      "integrity": "sha512-icY6jVao7MfKCieyo1AyxFYm1baiM+fA00qW/KrNNVlkxHAd34riEKuEkUe4bBb3gJwLJZM+xT60Yj1QL8rHiA==",
+      "version": "2.15.0",
+      "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.15.0.tgz",
+      "integrity": "sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==",
       "bin": {
         "sdp-verify": "checker.js"
       }
@@ -9004,9 +9002,10 @@
       "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
     },
     "node_modules/tslib": {
-      "version": "2.6.2",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
-      "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+      "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
+      "license": "0BSD"
     },
     "node_modules/type": {
       "version": "1.2.0",
@@ -10939,9 +10938,9 @@
       }
     },
     "@jambonz/verb-specifications": {
-      "version": "0.0.91",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.91.tgz",
-      "integrity": "sha512-C9UIKytogOdUp5sGqfEetl82/8lDQ8zgEU5diQKpU4dZeA3M8qItqjJGSXexNh4PK9ECUYNorsPoRGNxGcr2IA==",
+      "version": "0.0.94",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.94.tgz",
+      "integrity": "sha512-gFqZvbzM+us9T2CLkMaFSjlnclTGCMzuP9BD9fDPJNU36/CaIX3TO+3/EDCcIVhg4+b/smr6cqTHjaLQ5fZn4g==",
       "requires": {
         "debug": "^4.3.4",
         "pino": "^8.8.0"
@@ -12652,17 +12651,17 @@
       }
     },
     "drachtio-fsmrf": {
-      "version": "3.0.46",
-      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.46.tgz",
-      "integrity": "sha512-ScgyOnsOL45feuKKquT2Gij/jDRdjVha2TnQ6/Me2/M/C+29c9W7cdYlt3UZB3GyodazBukwIy5RrZf1iuXBGw==",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-4.0.1.tgz",
+      "integrity": "sha512-QjXHe1d0D7KJTmXAyiKr07TNKPIT6BAPYHqBU8lp51Ms7MgN/nNUCVUPvL3RRUA6FfU7gxaNtYy0LIfdskHVJQ==",
       "requires": {
         "camel-case": "^4.1.2",
         "debug": "^2.6.9",
         "delegates": "^0.1.0",
         "drachtio-modesl": "^1.2.9",
-        "drachtio-srf": "^4.5.38",
+        "drachtio-srf": "^5.0.1",
         "only": "^0.0.2",
-        "sdp-transform": "^2.14.1",
+        "sdp-transform": "^2.15.0",
         "snake-case": "^3.0.4",
         "uuid-random": "^1.3.2"
       },
@@ -12704,15 +12703,15 @@
       }
     },
     "drachtio-srf": {
-      "version": "4.5.40",
-      "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-4.5.40.tgz",
-      "integrity": "sha512-rybIu6kcY8oyw98z9eYQ5lSOALdEINPVc82N3Updl78qgLJvo9qEQ/j+idawUfMlIuqJ0Ab38wkqNeg5Cn3jNA==",
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-5.0.1.tgz",
+      "integrity": "sha512-HMPn+xLm2TCYY6cJoRpTlhCRakyKgPD93lM1gg6nNIlO9gPSKlUjX3mxL4iaFKuh+UFMMTAoB26fj892A6Lr4Q==",
       "requires": {
         "debug": "^3.2.7",
         "delegates": "^0.1.0",
         "node-noop": "^0.0.1",
         "only": "^0.0.2",
-        "sdp-transform": "^2.14.1",
+        "sdp-transform": "^2.15.0",
         "short-uuid": "^4.2.2",
         "sip-methods": "^0.3.0",
         "sip-status": "^0.1.0",
@@ -15777,9 +15776,9 @@
       "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA=="
     },
     "sdp-transform": {
-      "version": "2.14.2",
-      "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.14.2.tgz",
-      "integrity": "sha512-icY6jVao7MfKCieyo1AyxFYm1baiM+fA00qW/KrNNVlkxHAd34riEKuEkUe4bBb3gJwLJZM+xT60Yj1QL8rHiA=="
+      "version": "2.15.0",
+      "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.15.0.tgz",
+      "integrity": "sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw=="
     },
     "semver": {
       "version": "7.5.4",
@@ -16409,9 +16408,9 @@
       "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
     },
     "tslib": {
-      "version": "2.6.2",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
-      "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
+      "version": "2.8.1",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
+      "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
     },
     "type": {
       "version": "1.2.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jambonz-feature-server",
-  "version": "0.9.2",
+  "version": "0.9.3",
   "main": "app.js",
   "engines": {
     "node": ">= 18.x"
@@ -33,7 +33,7 @@
     "@jambonz/realtimedb-helpers": "^0.8.8",
     "@jambonz/speech-utils": "^0.2.1",
     "@jambonz/stats-collector": "^0.1.10",
-    "@jambonz/verb-specifications": "^0.0.91",
+    "@jambonz/verb-specifications": "^0.0.94",
     "@jambonz/time-series": "^0.2.13",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/exporter-jaeger": "^1.23.0",
@@ -47,8 +47,8 @@
     "bent": "^7.3.12",
     "debug": "^4.3.4",
     "deepcopy": "^2.1.0",
-    "drachtio-fsmrf": "^3.0.46",
-    "drachtio-srf": "^4.5.35",
+    "drachtio-fsmrf": "^4.0.1",
+    "drachtio-srf": "^5.0.1",
     "express": "^4.19.2",
     "express-validator": "^7.0.1",
     "moment": "^2.30.1",
@@ -56,7 +56,7 @@
     "pino": "^8.20.0",
     "polly-ssml-split": "^0.1.0",
     "proxyquire": "^2.1.3",
-    "sdp-transform": "^2.14.2",
+    "sdp-transform": "^2.15.0",
     "short-uuid": "^5.1.0",
     "sinon": "^17.0.1",
     "to-snake-case": "^1.0.0",