support assemblyai v3 (#1265)

* support assemblyai v3

* wip

* wip

* wip

* wip

* wip

* wip

.gitignore

@@ -2,6 +2,9 @@
 logs
 *.log
 
+.claude/
+CLAUDE.md 
+
 # Runtime data
 pids
 *.pid

data/example-voicemail-greetings.json

@@ -163,5 +163,16 @@
     "wird sich bei Ihnen melden",
     "ich melde mich bei dir",
     "wir können nicht"
+  ],
+  "it-IT": [
+    "segreteria telefonica",
+    "risponde la segreteria telefonica",
+    "lascia un messaggio",
+    "puoi lasciare un messaggio dopo il segnale",
+    "dopo il segnale acustico",
+    "il numero chiamato non è raggiungibile",
+    "non è raggiungibile",
+    "lascia pure un messaggio",
+    "puoi lasciare un messaggio"
   ]
 }

lib/http-routes/api/create-call.js

@@ -13,7 +13,7 @@ const WsRequestor = require('../../utils/ws-requestor');
 const RootSpan = require('../../utils/call-tracer');
 const dbUtils = require('../../utils/db-utils');
 const { decrypt } = require('../../utils/encrypt-decrypt');
-const { mergeSdpMedia, extractSdpMedia } = require('../../utils/sdp-utils');
+const { mergeSdpMedia, extractSdpMedia, removeVideoSdp } = require('../../utils/sdp-utils');
 const { createCallSchema, customSanitizeFunction } = require('../schemas/create-call');
 const { selectHostPort } = require('../../utils/network');
 const { JAMBONES_DIAL_SBC_FOR_REGISTERED_USER } = require('../../config');
@@ -184,7 +184,11 @@ router.post('/',
         dualEp = await ms.createEndpoint();
         localSdp = mergeSdpMedia(localSdp, dualEp.local.sdp);
       }
-
+      if (process.env.JAMBONES_VIDEO_CALLS_ENABLED_IN_FS) {
+        logger.debug('createCall: removing video sdp');
+        localSdp = removeVideoSdp(localSdp);
+        ep.modify(localSdp);
+      }
       const connectStream = async(remoteSdp) => {
         if (remoteSdp !== sdp) {
           sdp = remoteSdp;

lib/http-routes/schemas/create-call.js

@@ -116,8 +116,8 @@ const customSanitizeFunction = (value) => {
       /* trims characters at the beginning and at the end of a string */
       value = value.trim();
 
-      /* Verify strings including 'http' via new URL */
-      if (value.includes('http')) {
+      // Only attempt to parse if the whole string is a URL
+      if (/^https?:\/\/\S+$/.test(value)) {
         value = new URL(value).toString();
       }
     }

lib/middleware.js

@@ -332,7 +332,7 @@ module.exports = function(srf, logger) {
       }
 
       // Resolve application.speech_synthesis_voice if it's custom voice
-      if (app2.speech_synthesis_vendor === 'google' && app2.speech_synthesis_voice.startsWith('custom_')) {
+      if (app2.speech_synthesis_vendor === 'google' && app2.speech_synthesis_voice?.startsWith('custom_')) {
         const arr = /custom_(.*)/.exec(app2.speech_synthesis_voice);
         if (arr) {
           const google_custom_voice_sid = arr[1];

lib/session/call-session.js

@@ -220,6 +220,18 @@ class CallSession extends Emitter {
     this._synthesizer = synth;
   }
 
+  /**
+   * Say stream enabled
+   */
+
+  get autoStreamTts() {
+    return this._autoStreamTts || false;
+  }
+
+  set autoStreamTts(i) {
+    this._autoStreamTts = i;
+  }
+
   /**
    * ASR TTS fallback
    */
@@ -1085,6 +1097,7 @@ class CallSession extends Emitter {
           return {
             api_key: credential.api_key,
             model_id: credential.model_id,
+            stt_model_id: credential.stt_model_id,
             embedding: credential.embedding,
             options: credential.options
           };
@@ -1096,10 +1109,18 @@ class CallSession extends Emitter {
             options: credential.options
           };
         }
+        else if ('inworld' === vendor) {
+          return {
+            api_key: credential.api_key,
+            model_id: credential.model_id,
+            options: credential.options
+          };
+        }
         else if ('assemblyai' === vendor) {
           return {
             speech_credential_sid: credential.speech_credential_sid,
-            api_key: credential.api_key
+            api_key: credential.api_key,
+            service_version: credential.service_version
           };
         }
         else if ('voxist' === vendor) {
@@ -1799,6 +1820,10 @@ Duration=${duration} `
       .catch((err) => this.logger.debug({err}, 'CallSession:_notifyTaskStatus - Error sending'));
   }
 
+  async _internalTtsStreamingBufferTokens(tokens) {
+    return await this.ttsStreamingBuffer?.bufferTokens(tokens) || {status: 'failed', reason: 'no tts streaming buffer'};
+  }
+
   _lccTtsFlush(opts) {
     this.ttsStreamingBuffer?.flush(opts);
   }

lib/tasks/config.js

@@ -17,7 +17,8 @@ class TaskConfig extends Task {
       'actionHookDelayAction',
       'boostAudioSignal',
       'vad',
-      'ttsStream'
+      'ttsStream',
+      'autoStreamTts'
     ].forEach((k) => this[k] = this.data[k] || {});
 
     if ('notifyEvents' in this.data) {
@@ -117,6 +118,7 @@ class TaskConfig extends Task {
     if (this.hasTtsStream) {
       phrase.push(`${this.ttsStream.enable ? 'enable' : 'disable'} ttsStream`);
     }
+    if ('autoStreamTts' in this.data) phrase.push(`enable Say.stream value ${this.data.autoStreamTts ? 'on' : 'off'}`);
     return `${this.name}{${phrase.join(',')}}`;
   }
 
@@ -296,6 +298,11 @@ class TaskConfig extends Task {
       });
     }
 
+    if ('autoStreamTts' in this.data) {
+      this.logger.info(`Config: autoStreamTts set to ${this.data.autoStreamTts}`);
+      cs.autoStreamTts = this.data.autoStreamTts;
+    }
+
     if (this.hasFillerNoise) {
       const {enable, ...opts} = this.fillerNoise;
       this.logger.info({fillerNoise: this.fillerNoise}, 'Config: fillerNoise');
@@ -330,7 +337,9 @@ class TaskConfig extends Task {
       };
       this.logger.info({opts: this.gatherOpts}, 'Config: enabling ttsStream');
       cs.enableBackgroundTtsStream(this.sayOpts);
-    } else if (!this.ttsStream.enable) {
+    }
+    // only disable ttsStream if it specifically set to false
+    else if (this.ttsStream.enable === false) {
       this.logger.info('Config: disabling ttsStream');
       cs.disableTtsStream();
     }

lib/tasks/dial.js

@@ -271,7 +271,12 @@ class TaskDial extends Task {
     }
     this._removeDtmfDetection(cs.dlg);
     this._removeDtmfDetection(this.dlg);
-    await this._killOutdials();
+    try {
+      await this._killOutdials();
+    }
+    catch (err) {
+      this.logger.info({err}, 'Dial:kill - error killing outdials');
+    }
     if (this.sd) {
       const byeReasonHeader = this.killReason === KillReason.MediaTimeout ? 'Media Timeout' : undefined;
       this.sd.kill(byeReasonHeader);
@@ -281,13 +286,22 @@ class TaskDial extends Task {
     }
     if (this.callSid) sessionTracker.remove(this.callSid);
     if (this.listenTask) {
-      await this.listenTask.kill(cs);
-      this.listenTask.span.end();
+      try {
+        await this.listenTask.kill(cs);
+        this.listenTask?.span?.end();
+      }
+      catch (err) {
+        this.logger.error({err}, 'Dial:kill - error killing listen task');
+      }
       this.listenTask = null;
     }
     if (this.transcribeTask) {
-      await this.transcribeTask.kill(cs);
-      this.transcribeTask.span.end();
+      try {
+        await this.transcribeTask.kill(cs);
+        this.transcribeTask?.span?.end();
+      } catch (err) {
+        this.logger.error({err}, 'Dial:kill - error killing transcribe task');
+      }
       this.transcribeTask = null;
     }
     this.notifyTaskDone();
@@ -411,6 +425,12 @@ class TaskDial extends Task {
         if (this.ep) this.ep.unbridge();
         res.send(202);
         this.logger.info('DialTask:handleRefer - sent 202 Accepted');
+
+        /* if we got the REFER on the parent leg, end the dial task after completing the refer */
+        if (!isChild) {
+          this.logger.info('DialTask:handleRefer - killing dial task after processing REFER on parent leg');
+          cs.currentTask?.kill(cs, KillReason.ReferComplete);
+        }
       } catch (err) {
         this.logger.info({err}, 'DialTask:handleRefer - error processing incoming REFER');
         res.send(err.statusCode || 501);

lib/tasks/dub.js

@@ -83,7 +83,8 @@ class TaskDub extends TtsTask {
       action: 'playOnTrack',
       track: this.track,
       play: this.play,
-      loop: this.loop ? 'loop' : 'once',
+      // drachtio-fsmrf will convert loop from boolean to 'loop' or 'once'
+      loop: this.loop,
       gain: this.gain
     });
   }

lib/tasks/gather.js

@@ -12,6 +12,7 @@ const {
   JambonzTranscriptionEvents,
   AssemblyAiTranscriptionEvents,
   VoxistTranscriptionEvents,
+  CartesiaTranscriptionEvents,
   OpenAITranscriptionEvents,
   VadDetection,
   VerbioTranscriptionEvents,
@@ -138,7 +139,11 @@ class TaskGather extends SttTask {
     try {
       await this.handling(cs, obj);
     } catch (error) {
-      if (error instanceof SpeechCredentialError) {
+      if (
+        // avoid bargein task with sticky will restart forever
+        // throw exception to stop the loop.
+        !this.sticky &&
+        error instanceof SpeechCredentialError) {
         this.logger.info('Gather failed due to SpeechCredentialError, finished!');
         this.notifyTaskDone();
         return;
@@ -546,6 +551,17 @@ class TaskGather extends SttTask {
           this._onVendorConnectFailure.bind(this, cs, ep));
         break;
 
+      case 'cartesia':
+        this.bugname = `${this.bugname_prefix}cartesia_transcribe`;
+        this.addCustomEventListener(ep, CartesiaTranscriptionEvents.Transcription,
+          this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(
+          ep, CartesiaTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, CartesiaTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
+        this.addCustomEventListener(ep, CartesiaTranscriptionEvents.ConnectFailure,
+          this._onVendorConnectFailure.bind(this, cs, ep));
+        break;
+
       case 'speechmatics':
         this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
         this.addCustomEventListener(

lib/tasks/llm/llms/ultravox_s2s.js

@@ -32,12 +32,14 @@ class TaskLlmUltravox_S2S extends Task {
     this.auth = this.parent.auth;
     this.connectionOptions = this.parent.connectOptions;
 
-    const {apiKey} = this.auth || {};
+    const {apiKey, agent_id} = this.auth || {};
     if (!apiKey) throw new Error('auth.apiKey is required for Vendor: Ultravox');
     this.apiKey = apiKey;
+    this.agentId = agent_id;
     this.actionHook = this.data.actionHook;
     this.eventHook = this.data.eventHook;
     this.toolHook = this.data.toolHook;
+    this.llmOptions = this.data.llmOptions || {};
 
     this.results = {
       completionReason: 'normal conversation end'
@@ -105,24 +107,29 @@ class TaskLlmUltravox_S2S extends Task {
       }
       );
       // merge with any existing tools
-      this.data.llmOptions.selectedTools = [
+      this.llmOptions.selectedTools = [
         ...convertedTools,
-        ...(this.data.llmOptions.selectedTools || [])
+        ...(this.llmOptions.selectedTools || [])
       ];
     }
 
     const payload = {
-      ...this.data.llmOptions,
-      model: this.model,
+      ...this.llmOptions,
+      ...(!this.agentId && {
+        model: this.model,
+      }),
       medium: {
-        ...(this.data.llmOptions.medium || {}),
+        ...(this.llmOptions.medium || {}),
         serverWebSocket: {
           inputSampleRate: 8000,
           outputSampleRate: 8000,
         }
       }
     };
-    const {statusCode, body} = await request('https://api.ultravox.ai/api/calls', {
+    const baseUrl = 'https://api.ultravox.ai';
+    const url = this.agentId ?
+      `${baseUrl}/api/agents/${this.agentId}/calls` : `${baseUrl}/api/calls`;
+    const {statusCode, body} = await request(url, {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',

lib/tasks/play.js

@@ -48,7 +48,7 @@ class TaskPlay extends Task {
          */
         ep.once('playback-start', (evt) => {
           this.logger.debug({evt}, 'Play got playback-start');
-          this.cs.stickyEventEmitter.once('uuid_break', (t) => {
+          this.cs.stickyEventEmitter?.once('uuid_break', (t) => {
             if (t?.taskId === this.taskId) {
               this.logger.debug(`Play got kill-playback, executing uuid_break, taskId: ${t?.taskId}`);
               this.ep.api('uuid_break', this.ep.uuid).catch((err) => this.logger.info(err, 'Error killing audio'));

lib/tasks/say.js

@@ -3,24 +3,32 @@ const TtsTask = require('./tts-task');
 const {TaskName, TaskPreconditions} = require('../utils/constants');
 const pollySSMLSplit = require('polly-ssml-split');
 const { SpeechCredentialError } = require('../utils/error');
+const { sleepFor } = require('../utils/helpers');
 
-const breakLengthyTextIfNeeded = (logger,  text) => {
-  const chunkSize = 1000;
+const breakLengthyTextIfNeeded = (logger, text) => {
+  // As The text can be used for tts streaming, we need to break lengthy text into smaller chunks
+// HIGH_WATER_BUFFER_SIZE defined in tts-streaming-buffer.js
+  const chunkSize = 900;
   const isSSML = text.startsWith('<speak>');
-  if (text.length <= chunkSize || !isSSML) return [text];
   const options = {
-    // MIN length
     softLimit: 100,
-    // MAX length, exclude 15 characters <speak></speak>
     hardLimit: chunkSize - 15,
-    // Set of extra split characters (Optional property)
     extraSplitChars: ',;!?',
   };
   pollySSMLSplit.configure(options);
   try {
-    return pollySSMLSplit.split(text);
+    if (text.length <= chunkSize) return [text];
+    if (isSSML) {
+      return pollySSMLSplit.split(text);
+    } else {
+      // Wrap with <speak> and split
+      const wrapped = `<speak>${text}</speak>`;
+      const splitArr = pollySSMLSplit.split(wrapped);
+      // Remove <speak> and </speak> from each chunk
+      return splitArr.map((str) => str.replace(/^<speak>/, '').replace(/<\/speak>$/, ''));
+    }
   } catch (err) {
-    logger.info({err}, 'Error spliting SSML long text');
+    logger.info({err}, 'Error splitting SSML long text');
     return [text];
   }
 };
@@ -39,6 +47,9 @@ class TaskSay extends TtsTask {
     assert.ok((typeof this.data.text === 'string' || Array.isArray(this.data.text)) || this.data.stream === true,
       'Say: either text or stream:true is required');
 
+    this.text = this.data.text ? (Array.isArray(this.data.text) ? this.data.text : [this.data.text])
+      .map((t) => breakLengthyTextIfNeeded(this.logger, t))
+      .flat() : [];
 
     if (this.data.stream === true) {
       this._isStreamingTts = true;
@@ -46,10 +57,6 @@ class TaskSay extends TtsTask {
     }
     else {
       this._isStreamingTts = false;
-      this.text = (Array.isArray(this.data.text) ? this.data.text : [this.data.text])
-        .map((t) => breakLengthyTextIfNeeded(this.logger, t))
-        .flat();
-
       this.loop = this.data.loop || 1;
       this.isHandledByPrimaryProvider = true;
     }
@@ -85,6 +92,10 @@ class TaskSay extends TtsTask {
     }
 
     try {
+      this._isStreamingTts = this._isStreamingTts || cs.autoStreamTts;
+      if (this.isStreamingTts) {
+        this.closeOnStreamEmpty = this.closeOnStreamEmpty || this.text.length !== 0;
+      }
       if (this.isStreamingTts) await this.handlingStreaming(cs, obj);
       else await this.handling(cs, obj);
       this.emit('playDone');
@@ -116,6 +127,54 @@ class TaskSay extends TtsTask {
 
       cs.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_open'})
         .catch((err) => this.logger.info({err}, 'TaskSay:handlingStreaming - Error sending'));
+
+      if (this.text.length !== 0) {
+        this.logger.info('TaskSay:handlingStreaming - sending text to TTS stream');
+        for (const t of this.text) {
+          const result = await cs._internalTtsStreamingBufferTokens(t);
+          if (result?.status === 'failed') {
+            if (result.reason === 'full') {
+              // Retry logic for full buffer
+              const maxRetries = 5;
+              let backoffMs = 1000;
+              for (let retryCount = 0; retryCount < maxRetries && !this.killed; retryCount++) {
+                this.logger.info(
+                  `TaskSay:handlingStreaming - retry ${retryCount + 1}/${maxRetries} after ${backoffMs}ms`);
+                await sleepFor(backoffMs);
+
+                const retryResult = await cs._internalTtsStreamingBufferTokens(t);
+
+                // Exit retry loop on success
+                if (retryResult?.status !== 'failed') {
+                  break;
+                }
+
+                // Handle failure for reason other than full buffer
+                if (retryResult.reason !== 'full') {
+                  this.logger.info(
+                    {result: retryResult}, 'TaskSay:handlingStreaming - TTS stream failed to buffer tokens');
+                  throw new Error(`TTS stream failed to buffer tokens: ${retryResult.reason}`);
+                }
+
+                // Last retry attempt failed
+                if (retryCount === maxRetries - 1) {
+                  this.logger.info('TaskSay:handlingStreaming - Maximum retries exceeded for full buffer');
+                  throw new Error('TTS stream buffer full - maximum retries exceeded');
+                }
+
+                // Increase backoff for next retry
+                backoffMs = Math.min(backoffMs * 1.5, 10000);
+              }
+            } else {
+              // Immediate failure for non-full buffer issues
+              this.logger.info({result}, 'TaskSay:handlingStreaming - TTS stream failed to buffer tokens');
+              throw new Error(`TTS stream failed to buffer tokens: ${result.reason}`);
+            }
+          } else {
+            await cs._lccTtsFlush();
+          }
+        }
+      }
     } catch (err) {
       this.logger.info({err}, 'TaskSay:handlingStreaming - Error setting channel vars');
       cs.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_closed'})
@@ -338,6 +397,7 @@ class TaskSay extends TtsTask {
           .replace('playht_', 'playht.')
           .replace('cartesia_', 'cartesia.')
           .replace('rimelabs_', 'rimelabs.')
+          .replace('inworld_', 'inworld.')
           .replace('verbio_', 'verbio.')
           .replace('elevenlabs_', 'elevenlabs.');
         if (spanMapping[newKey]) newKey = spanMapping[newKey];
@@ -402,6 +462,11 @@ const spanMapping = {
   'rimelabs.name_lookup_time_ms': 'name_lookup_ms',
   'rimelabs.connect_time_ms': 'connect_ms',
   'rimelabs.final_response_time_ms': 'final_response_ms',
+  // inworld
+  'inworld.name_lookup_time_ms': 'name_lookup_ms',
+  'inworld.connect_time_ms': 'connect_ms',
+  'inworld.final_response_time_ms': 'final_response_ms',
+  'inworld.x_envoy_upstream_service_time': 'upstream_service_time',
   // verbio
   'verbio.name_lookup_time_ms': 'name_lookup_ms',
   'verbio.connect_time_ms': 'connect_ms',

lib/tasks/transcribe.js

@@ -14,6 +14,7 @@ const {
   TranscribeStatus,
   AssemblyAiTranscriptionEvents,
   VoxistTranscriptionEvents,
+  CartesiaTranscriptionEvents,
   OpenAITranscriptionEvents,
   VerbioTranscriptionEvents,
   SpeechmaticsTranscriptionEvents
@@ -312,6 +313,17 @@ class TaskTranscribe extends SttTask {
           this._onVendorConnectFailure.bind(this, cs, ep, channel));
         break;
 
+      case 'cartesia':
+        this.bugname = `${this.bugname_prefix}cartesia_transcribe`;
+        this.addCustomEventListener(ep, CartesiaTranscriptionEvents.Transcription,
+          this._onTranscription.bind(this, cs, ep, channel));
+        this.addCustomEventListener(ep,
+          CartesiaTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, CartesiaTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
+        this.addCustomEventListener(ep, CartesiaTranscriptionEvents.ConnectFailure,
+          this._onVendorConnectFailure.bind(this, cs, ep, channel));
+        break;
+
       case 'speechmatics':
         this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
         this.addCustomEventListener(

lib/tasks/tts-task.js

@@ -185,6 +185,9 @@ class TtsTask extends Task {
     } else if (vendor === 'rimelabs') {
       credentials = credentials || {};
       credentials.model_id = this.options.model_id || credentials.model_id;
+    } else if (vendor === 'inworld') {
+      credentials = credentials || {};
+      credentials.model_id = this.options.model_id || credentials.model_id;
     } else if (vendor === 'whisper') {
       credentials = credentials || {};
       credentials.model_id = this.options.model_id || credentials.model_id;

lib/utils/constants.json

@@ -167,6 +167,12 @@
     "ConnectFailure": "voxist_transcribe::connect_failed",
     "Connect": "voxist_transcribe::connect"
   },
+  "CartesiaTranscriptionEvents": {
+    "Transcription": "cartesia_transcribe::transcription",
+    "Error": "cartesia_transcribe::error",
+    "ConnectFailure": "cartesia_transcribe::connect_failed",
+    "Connect": "cartesia_transcribe::connect"
+  },
   "VadDetection": {
     "Detection": "vad_detect:detection"
   },
@@ -237,6 +243,7 @@
   "KillReason": {
     "Hangup": "hangup",
     "Replaced": "replaced",
+    "ReferComplete": "refer-complete",
     "MediaTimeout": "media_timeout"
   },
   "HookMsgTypes": [

lib/utils/db-utils.js

@@ -110,6 +110,7 @@ const speechMapper = (cred) => {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
       obj.model_id = o.model_id;
+      obj.stt_model_id = o.stt_model_id;
       obj.embedding = o.embedding;
       obj.options = o.options;
     }
@@ -119,9 +120,16 @@ const speechMapper = (cred) => {
       obj.model_id = o.model_id;
       obj.options = o.options;
     }
+    else if ('inworld' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = o.api_key;
+      obj.model_id = o.model_id;
+      obj.options = o.options;
+    }
     else if ('assemblyai' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
+      obj.service_version = o.service_version;
     }
     else if ('voxist' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));

lib/utils/transcription-utils.js

@@ -110,6 +110,10 @@ const stickyVars = {
   voxist: [
     'VOXIST_API_KEY',
   ],
+  cartesia: [
+    'CARTESIA_API_KEY',
+    'CARTESIA_MODEL_ID'
+  ],
   speechmatics: [
     'SPEECHMATICS_API_KEY',
     'SPEECHMATICS_HOST',
@@ -519,16 +523,27 @@ const normalizeAws = (evt, channel, language) => {
 
 const normalizeAssemblyAi = (evt, channel, language) => {
   const copy = JSON.parse(JSON.stringify(evt));
+  const alternatives = [];
+  let is_final = false;
+  if (evt.type && evt.type === 'Turn') {
+    // v3 is here
+    alternatives.push({
+      confidence: evt.end_of_turn_confidence,
+      transcript: evt.transcript,
+    });
+    is_final = evt.end_of_turn;
+  } else {
+    alternatives.push({
+      confidence: evt.confidence,
+      transcript: evt.text,
+    });
+    is_final = evt.message_type === 'FinalTranscript';
+  }
   return {
     language_code: language,
     channel_tag: channel,
-    is_final: evt.message_type === 'FinalTranscript',
-    alternatives: [
-      {
-        confidence: evt.confidence,
-        transcript: evt.text,
-      }
-    ],
+    is_final,
+    alternatives,
     vendor: {
       name: 'assemblyai',
       evt: copy
@@ -555,6 +570,25 @@ const normalizeVoxist = (evt, channel, language) => {
   };
 };
 
+const normalizeCartesia = (evt, channel, language) => {
+  const copy = JSON.parse(JSON.stringify(evt));
+  return {
+    language_code: language,
+    channel_tag: channel,
+    is_final: evt.is_final,
+    alternatives: [
+      {
+        confidence: 1.00,
+        transcript: evt.text,
+      }
+    ],
+    vendor: {
+      name: 'cartesia',
+      evt: copy
+    }
+  };
+};
+
 const normalizeSpeechmatics = (evt, channel, language) => {
   const copy = JSON.parse(JSON.stringify(evt));
   const is_final = evt.message === 'AddTranscript';
@@ -636,6 +670,8 @@ module.exports = (logger) => {
         return normalizeAssemblyAi(evt, channel, language, shortUtterance);
       case 'voxist':
         return normalizeVoxist(evt, channel, language);
+      case 'cartesia':
+        return normalizeCartesia(evt, channel, language);
       case 'verbio':
         return normalizeVerbio(evt, channel, language);
       case 'speechmatics':
@@ -993,8 +1029,28 @@ module.exports = (logger) => {
       };
     }
     else if ('assemblyai' === vendor) {
+      const serviceVersion = rOpts.assemblyAiOptions?.serviceVersion || sttCredentials.service_version || 'v2';
+      const {
+        format_turns,
+        end_of_turn_confidence_threshold,
+        min_end_of_turn_silence_when_confident,
+        max_turn_silence
+      } = rOpts.assemblyAiOptions || {};
       opts = {
         ...opts,
+        ASSEMBLYAI_API_VERSION: serviceVersion,
+        ...(serviceVersion === 'v3' && {
+          ...(format_turns && {
+            ASSEMBLYAI_FORMAT_TURNS: format_turns
+          }),
+          ...(end_of_turn_confidence_threshold && {
+            ASSEMBLYAI_END_OF_TURN_CONFIDENCE_THRESHOLD: end_of_turn_confidence_threshold
+          }),
+          ASSEMBLYAI_MIN_END_OF_TURN_SILENCE_WHEN_CONFIDENT: min_end_of_turn_silence_when_confident || 500,
+          ...(max_turn_silence && {
+            ASSEMBLYAI_MAX_TURN_SILENCE: max_turn_silence
+          }),
+        }),
         ...(sttCredentials.api_key) &&
           {ASSEMBLYAI_API_KEY: sttCredentials.api_key},
         ...(rOpts.hints?.length > 0 &&
@@ -1008,6 +1064,16 @@ module.exports = (logger) => {
           {VOXIST_API_KEY: sttCredentials.api_key},
       };
     }
+    else if ('cartesia' === vendor) {
+      opts = {
+        ...opts,
+        ...(sttCredentials.api_key &&
+          {CARTESIA_API_KEY: sttCredentials.api_key}),
+        ...(sttCredentials.stt_model_id && {
+          CARTESIA_MODEL_ID: sttCredentials.stt_model_id
+        })
+      };
+    }
     else if ('openai' === vendor) {
       const {openaiOptions = {}} = rOpts;
       const model = openaiOptions.model || rOpts.model || sttCredentials.model_id || 'whisper-1';

lib/utils/ws-requestor.js

@@ -132,8 +132,8 @@ class WsRequestor extends BaseRequestor {
 
         while (retryCount <= this.maxReconnects) {
           try {
-            this.logger.error({retryCount, maxReconnects: this.maxReconnects},
-              'WsRequestor:request - attempting connection');
+            this.logger.debug({retryCount, maxReconnects: this.maxReconnects},
+              'WsRequestor:request - attempting connection retry');
 
             // Ensure clean state before each connection attempt
             if (this.ws) {
@@ -141,38 +141,29 @@ class WsRequestor extends BaseRequestor {
               this.ws = null;
             }
 
-            this.logger.error({retryCount}, 'WsRequestor:request - calling _connect()');
             const startAt = process.hrtime();
             await this._connect();
             const rtt = this._roundTrip(startAt);
             this.stats.histogram('app.hook.connect_time', rtt, ['hook_type:app']);
-            this.logger.error({retryCount}, 'WsRequestor:request - connection successful, exiting retry loop');
             lastError = null;
             break;
           } catch (error) {
             lastError = error;
             retryCount++;
-            this.logger.error({error: error.message, retryCount, maxReconnects: this.maxReconnects},
-              'WsRequestor:request - connection attempt failed');
 
             if (retryCount <= this.maxReconnects &&
                 this.retryPolicyValues?.length &&
                 this._shouldRetry(error, this.retryPolicyValues)) {
 
-              this.logger.error(
-                {url, error, retryCount, maxRetries: this.maxReconnects},
-                `WsRequestor:request - connection failed, retrying (${retryCount}/${this.maxReconnects})`
-              );
-
               const delay = this.backoffMs;
               this.backoffMs = this.backoffMs < 2000 ? this.backoffMs * 2 : (this.backoffMs + 2000);
-              this.logger.error({delay}, 'WsRequestor:request - waiting before retry');
+              this.logger.debug({delay}, 'WsRequestor:request - waiting before retry');
               await new Promise((resolve) => setTimeout(resolve, delay));
-              this.logger.error('WsRequestor:request - retry delay complete, attempting retry');
               continue;
             }
-            this.logger.error({lastError: lastError.message, retryCount, maxReconnects: this.maxReconnects},
-              'WsRequestor:request - throwing last error');
+
+            this.logger.error({error: error.message, retryCount, maxReconnects: this.maxReconnects},
+              'WsRequestor:request - all connection attempts failed');
             throw lastError;
           }
         }
@@ -370,7 +361,7 @@ class WsRequestor extends BaseRequestor {
 
       this
         .once('ready', (ws) => {
-          this.logger.error({retryCount: 'unknown'}, 'WsRequestor:_connect - ready event fired, resolving Promise');
+          this.logger.debug('WsRequestor:_connect - ready event fired, resolving Promise');
           this.removeAllListeners('not-ready');
           if (this.connections > 1) this.request('session:reconnect', this.url);
           resolve();

package-lock.json

@@ -15,10 +15,10 @@
         "@jambonz/http-health-check": "^0.0.1",
         "@jambonz/mw-registrar": "^0.2.7",
         "@jambonz/realtimedb-helpers": "^0.8.13",
-        "@jambonz/speech-utils": "^0.2.11",
+        "@jambonz/speech-utils": "^0.2.13",
         "@jambonz/stats-collector": "^0.1.10",
         "@jambonz/time-series": "^0.2.13",
-        "@jambonz/verb-specifications": "^0.0.104",
+        "@jambonz/verb-specifications": "^0.0.106",
         "@modelcontextprotocol/sdk": "^1.9.0",
         "@opentelemetry/api": "^1.8.0",
         "@opentelemetry/exporter-jaeger": "^1.23.0",
@@ -32,7 +32,7 @@
         "bent": "^7.3.12",
         "debug": "^4.3.4",
         "deepcopy": "^2.1.0",
-        "drachtio-fsmrf": "^4.0.3",
+        "drachtio-fsmrf": "^4.0.4",
         "drachtio-srf": "^5.0.5",
         "express": "^4.19.2",
         "express-validator": "^7.0.1",
@@ -1466,9 +1466,10 @@
       }
     },
     "node_modules/@jambonz/speech-utils": {
-      "version": "0.2.11",
-      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.2.11.tgz",
-      "integrity": "sha512-5V+OJUUnK1CpKKrB0PrAbGVDcwRGQYH/ZPHFMBayW67XaNRpiL3b9jDpMvq35yzx8MGY18JpzCPgVKHTxERlqQ==",
+      "version": "0.2.13",
+      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.2.13.tgz",
+      "integrity": "sha512-8ISTWTfz3fWtPmzPDsZG8zgnf6pTjLA1WasMAF/d/ktGswqVsbhoPcDh5ZyZ7BsEqOMLMIv2Hn0ESmrBuMn5kw==",
+      "license": "MIT",
       "dependencies": {
         "23": "^0.0.0",
         "@aws-sdk/client-polly": "^3.496.0",
@@ -1504,9 +1505,9 @@
       }
     },
     "node_modules/@jambonz/verb-specifications": {
-      "version": "0.0.104",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.104.tgz",
-      "integrity": "sha512-G1LjK6ISujdg0zALudtUvdaPXmvA4FU6x3s8S9MwUbWbFo2WERMUcNOgQAutDZwOMrLH9DnbPL8ZIdnTCKnlkA==",
+      "version": "0.0.106",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.106.tgz",
+      "integrity": "sha512-xBCGKKW5QC7ItZyeF22esytpG2yIhkGWIvBgTaf97CilQmUdLGo3rWG3i7qnRvU9MPXFCtVCMt/aaMt1Ep6V2g==",
       "license": "MIT",
       "dependencies": {
         "debug": "^4.3.4",
@@ -3224,7 +3225,9 @@
       "license": "MIT"
     },
     "node_modules/brace-expansion": {
-      "version": "1.1.11",
+      "version": "1.1.12",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
+      "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==",
       "dev": true,
       "license": "MIT",
       "dependencies": {
@@ -3874,14 +3877,14 @@
       }
     },
     "node_modules/drachtio-fsmrf": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-4.0.3.tgz",
-      "integrity": "sha512-5j8LqPMHJEgK56gI6MTVbasxCS4cUjo9UdPO8P9qJGJfLG/k/LI6QQAzPrFUcGlpOQ3WYZNkOp/drsKdttlk2Q==",
+      "version": "4.0.4",
+      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-4.0.4.tgz",
+      "integrity": "sha512-YJZya9b0CXQWSpdUMS7Gcu4fwLSCNm3+RHIUOuCPjs/b+QHlFEKtpNnaBVAh+BI2pXXSq9bN+YTWnU0ww6vUMA==",
       "dependencies": {
         "camel-case": "^4.1.2",
-        "debug": "^2.6.9",
+        "debug": "^4.4.0",
         "delegates": "^0.1.0",
-        "drachtio-modesl": "^1.2.9",
+        "drachtio-modesl": "^1.3.1",
         "drachtio-srf": "^5.0.1",
         "only": "^0.0.2",
         "sdp-transform": "^2.15.0",
@@ -3892,44 +3895,19 @@
         "node": ">= 6.9.3"
       }
     },
-    "node_modules/drachtio-fsmrf/node_modules/debug": {
-      "version": "2.6.9",
-      "license": "MIT",
-      "dependencies": {
-        "ms": "2.0.0"
-      }
-    },
-    "node_modules/drachtio-fsmrf/node_modules/ms": {
-      "version": "2.0.0",
-      "license": "MIT"
-    },
     "node_modules/drachtio-modesl": {
-      "version": "1.2.9",
-      "resolved": "https://registry.npmjs.org/drachtio-modesl/-/drachtio-modesl-1.2.9.tgz",
-      "integrity": "sha512-Ob/N0ntwd/Qu6IWjRbUr17DSpw9dTpPNMwmi6ZTh8ryGRE29zlx6U446y/VYpN8ZN9rEi0OgTyAmUt3RjLoRyQ==",
-      "license": "MPL-2.0",
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/drachtio-modesl/-/drachtio-modesl-1.3.1.tgz",
+      "integrity": "sha512-v8BozvMz9qfAdQezCXfvoe+virdXjFa12+qTPN9CeGzu0DDZ1bySsHAk40E1Rkegt2v3SLZVpoIYnqxSBnmJSg==",
       "dependencies": {
         "eventemitter2": "^6.4.4",
         "uuid-random": "^1.3.2",
-        "xml2js": "^0.4.19"
+        "xml2js": "^0.6.2"
       },
       "engines": {
         "node": ">=12.x"
       }
     },
-    "node_modules/drachtio-modesl/node_modules/xml2js": {
-      "version": "0.4.23",
-      "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
-      "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
-      "license": "MIT",
-      "dependencies": {
-        "sax": ">=0.6.0",
-        "xmlbuilder": "~11.0.0"
-      },
-      "engines": {
-        "node": ">=4.0.0"
-      }
-    },
     "node_modules/drachtio-srf": {
       "version": "5.0.5",
       "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-5.0.5.tgz",
@@ -4487,8 +4465,7 @@
     "node_modules/eventemitter2": {
       "version": "6.4.9",
       "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.9.tgz",
-      "integrity": "sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==",
-      "license": "MIT"
+      "integrity": "sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg=="
     },
     "node_modules/events": {
       "version": "3.3.0",

package.json

@@ -31,10 +31,10 @@
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/mw-registrar": "^0.2.7",
     "@jambonz/realtimedb-helpers": "^0.8.13",
-    "@jambonz/speech-utils": "^0.2.11",
+    "@jambonz/speech-utils": "^0.2.13",
     "@jambonz/stats-collector": "^0.1.10",
     "@jambonz/time-series": "^0.2.13",
-    "@jambonz/verb-specifications": "^0.0.104",
+    "@jambonz/verb-specifications": "^0.0.106",
     "@modelcontextprotocol/sdk": "^1.9.0",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/exporter-jaeger": "^1.23.0",
@@ -48,7 +48,7 @@
     "bent": "^7.3.12",
     "debug": "^4.3.4",
     "deepcopy": "^2.1.0",
-    "drachtio-fsmrf": "^4.0.3",
+    "drachtio-fsmrf": "^4.0.4",
     "drachtio-srf": "^5.0.5",
     "express": "^4.19.2",
     "express-validator": "^7.0.1",

test/index.js

@@ -16,6 +16,7 @@ require('./sip-request-tests');
 require('./create-call-test');
 require('./play-tests');
 require('./sip-refer-tests');
+require('./sip-refer-handler-tests');
 require('./listen-tests');
 require('./config-test');
 require('./queue-test');

test/scenarios/uas-dial-refer.xml

@@ -0,0 +1,117 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+<!DOCTYPE scenario SYSTEM "sipp.dtd">
+
+<scenario name="UAS that accepts call and sends REFER">
+  <!-- Receive incoming INVITE -->
+  <recv request="INVITE" crlf="true">
+    <action>
+      <ereg regexp=".*" search_in="hdr" header="Subject:" assign_to="1" />
+      <ereg regexp=".*" search_in="hdr" header="From:" assign_to="2" />
+    </action>
+  </recv>
+
+  <!-- Send 180 Ringing -->
+  <send>
+    <![CDATA[
+      SIP/2.0 180 Ringing
+      [last_Via:]
+      [last_From:]
+      [last_To:];tag=[pid]SIPpTag01[call_number]
+      [last_Call-ID:]
+      [last_CSeq:]
+      [last_Record-Route:]
+      Subject:[$1]
+      Content-Length: 0
+    ]]>
+  </send>
+
+  <!-- Send 200 OK with SDP -->
+  <send>
+    <![CDATA[
+      SIP/2.0 200 OK
+      [last_Via:]
+      [last_From:]
+      [last_To:];tag=[pid]SIPpTag01[call_number]
+      [last_Call-ID:]
+      [last_CSeq:]
+      [last_Record-Route:]
+      Subject:[$1]
+      Contact: <sip:[local_ip]:[local_port];transport=[transport]>
+      Content-Type: application/sdp
+      Content-Length: [len]
+
+      v=0
+      o=user1 53655765 2353687637 IN IP[local_ip_type] [local_ip]
+      s=-
+      c=IN IP[media_ip_type] [media_ip]
+      t=0 0
+      m=audio [media_port] RTP/AVP 0
+      a=rtpmap:0 PCMU/8000
+    ]]>
+  </send>
+
+<recv request="ACK" rtd="true" crlf="true">
+  <action>
+    <!-- Check if this is NOT the first call (tag ends with 012 or higher) -->
+    <ereg regexp="tag=1SIPpTag01[2-9]" search_in="hdr" header="To:" assign_to="3" />
+    <log message="Not first call check result: [$3]"/>
+  </action>
+</recv>
+
+<!-- Skip REFER if we found a non-first call tag -->
+<nop next="skip_refer" test="3" value="" compare="not_equal">
+  <action>
+    <log message="Found non-first call tag [$3], skipping REFER"/>
+  </action>
+</nop>
+
+<!-- Wait a moment, then send REFER (only on first call) -->
+<pause milliseconds="1000"/>
+
+<nop>
+  <action>
+    <log message="Sending REFER for first call"/>
+  </action>
+</nop>
+
+<!-- Send REFER (only on first iteration) -->
+  <send retrans="500">
+    <![CDATA[
+      REFER sip:service@[remote_ip]:[remote_port] SIP/2.0
+      Via: SIP/2.0/[transport] [local_ip]:[local_port];branch=[branch]
+      From: <sip:[local_ip]:[local_port]>;tag=[pid]SIPpTag01[call_number]
+      To: [$2]
+      [last_Call-ID:]
+      CSeq: 2 REFER
+      Contact: <sip:[local_ip]:[local_port];transport=[transport]>
+      Max-Forwards: 70
+      X-Call-Number: [call_number]
+      Refer-To: <sip:+15551234567@example.com>
+      Referred-By: <sip:[local_ip]:[local_port]>
+      Content-Length: 0
+    ]]>
+  </send>
+
+  <!-- Expect 202 Accepted (only on first iteration) -->
+  <recv response="202"/>
+  
+  <label id="skip_refer"/>
+
+  <!-- Wait for BYE from feature server -->
+  <recv request="BYE"/>
+
+  <!-- Send 200 OK to BYE -->
+  <send>
+    <![CDATA[
+      SIP/2.0 200 OK
+      [last_Via:]
+      [last_From:]
+      [last_To:]
+      [last_Call-ID:]
+      [last_CSeq:]
+      Contact: <sip:[local_ip]:[local_port];transport=[transport]>
+      Content-Length: 0
+    ]]>
+  </send>
+
+</scenario>

test/sip-refer-handler-tests.js

@@ -0,0 +1,90 @@
+const test = require('tape');
+const { sippUac } = require('./sipp')('test_fs');
+const bent = require('bent');
+const getJSON = bent('json')
+const clearModule = require('clear-module');
+const {provisionCallHook} = require('./utils');
+const { sleepFor } = require('../lib/utils/helpers');
+
+process.on('unhandledRejection', (reason, p) => {
+  console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+});
+  
+function connect(connectable) {
+  return new Promise((resolve, reject) => {
+    connectable.on('connect', () => {
+      return resolve();
+    });
+  });
+}
+
+test('when parent leg recvs REFER it should end the dial after adulting child leg', async(t) => {
+    clearModule.all();
+    const {srf, disconnect} = require('../app');
+    try {
+      await connect(srf);
+      // wait for fs connected to drachtio server.
+      await sleepFor(1000);
+      
+      // GIVEN
+      const from = "dial_refer_handler";
+      let verbs = [
+        {
+          "verb": "dial",
+          "callerId": from,
+          "actionHook": "/actionHook",
+          "referHook": "/referHook",
+          "anchorMedia": true,
+          "target": [
+              {
+                  "type": "phone",
+                  "number": "15083084809"
+              }
+          ]
+        }
+      ];
+  
+      await provisionCallHook(from, verbs);
+  
+      // THEN
+      //const p = sippUac('uas-dial.xml', '172.38.0.10', undefined, undefined, 2);
+      const p = sippUac('uas-dial-refer.xml', '172.38.0.10', undefined, undefined, 2);
+      await sleepFor(1000);
+  
+      let account_sid = '622f62e4-303a-49f2-bbe0-eb1e1714e37a';
+  
+      let post = bent('http://127.0.0.1:3000/', 'POST', 'json', 201);
+      post('v1/createCall', {
+        'account_sid':account_sid,
+        "call_hook": {
+          "url": "http://127.0.0.1:3100/",
+          "method": "POST",
+        },
+        "from": from,
+        "to": {
+          "type": "phone",
+          "number": "15583084808"
+      }});
+  
+      await p;
+  
+      // Verify that the referHook was called
+      const obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_referHook`);
+      t.ok(obj.body.from === from,
+        'dial-refer-handler: referHook was called with correct from');
+      t.ok(obj.body.refer_details && obj.body.refer_details.sip_refer_to,
+        'dial-refer-handler: refer_details included in referHook');
+      t.ok(obj.body.refer_details.refer_to_user === '+15551234567',
+        'dial-refer-handler: refer_to_user correctly parsed');
+      t.ok(obj.body.refer_details.referring_call_sid,
+        'dial-refer-handler: referring_call_sid included');
+      t.ok(obj.body.refer_details.referred_call_sid,
+        'dial-refer-handler: referred_call_sid included');
+  
+      disconnect();
+    } catch (err) {
+      console.log(`error received: ${err}`);
+      disconnect();
+      t.error(err);
+    }
+});

test/webhook/app.js

@@ -99,6 +99,24 @@ app.post('/actionHook', (req, res) => {
   return res.sendStatus(200);
 });
 
+/*
+ * referHook
+ */
+app.post('/referHook', (req, res) => {
+  console.log({payload: req.body}, 'POST /referHook');
+  let key = req.body.from + "_referHook"
+  addRequestToMap(key, req, hook_mapping);
+  return res.json([{"verb": "pause", "length": 2}]);
+});
+
+/*
+ * adultingHook
+ */
+app.post('/adulting', (req, res) => {
+  console.log({payload: req.body}, 'POST /adulting');
+  return res.sendStatus(200);
+});
+
 /*
 * customHook
 * For the hook to return