catch error (#1002)

* catch error

* remove notifyTaskDone

.gitignore

@@ -42,4 +42,5 @@ ecosystem.config.js
 test/credentials/*.json
 run-tests.sh
 run-coverage.sh
-.vscode
+.vscode
+.env

app.js

@@ -100,8 +100,13 @@ createHttpListener(logger, srf)
   });
 
 
-setInterval(() => {
+setInterval(async() => {
   srf.locals.stats.gauge('fs.sip.calls.count', sessionTracker.count);
+  // Checking system log level
+  const systemInformation = await srf.locals.dbHelpers.lookupSystemInformation();
+  if (systemInformation && systemInformation.log_level) {
+    logger.level = systemInformation.log_level;
+  }
 }, 20000);
 
 const disconnect = () => {

lib/config.js

@@ -108,6 +108,8 @@ const DEEPGRAM_API_KEY = process.env.DEEPGRAM_API_KEY;
 const ANCHOR_MEDIA_ALWAYS = process.env.ANCHOR_MEDIA_ALWAYS;
 const VMD_HINTS_FILE = process.env.VMD_HINTS_FILE;
 
+const JAMBONES_AWS_TRANSCRIBE_USE_GRPC = process.env.JAMBONES_AWS_TRANSCRIBE_USE_GRPC;
+
 /* security, secrets */
 const LEGACY_CRYPTO = !!process.env.LEGACY_CRYPTO;
 const JWT_SECRET = process.env.JWT_SECRET;
@@ -190,6 +192,7 @@ module.exports = {
   ANCHOR_MEDIA_ALWAYS,
   VMD_HINTS_FILE,
   JAMBONES_FREESWITCH_MAX_CALL_DURATION_MINS,
+  JAMBONES_AWS_TRANSCRIBE_USE_GRPC,
 
   LEGACY_CRYPTO,
   JWT_SECRET,

lib/http-routes/api/create-call.js

@@ -258,6 +258,8 @@ router.post('/',
               callId: inviteReq.get('Call-ID'),
               accountSid,
               traceId: rootSpan.traceId
+            }, {
+              ...(account.enable_debug_log && {level: 'debug'})
             });
             app.requestor.logger = app.notifier.logger = sipLogger;
             const callInfo = new CallInfo({
@@ -291,6 +293,8 @@ router.post('/',
           },
           cbProvisional: (prov) => {
             const callStatus = prov.body ? CallStatus.EarlyMedia : CallStatus.Ringing;
+            // Update call-id for sbc outbound INVITE
+            cs.callInfo.sbcCallid = prov.get('X-CID');
             if ([180, 183].includes(prov.status) && prov.body) connectStream(prov.body);
             restDial.emit('callStatus', prov.status, !!prov.body);
             cs.emit('callStatusChange', {callStatus, sipStatus: prov.status});

lib/middleware.js

@@ -187,14 +187,20 @@ module.exports = function(srf, logger) {
 
     const {span} = rootSpan.startChildSpan('lookupAccountDetails');
     try {
-      req.locals.accountInfo = await lookupAccountDetails(account_sid);
-      req.locals.service_provider_sid = req.locals.accountInfo?.account?.service_provider_sid;
+      const accountDetail = await lookupAccountDetails(account_sid);
+      const account = accountDetail?.account;
+      req.locals.accountInfo = accountDetail;
+      req.locals.service_provider_sid = account?.service_provider_sid;
       span.end();
-      if (!req.locals.accountInfo.account.is_active) {
+      if (!account?.is_active) {
         logger.info(`Account is inactive or suspended ${account_sid}`);
         // TODO: alert
         return res.send(503, {headers: {'X-Reason': 'Account exists but is inactive'}});
       }
+      // Change the default log level to debug
+      if (account?.enable_debug_log) {
+        req.locals.logger.level = 'debug';
+      }
       logger.debug({accountInfo: req.locals?.accountInfo?.account}, `retrieved account info for ${account_sid}`);
       next();
     } catch (err) {
@@ -379,7 +385,7 @@ module.exports = function(srf, logger) {
     const {rootSpan, siprec, application:app} = req.locals;
     let span;
     try {
-      if (app.tasks && !JAMBONES_MYSQL_REFRESH_TTL) {
+      if (app.tasks && app.tasks?.length > 0 && !JAMBONES_MYSQL_REFRESH_TTL) {
         app.tasks = normalizeJambones(logger, app.tasks).map((tdata) => makeTask(logger, tdata));
         if (0 === app.tasks.length) throw new Error('no application provided');
         return next();

lib/session/call-info.js

@@ -32,6 +32,7 @@ class CallInfo {
       this.sipStatus = 100;
       this.sipReason = 'Trying';
       this.callStatus = CallStatus.Trying;
+      this.sbcCallid = req.get('X-CID');
       this.originatingSipIp = req.get('X-Forwarded-For');
       this.originatingSipTrunkName = req.get('X-Originating-Carrier');
       const {siprec} = req.locals;
@@ -129,6 +130,7 @@ class CallInfo {
       from: this.from,
       to: this.to,
       callId: this.callId,
+      sbcCallid: this.sbcCallid,
       sipStatus: this.sipStatus,
       sipReason: this.sipReason,
       callStatus: this.callStatus,

lib/session/call-session.js

@@ -8,8 +8,7 @@ const {
   KillReason,
   RecordState,
   AllowedSipRecVerbs,
-  AllowedConfirmSessionVerbs,
-  TranscribeStatus
+  AllowedConfirmSessionVerbs
 } = require('../utils/constants');
 const moment = require('moment');
 const assert = require('assert');
@@ -30,7 +29,6 @@ const {
 } = require('../config');
 const bent = require('bent');
 const BackgroundTaskManager = require('../utils/background-task-manager');
-const { isOnhold } = require('../utils/sdp-utils');
 const BADPRECONDITIONS = 'preconditions not met';
 const CALLER_CANCELLED_ERR_MSG = 'Response not sent due to unknown transaction';
 
@@ -965,42 +963,56 @@ class CallSession extends Emitter {
             speech_credential_sid: credential.speech_credential_sid,
             cobalt_server_uri: credential.cobalt_server_uri
           };
-        } else if ('elevenlabs' === vendor) {
+        }
+        else if ('elevenlabs' === vendor) {
           return {
             api_key: credential.api_key,
             model_id: credential.model_id,
             options: credential.options
           };
-        } else if ('playht' === vendor) {
+        }
+        else if ('playht' === vendor) {
           return {
             api_key: credential.api_key,
             user_id: credential.user_id,
             voice_engine: credential.voice_engine,
             options: credential.options
           };
-        } else if ('rimelabs' === vendor) {
+        }
+        else if ('rimelabs' === vendor) {
           return {
             api_key: credential.api_key,
             model_id: credential.model_id,
             options: credential.options
           };
-        } else if ('assemblyai' === vendor) {
+        }
+        else if ('assemblyai' === vendor) {
           return {
             speech_credential_sid: credential.speech_credential_sid,
             api_key: credential.api_key
           };
-        } else if ('whisper' === vendor) {
+        }
+        else if ('whisper' === vendor) {
           return {
             api_key: credential.api_key,
             model_id: credential.model_id
           };
-        }  else if ('verbio' === vendor) {
+        }
+        else if ('verbio' === vendor) {
           return {
             client_id: credential.client_id,
             client_secret: credential.client_secret,
             engine_version: credential.engine_version
           };
-        } else if (vendor.startsWith('custom:')) {
+        }
+        else if ('speechmatics' === vendor) {
+          this.logger.info({credential}, 'CallSession:getSpeechCredentials - speechmatics credential');
+          return {
+            api_key: credential.api_key,
+            speechmatics_stt_uri: credential.speechmatics_stt_uri,
+          };
+        }
+        else if (vendor.startsWith('custom:')) {
           return {
             speech_credential_sid: credential.speech_credential_sid,
             auth_token: credential.auth_token,
@@ -1338,39 +1350,6 @@ class CallSession extends Emitter {
     listenTask.updateListen(opts.listen_status);
   }
 
-  // Jambonz only update transcribe task when it's received RE-INVITE:
-  // 1. Pause transcribe task if the task is running and hold RE-INVITE is received
-  // 2. Resume transcribe task if the task was paused by jambonz due to hold and unhold RE-INVITE is received
-  async pauseOrResumeTranscribeTaskByReInvite(req) {
-    const task = this.currentTask;
-    const transcribeTasks = [];
-    // Gather running transcribe tasks
-    if (this.backgroundTaskManager.isTaskRunning('transcribe')) {
-      transcribeTasks.push(this.backgroundTaskManager.getTask('transcribe'));
-    }
-
-    // Include current task if it is transcribe-related
-    if (task && [TaskName.Dial, TaskName.Transcribe].includes(task.name)) {
-      const transcribeTask = task.name === TaskName.Transcribe ? task : task.transcribeTask;
-      if (transcribeTask) transcribeTasks.push(transcribeTask);
-    }
-
-    const isHold = isOnhold(req.body);
-    const transcribe_status = isHold ? TranscribeStatus.Pause : TranscribeStatus.Resume;
-    this.logger.debug(`updating transcribe_status for ${transcribeTasks.length} transcribe tasks.`);
-    transcribeTasks.forEach((t) => {
-      if (isHold && !t.paused) {
-        t.updateTranscribe(transcribe_status);
-        t.pausedByHold = true;
-      } else if (!isHold && t.pausedByHold) {
-        if (t.paused) {
-          t.updateTranscribe(transcribe_status);
-        }
-        t.pausedByHold = false;
-      }
-    });
-  }
-
   /**
    * perform live call control -- change Transcribe status
    * @param {object} opts
@@ -1610,6 +1589,29 @@ Duration=${duration} `
     this.logger.info({response}, '_lccBoostAudioSignal: response from freeswitch');
   }
 
+  _lccToolOutput(tool_call_id, opts, callSid) {
+    // only valid if we are in an LLM verb
+    const task = this.currentTask;
+    if (!task || !task.name.startsWith('Llm')) {
+      return this.logger.info('CallSession:_lccToolOutput - invalid command since we are not in an llm');
+    }
+
+    task.processToolOutput(tool_call_id, opts, callSid)
+      .catch((err) => this.logger.error(err, 'CallSession:_lccToolOutput'));
+  }
+
+
+  _lccLlmUpdate(opts, callSid) {
+    // only valid if we are in an LLM verb
+    const task = this.currentTask;
+    if (!task || !task.name.startsWith('Llm')) {
+      return this.logger.info('CallSession:_lccLlmUpdate - invalid command since we are not in an llm');
+    }
+
+    task.processLlmUpdate(opts, callSid)
+      .catch((err) => this.logger.error(err, 'CallSession:_lccLlmUpdate'));
+  }
+
 
   /**
    * perform call hangup by jambonz
@@ -1670,6 +1672,12 @@ Duration=${duration} `
     else if (opts.boostAudioSignal) {
       return this._lccBoostAudioSignal(opts, callSid);
     }
+    else if (opts.llm_tool_output) {
+      return this._lccToolOutput(opts.tool_call_id, opts.llm_tool_output, callSid);
+    }
+    else if (opts.llm_update) {
+      return this._lccLlmUpdate(opts.llm_update, callSid);
+    }
 
     // whisper may be the only thing we are asked to do, or it may that
     // we are doing a whisper after having muted, paused recording etc..
@@ -1866,7 +1874,7 @@ Duration=${duration} `
     this._jambonzHangup();
   }
 
-  async _onCommand({msgid, command, call_sid, queueCommand, data}) {
+  async _onCommand({msgid, command, call_sid, queueCommand, tool_call_id, data}) {
     this.logger.info({msgid, command, queueCommand, data}, 'CallSession:_onCommand - received command');
     let resolution;
     switch (command) {
@@ -1967,6 +1975,14 @@ Duration=${duration} `
           });
         break;
 
+      case 'llm:tool-output':
+        this._lccToolOutput(tool_call_id, data, call_sid);
+        break;
+
+      case 'llm:update':
+        this._lccLlmUpdate(data, call_sid);
+        break;
+
       default:
         this.logger.info(`CallSession:_onCommand - invalid command ${command}`);
     }
@@ -2276,8 +2292,6 @@ Duration=${duration} `
         this.logger.info('got reINVITE but no endpoint and media has not been released');
         res.send(488);
       }
-
-      await this.pauseOrResumeTranscribeTaskByReInvite(req);
     } catch (err) {
       this.logger.error(err, 'Error handling reinvite');
     }

lib/session/inbound-call-session.js

@@ -72,6 +72,8 @@ class InboundCallSession extends CallSession {
 
   _jambonzHangup() {
     this.dlg?.destroy();
+    // kill current task or wakeup the call session.
+    this._callReleased();
   }
 
   _hangup(terminatedBy = 'jambonz') {

lib/tasks/dial.js

@@ -340,15 +340,17 @@ class TaskDial extends Task {
 
         const to = parseUri(req.getParsedHeader('Refer-To').uri);
         const by = parseUri(req.getParsedHeader('Referred-By').uri);
+        const referredBy = req.get('Referred-By');
+        const userAgent = req.get('User-Agent');
         this.logger.info({to}, 'refer to parsed');
         const json = await cs.requestor.request('verb:hook', this.referHook, {
           ...(callInfo.toJSON()),
           refer_details: {
             sip_refer_to: req.get('Refer-To'),
-            sip_referred_by: req.get('Referred-By'),
-            sip_user_agent: req.get('User-Agent'),
             refer_to_user: to.scheme === 'tel' ? to.number : to.user,
-            referred_by_user: by.scheme === 'tel' ? by.number : by.user,
+            ...(referredBy && {sip_referred_by: referredBy}),
+            ...(userAgent && {sip_user_agent: userAgent}),
+            ...(by && {referred_by_user: by.scheme === 'tel' ? by.number : by.user}),
             referring_call_sid,
             referred_call_sid
           }
@@ -379,6 +381,7 @@ class TaskDial extends Task {
         res.send(202);
         this.logger.info('DialTask:handleRefer - sent 202 Accepted');
       } catch (err) {
+        this.logger.info({err}, 'DialTask:handleRefer - error processing incoming REFER');
         res.send(err.statusCode || 501);
       }
     }
@@ -484,7 +487,7 @@ class TaskDial extends Task {
   }
 
   async _attemptCalls(cs) {
-    const {req, srf} = cs;
+    const {req, callInfo, direction, srf} = cs;
     const {getSBC} = srf.locals;
     const {lookupTeamsByAccount, lookupAccountBySid} = srf.locals.dbHelpers;
     const {lookupCarrier, lookupCarrierByPhoneNumber} = dbUtils(this.logger, cs.srf);
@@ -496,8 +499,11 @@ class TaskDial extends Task {
     this.headers = {
       'X-Account-Sid': cs.accountSid,
       ...(req && req.has('X-CID') && {'X-CID': req.get('X-CID')}),
-      ...(req && req.has('P-Asserted-Identity') && !JAMBONZ_DISABLE_DIAL_PAI_HEADER &&
-        {'P-Asserted-Identity': req.get('P-Asserted-Identity')}),
+      ...(direction === 'outbound' && callInfo.sbcCallid && {'X-CID': callInfo.sbcCallid}),
+      ...(!JAMBONZ_DISABLE_DIAL_PAI_HEADER && req && {
+        ...(req.has('P-Asserted-Identity') && {'P-Asserted-Identity': req.get('P-Asserted-Identity')}),
+        ...(req.has('Privacy') && {'Privacy': req.get('Privacy')}),
+      }),
       ...(req && req.has('X-Voip-Carrier-Sid') && {'X-Voip-Carrier-Sid': req.get('X-Voip-Carrier-Sid')}),
       // Put headers at the end to make sure opt.headers override all default behavior.
       ...this.headers
@@ -612,6 +618,7 @@ class TaskDial extends Task {
                 dialCallStatus: obj.callStatus,
                 dialSipStatus: obj.sipStatus,
                 dialCallSid: sd.callSid,
+                dialSbcCallid: sd.callInfo.sbcCallid
               });
             }
             switch (obj.callStatus) {

lib/tasks/gather.js

@@ -12,7 +12,8 @@ const {
   JambonzTranscriptionEvents,
   AssemblyAiTranscriptionEvents,
   VadDetection,
-  VerbioTranscriptionEvents
+  VerbioTranscriptionEvents,
+  SpeechmaticsTranscriptionEvents
 } = require('../utils/constants.json');
 const {
   JAMBONES_GATHER_EARLY_HINTS_MATCH,
@@ -514,6 +515,24 @@ class TaskGather extends SttTask {
         this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.ConnectFailure,
           this._onVendorConnectFailure.bind(this, cs, ep));
         break;
+
+      case 'speechmatics':
+        this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
+        this.addCustomEventListener(
+          ep, SpeechmaticsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Info,
+          this._onSpeechmaticsInfo.bind(this, cs, ep));
+        this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.RecognitionStarted,
+          this._onSpeechmaticsRecognitionStarted.bind(this, cs, ep));
+        this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Connect,
+          this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.ConnectFailure,
+          this._onVendorConnectFailure.bind(this, cs, ep));
+        this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Error,
+          this._onSpeechmaticsErrror.bind(this, cs, ep));
+
+        break;
+
       default:
         if (this.vendor.startsWith('custom:')) {
           this.bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
@@ -752,7 +771,7 @@ class TaskGather extends SttTask {
 
     evt = this.normalizeTranscription(evt, this.vendor, 1, this.language,
       this.shortUtterance, this.data.recognizer.punctuation);
-    //this.logger.debug({evt, bugname, finished, vendor: this.vendor}, 'Gather:_onTranscription normalized transcript');
+    this.logger.debug({evt, bugname, finished, vendor: this.vendor}, 'Gather:_onTranscription normalized transcript');
 
     if (evt.alternatives.length === 0) {
       this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, continue listening');
@@ -987,10 +1006,11 @@ class TaskGather extends SttTask {
       if (code === 413 && error === 'Too much speech') return this._resolve('timeout');
     }
     this.logger.info({evt}, 'TaskGather:_onJambonzError');
+    const errMessage = evt.error || evt.Message;
     writeAlerts({
       account_sid: cs.accountSid,
       alert_type: AlertType.STT_FAILURE,
-      message: `Custom speech vendor ${this.vendor} error: ${evt.error}`,
+      message: `Custom speech vendor ${this.vendor} error: ${errMessage}`,
       vendor: this.vendor,
       target_sid: cs.callSid
     }).catch((err) => this.logger.info({err}, 'Error generating alert for jambonz custom connection failure'));
@@ -1006,12 +1026,25 @@ class TaskGather extends SttTask {
     }
   }
 
+  async _onSpeechmaticsErrror(cs, _ep, evt) {
+    // eslint-disable-next-line no-unused-vars
+    const {message, ...e} = evt;
+    this._onVendorError(cs, _ep, {error: JSON.stringify(e)});
+  }
+
   async _onVendorError(cs, _ep, evt) {
     super._onVendorError(cs, _ep, evt);
     if (!(await this._startFallback(cs, _ep, evt))) {
       this._resolve('stt-error', evt);
     }
   }
+  async _onSpeechmaticsRecognitionStarted(_cs, _ep, evt) {
+    this.logger.debug({evt}, 'TaskGather:_onSpeechmaticsRecognitionStarted');
+  }
+
+  async _onSpeechmaticsInfo(_cs, _ep, evt) {
+    this.logger.debug({evt}, 'TaskGather:_onSpeechmaticsInfo');
+  }
 
   _onVadDetected(cs, ep) {
     if (this.bargein && this.minBargeinWordCount === 0) {

lib/tasks/llm/index.js

@@ -0,0 +1,96 @@
+const Task = require('../task');
+const {TaskPreconditions} = require('../../utils/constants');
+const TaskLlmOpenAI_S2S = require('./llms/openai_s2s');
+
+class TaskLlm extends Task {
+  constructor(logger, opts) {
+    super(logger, opts);
+    this.preconditions = TaskPreconditions.Endpoint;
+
+    ['vendor', 'model', 'auth', 'connectOptions'].forEach((prop) => {
+      this[prop] = this.data[prop];
+    });
+
+    this.eventHandlers = [];
+
+    // delegate to the specific llm model
+    this.llm = this.createSpecificLlm();
+  }
+
+  get name() { return this.llm.name ; }
+
+  get toolHook() { return this.llm?.toolHook; }
+
+  get eventHook() { return this.llm?.eventHook; }
+
+  get ep() { return this.cs.ep; }
+
+  async exec(cs, {ep}) {
+    await super.exec(cs, {ep});
+    await this.llm.exec(cs, {ep});
+  }
+
+  async kill(cs) {
+    super.kill(cs);
+    await this.llm.kill(cs);
+  }
+
+  createSpecificLlm() {
+    let llm;
+    switch (this.vendor) {
+      case 'openai':
+      case 'microsoft':
+        if (this.model.startsWith('gpt-4o-realtime')) {
+          llm = new TaskLlmOpenAI_S2S(this.logger, this.data, this);
+        }
+        break;
+      default:
+        throw new Error(`Unsupported vendor ${this.vendor} for LLM`);
+    }
+
+    if (!llm) {
+      throw new Error(`Unsupported vendor:model ${this.vendor}:${this.model}`);
+    }
+    return llm;
+  }
+
+  addCustomEventListener(ep, event, handler) {
+    this.eventHandlers.push({ep, event, handler});
+    ep.addCustomEventListener(event, handler);
+  }
+
+  removeCustomEventListeners() {
+    this.eventHandlers.forEach((h) => h.ep.removeCustomEventListener(h.event, h.handler));
+  }
+
+  async sendEventHook(data) {
+    await this.cs?.requestor.request('llm:event', this.eventHook, data);
+  }
+
+  async sendToolHook(tool_call_id, data) {
+    await this.cs?.requestor.request('llm:tool-call', this.toolHook, {tool_call_id, ...data});
+  }
+
+  async processToolOutput(tool_call_id, data) {
+    if (!this.ep.connected) {
+      this.logger.info('TaskLlm:processToolOutput - no connected endpoint');
+      return;
+    }
+    this.llm.processToolOutput(this.ep, tool_call_id, data);
+  }
+
+  async processLlmUpdate(data, callSid) {
+    if (this.ep.connected) {
+      if (typeof this.llm.processLlmUpdate === 'function') {
+        this.llm.processLlmUpdate(this.ep, data, callSid);
+      }
+      else {
+        const {vendor, model} = this.llm;
+        this.logger.info({data, callSid},
+          `TaskLlm:_processLlmUpdate: LLM ${vendor}:${model} does not support llm:update`);
+      }
+    }
+  }
+}
+
+module.exports = TaskLlm;

lib/tasks/llm/llms/openai_s2s.js

@@ -0,0 +1,357 @@
+const Task = require('../../task');
+const TaskName = 'Llm_OpenAI_s2s';
+const {LlmEvents_OpenAI} = require('../../../utils/constants');
+const ClientEvent = 'client.event';
+const SessionDelete = 'session.delete';
+
+const openai_server_events = [
+  'error',
+  'session.created',
+  'session.updated',
+  'conversation.created',
+  'input_audio_buffer.committed',
+  'input_audio_buffer.cleared',
+  'input_audio_buffer.speech_started',
+  'input_audio_buffer.speech_stopped',
+  'conversation.item.created',
+  'conversation.item.input_audio_transcription.completed',
+  'conversation.item.input_audio_transcription.failed',
+  'conversation.item.truncated',
+  'conversation.item.deleted',
+  'response.created',
+  'response.done',
+  'response.output_item.added',
+  'response.output_item.done',
+  'response.content_part.added',
+  'response.content_part.done',
+  'response.text.delta',
+  'response.text.done',
+  'response.audio_transcript.delta',
+  'response.audio_transcript.done',
+  'response.audio.delta',
+  'response.audio.done',
+  'response.function_call_arguments.delta',
+  'response.function_call_arguments.done',
+  'rate_limits.updated',
+  'output_audio.playback_started',
+  'output_audio.playback_stopped',
+];
+
+const expandWildcards = (events) => {
+  const expandedEvents = [];
+
+  events.forEach((evt) => {
+    if (evt.endsWith('.*')) {
+      const prefix = evt.slice(0, -2); // Remove the wildcard ".*"
+      const matchingEvents = openai_server_events.filter((e) => e.startsWith(prefix));
+      expandedEvents.push(...matchingEvents);
+    } else {
+      expandedEvents.push(evt);
+    }
+  });
+
+  return expandedEvents;
+};
+
+class TaskLlmOpenAI_S2S extends Task {
+  constructor(logger, opts, parentTask) {
+    super(logger, opts, parentTask);
+    this.parent = parentTask;
+
+    this.vendor = this.parent.vendor;
+    this.model = this.parent.model;
+    this.auth = this.parent.auth;
+    this.connectionOptions = this.parent.connectOptions;
+
+    const {apiKey} = this.auth || {};
+    if (!apiKey) throw new Error('auth.apiKey is required for OpenAI S2S');
+
+    if (['openai', 'microsoft'].indexOf(this.vendor) === -1) {
+      throw new Error(`Invalid vendor ${this.vendor} for OpenAI S2S`);
+    }
+
+    if ('microsoft' === this.vendor && !this.connectionOptions?.host) {
+      throw new Error('connectionOptions.host is required for Microsoft OpenAI S2S');
+    }
+
+    this.apiKey = apiKey;
+    this.authType = 'microsoft' === this.vendor ? 'query' : 'bearer';
+    this.actionHook = this.data.actionHook;
+    this.eventHook = this.data.eventHook;
+    this.toolHook = this.data.toolHook;
+    const {response_create, session_update} = this.data.llmOptions;
+
+    if (typeof response_create !== 'object') {
+      throw new Error('llmOptions with an initial response.create is required for OpenAI S2S');
+    }
+
+    this.response_create = response_create;
+    this.session_update = session_update;
+
+    this.results = {
+      completionReason: 'normal conversation end'
+    };
+
+    /**
+     * only one of these will have items,
+     * if includeEvents, then these are the events to include
+     * if excludeEvents, then these are the events to exclude
+     */
+    this.includeEvents = [];
+    this.excludeEvents = [];
+
+    /* default to all events if user did not specify */
+    this._populateEvents(this.data.events || openai_server_events);
+
+    this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
+    this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
+  }
+
+  get name() { return TaskName; }
+
+  get host() {
+    const {host} = this.connectionOptions || {};
+    return host || (this.vendor === 'openai' ? 'api.openai.com' : void 0);
+  }
+
+  get path() {
+    const {path} = this.connectionOptions || {};
+    if (path) return path;
+
+    switch (this.vendor) {
+      case 'openai':
+        return 'v1/realtime?model=gpt-4o-realtime-preview-2024-10-01';
+      case 'microsoft':
+        return 'openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-1001&';
+    }
+  }
+
+  async _api(ep, args) {
+    const res = await ep.api('uuid_openai_s2s', `^^|${args.join('|')}`);
+    if (!res.body?.startsWith('+OK')) {
+      throw new Error({args}, `Error calling uuid_openai_s2s: ${res.body}`);
+    }
+  }
+
+  async exec(cs, {ep}) {
+    await super.exec(cs);
+
+    await this._startListening(cs, ep);
+
+    await this.awaitTaskDone();
+
+    /* note: the parent llm verb started the span, which is why this is necessary */
+    await this.parent.performAction(this.results);
+
+    this._unregisterHandlers();
+  }
+
+  async kill(cs) {
+    super.kill(cs);
+
+    this._api(cs.ep, [cs.ep.uuid, SessionDelete])
+      .catch((err) => this.logger.info({err}, 'TaskLlmOpenAI_S2S:kill - error deleting session'));
+
+    this.notifyTaskDone();
+  }
+
+  /**
+   * Send function call output to the OpenAI server in the form of conversation.item.create
+   * per https://platform.openai.com/docs/guides/realtime/function-calls
+   */
+  async processToolOutput(ep, tool_call_id, data) {
+    try {
+      this.logger.debug({tool_call_id, data}, 'TaskLlmOpenAI_S2S:processToolOutput');
+
+      if (!data.type || data.type !== 'conversation.item.create') {
+        this.logger.info({data},
+          'TaskLlmOpenAI_S2S:processToolOutput - invalid tool output, must be conversation.item.create');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+
+        // spec also recommends to send immediate response.create
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify({type: 'response.create'})]);
+      }
+    } catch (err) {
+      this.logger.info({err}, 'TaskLlmOpenAI_S2S:processToolOutput');
+    }
+  }
+
+  /**
+   * Send a session.update to the OpenAI server
+   * Note: creating and deleting conversation items also supported as well as interrupting the assistant
+   */
+  async processLlmUpdate(ep, data, _callSid) {
+    try {
+      this.logger.debug({data, _callSid}, 'TaskLlmOpenAI_S2S:processLlmUpdate');
+
+      if (!data.type || ![
+        'session.update',
+        'conversation.item.create',
+        'conversation.item.delete',
+        'response.cancel'
+      ].includes(data.type)) {
+        this.logger.info({data}, 'TaskLlmOpenAI_S2S:processLlmUpdate - invalid mid-call request');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+      }
+    } catch (err) {
+      this.logger.info({err}, 'TaskLlmOpenAI_S2S:processLlmUpdate');
+    }
+  }
+
+  async _startListening(cs, ep) {
+    this._registerHandlers(ep);
+
+    try {
+      const args = [ep.uuid, 'session.create', this.host, this.path, this.authType, this.apiKey];
+      await this._api(ep, args);
+    } catch (err) {
+      this.logger.error({err}, 'TaskLlmOpenAI_S2S:_startListening');
+      this.notifyTaskDone();
+    }
+  }
+
+  async _sendClientEvent(ep, obj) {
+    let ok = true;
+    this.logger.debug({obj}, 'TaskLlmOpenAI_S2S:_sendClientEvent');
+    try {
+      const args = [ep.uuid, ClientEvent, JSON.stringify(obj)];
+      await this._api(ep, args);
+    } catch (err) {
+      ok = false;
+      this.logger.error({err}, 'TaskLlmOpenAI_S2S:_sendClientEvent - Error');
+    }
+    return ok;
+  }
+
+  async _sendInitialMessage(ep) {
+    let obj = {type: 'response.create', response: this.response_create};
+    if (!await this._sendClientEvent(ep, obj)) {
+      this.notifyTaskDone();
+    }
+
+    /* send immediate session.update if present */
+    else if (this.session_update) {
+      obj = {type: 'session.update', session: this.session_update};
+      this.logger.debug({obj}, 'TaskLlmOpenAI_S2S:_sendInitialMessage - sending session.update');
+      if (!await this._sendClientEvent(ep, obj)) {
+        this.notifyTaskDone();
+      }
+    }
+  }
+
+  _registerHandlers(ep) {
+    this.addCustomEventListener(ep, LlmEvents_OpenAI.Connect, this._onConnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_OpenAI.ConnectFailure, this._onConnectFailure.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_OpenAI.Disconnect, this._onDisconnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_OpenAI.ServerEvent, this._onServerEvent.bind(this, ep));
+  }
+
+  _unregisterHandlers() {
+    this.removeCustomEventListeners();
+  }
+
+  _onError(ep, evt) {
+    this.logger.info({evt}, 'TaskLlmOpenAI_S2S:_onError');
+    this.notifyTaskDone();
+  }
+
+  _onConnect(ep) {
+    this.logger.debug('TaskLlmOpenAI_S2S:_onConnect');
+    this._sendInitialMessage(ep);
+  }
+  _onConnectFailure(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmOpenAI_S2S:_onConnectFailure');
+    this.results = {completionReason: 'connection failure'};
+    this.notifyTaskDone();
+  }
+  _onDisconnect(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmOpenAI_S2S:_onConnectFailure');
+    this.results = {completionReason: 'disconnect from remote end'};
+    this.notifyTaskDone();
+  }
+  async _onServerEvent(ep, evt) {
+    let endConversation = false;
+    const type = evt.type;
+    this.logger.info({evt}, 'TaskLlmOpenAI_S2S:_onServerEvent');
+
+    /* check for failures, such as rate limit exceeded, that should terminate the conversation */
+    if (type === 'response.done' && evt.response.status === 'failed') {
+      endConversation = true;
+      this.results = {
+        completionReason: 'server failure',
+        error: evt.response.status_details?.error
+      };
+    }
+
+    /* server errors of some sort */
+    else if (type === 'error') {
+      endConversation = true;
+      this.results = {
+        completionReason: 'server error',
+        error: evt.error
+      };
+    }
+
+    /* tool calls */
+    else if (type === 'response.output_item.done' && evt.item?.type === 'function_call') {
+      this.logger.debug({evt}, 'TaskLlmOpenAI_S2S:_onServerEvent - function_call');
+      if (!this.toolHook) {
+        this.logger.warn({evt}, 'TaskLlmOpenAI_S2S:_onServerEvent - no toolHook defined!');
+      }
+      else {
+        const {name, call_id} = evt.item;
+        const args = JSON.parse(evt.item.arguments);
+
+        try {
+          await this.parent.sendToolHook(call_id, {name, args});
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmOpenAI - error calling function');
+          this.results = {
+            completionReason: 'client error calling function',
+            error: err
+          };
+          endConversation = true;
+        }
+      }
+    }
+
+    /* check whether we should notify on this event */
+    if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
+      this.parent.sendEventHook(evt)
+        .catch((err) => this.logger.info({err}, 'TaskLlmOpenAI_S2S:_onServerEvent - error sending event hook'));
+    }
+
+    if (endConversation) {
+      this.logger.info({results: this.results}, 'TaskLlmOpenAI_S2S:_onServerEvent - ending conversation due to error');
+      this.notifyTaskDone();
+    }
+  }
+
+  _populateEvents(events) {
+    if (events.includes('all')) {
+      /* work by excluding specific events */
+      const exclude = events
+        .filter((evt) => evt.startsWith('-'))
+        .map((evt) => evt.slice(1));
+      if (exclude.length === 0) this.includeEvents = openai_server_events;
+      else this.excludeEvents = expandWildcards(exclude);
+    }
+    else {
+      /* work by including specific events */
+      const include = events
+        .filter((evt) => !evt.startsWith('-'));
+      this.includeEvents = expandWildcards(include);
+    }
+
+    this.logger.debug({
+      includeEvents: this.includeEvents,
+      excludeEvents: this.excludeEvents
+    }, 'TaskLlmOpenAI_S2S:_populateEvents');
+  }
+}
+
+module.exports = TaskLlmOpenAI_S2S;

lib/tasks/make_task.js

@@ -62,6 +62,9 @@ function makeTask(logger, obj, parent) {
     case TaskName.Message:
       const TaskMessage = require('./message');
       return new TaskMessage(logger, data, parent);
+    case TaskName.Llm:
+      const TaskLlm = require('./llm');
+      return new TaskLlm(logger, data, parent);
     case TaskName.Rasa:
       const TaskRasa = require('./rasa');
       return new TaskRasa(logger, data, parent);

lib/tasks/say.js

@@ -177,32 +177,22 @@ class TaskSay extends TtsTask {
                 account_sid,
                 alert_type: AlertType.TTS_FAILURE,
                 vendor,
-                detail: evt.variable_tts_error
+                detail: evt.variable_tts_error,
+                target_sid
               }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
             }
-            else {
-              this.logger.debug({evt}, 'Say got playback-stop');
-              if (evt.variable_tts_error) {
-                writeAlerts({
-                  account_sid,
-                  alert_type: AlertType.TTS_FAILURE,
-                  vendor,
-                  detail: evt.variable_tts_error,
-                  target_sid
-                }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
-              }
-              if (evt.variable_tts_cache_filename && !this.killed) {
-                const text = parseTextFromSayString(this.text[segment]);
-                addFileToCache(evt.variable_tts_cache_filename, {
-                  account_sid,
-                  vendor,
-                  language,
-                  voice,
-                  engine,
-                  text
-                }).catch((err) => this.logger.info({err}, 'Error adding file to cache'));
-              }
+            if (evt.variable_tts_cache_filename && !this.killed) {
+              const text = parseTextFromSayString(this.text[segment]);
+              addFileToCache(evt.variable_tts_cache_filename, {
+                account_sid,
+                vendor,
+                language,
+                voice,
+                engine,
+                text
+              }).catch((err) => this.logger.info({err}, 'Error adding file to cache'));
             }
+
             if (this._playResolve) {
               evt.variable_tts_error ? this._playReject(new Error(evt.variable_tts_error)) : this._playResolve();
             }

lib/tasks/sip_refer.js

@@ -94,7 +94,10 @@ class TaskSipRefer extends Task {
         }
         if (status >= 200) {
           this.referSpan.setAttributes({'refer.finalNotify': status});
-          await this.performAction({refer_status: 202, final_referred_call_status: status});
+          await this.performAction({refer_status: 202, final_referred_call_status: status})
+            .catch((err) => {
+              this.logger.error(err, 'TaskSipRefer:exec - error performing action finalNotify');
+            });
           this.notifyTaskDone();
         }
       }

lib/tasks/stt-task.js

@@ -3,6 +3,7 @@ const assert = require('assert');
 const crypto = require('crypto');
 const { TaskPreconditions, CobaltTranscriptionEvents } = require('../utils/constants');
 const { SpeechCredentialError } = require('../utils/error');
+const {JAMBONES_AWS_TRANSCRIBE_USE_GRPC} = require('../config');
 
 class SttTask extends Task {
 
@@ -17,12 +18,14 @@ class SttTask extends Task {
       normalizeTranscription,
       setSpeechCredentialsAtRuntime,
       compileSonioxTranscripts,
-      consolidateTranscripts
+      consolidateTranscripts,
+      updateSpeechmaticsPayload
     } = require('../utils/transcription-utils')(logger);
     this.setChannelVarsForStt = setChannelVarsForStt;
     this.normalizeTranscription = normalizeTranscription;
     this.compileSonioxTranscripts = compileSonioxTranscripts;
     this.consolidateTranscripts = consolidateTranscripts;
+    this.updateSpeechmaticsPayload = updateSpeechmaticsPayload;
     this.eventHandlers = [];
     this.isHandledByPrimaryProvider = true;
     /**
@@ -215,14 +218,25 @@ class SttTask extends Task {
           region,
           roleArn
         });
-      this.logger.debug({roleArn}, `got aws access token ${servedFromCache ? 'from cache' : ''}`);
+      this.logger.debug({roleArn}, `(roleArn) got aws access token ${servedFromCache ? 'from cache' : ''}`);
       credentials = {...credentials, accessKeyId, secretAccessKey, sessionToken};
-    } else if (vendor === 'verbio' && credentials.client_id && credentials.client_secret) {
+    }
+    else if (vendor === 'verbio' && credentials.client_id && credentials.client_secret) {
       const {access_token, servedFromCache} = await getVerbioAccessToken(credentials);
       this.logger.debug({client_id: credentials.client_id},
         `got verbio access token ${servedFromCache ? 'from cache' : ''}`);
       credentials.access_token = access_token;
     }
+    else if (vendor == 'aws' && !JAMBONES_AWS_TRANSCRIBE_USE_GRPC) {
+      /* get AWS access token */
+      const {accessKeyId, secretAccessKey, securityToken, region } = credentials;
+      if (!securityToken) {
+        const { servedFromCache, ...newCredentials} = await getAwsAuthToken({accessKeyId, secretAccessKey, region});
+        this.logger.debug({newCredentials}, `got aws security token ${servedFromCache ? 'from cache' : ''}`);
+        credentials = {...newCredentials, region};
+      }
+    }
+
     return credentials;
   }
 

lib/tasks/transcribe.js

@@ -12,7 +12,9 @@ const {
   NvidiaTranscriptionEvents,
   JambonzTranscriptionEvents,
   TranscribeStatus,
-  AssemblyAiTranscriptionEvents
+  AssemblyAiTranscriptionEvents,
+  VerbioTranscriptionEvents,
+  SpeechmaticsTranscriptionEvents
 } = require('../utils/constants.json');
 const { normalizeJambones } = require('@jambonz/verb-specifications');
 const SttTask = require('./stt-task');
@@ -237,6 +239,13 @@ class TaskTranscribe extends SttTask {
         this.addCustomEventListener(ep, SonioxTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
         break;
+
+      case 'verbio':
+        this.bugname = `${this.bugname_prefix}verbio_transcribe`;
+        this.addCustomEventListener(
+          ep, VerbioTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        break;
+
       case 'cobalt':
         this.bugname = `${this.bugname_prefix}cobalt_transcribe`;
         this.addCustomEventListener(ep, CobaltTranscriptionEvents.Transcription,
@@ -294,6 +303,22 @@ class TaskTranscribe extends SttTask {
           this._onVendorConnectFailure.bind(this, cs, ep, channel));
         break;
 
+      case 'speechmatics':
+        this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
+        this.addCustomEventListener(
+          ep, SpeechmaticsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Info,
+          this._onSpeechmaticsInfo.bind(this, cs, ep));
+        this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.RecognitionStarted,
+          this._onSpeechmaticsRecognitionStarted.bind(this, cs, ep));
+        this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Connect,
+          this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.ConnectFailure,
+          this._onVendorConnectFailure.bind(this, cs, ep));
+        this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Error,
+          this._onSpeechmaticsError.bind(this, cs, ep));
+        break;
+
       default:
         if (this.vendor.startsWith('custom:')) {
           this.bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
@@ -644,6 +669,20 @@ class TaskTranscribe extends SttTask {
     }
   }
 
+  async _onSpeechmaticsRecognitionStarted(_cs, _ep, evt) {
+    this.logger.debug({evt}, 'TaskGather:_onSpeechmaticsRecognitionStarted');
+  }
+
+  async _onSpeechmaticsInfo(_cs, _ep, evt) {
+    this.logger.debug({evt}, 'TaskGather:_onSpeechmaticsInfo');
+  }
+
+  async _onSpeechmaticsErrror(cs, _ep, evt) {
+    // eslint-disable-next-line no-unused-vars
+    const {message, ...e} = evt;
+    this._onVendorError(cs, _ep, {error: JSON.stringify(e)});
+  }
+
   _startAsrTimer(channel) {
     if (this.vendor === 'deepgram') return; // no need
     assert(this.isContinuousAsr);

lib/utils/action-hook-delay.js

@@ -24,10 +24,12 @@ class ActionHookDelayProcessor extends Emitter {
     this._active = false;
 
     const enabled = this.init(opts);
-    if (enabled && (!this.actions || !Array.isArray(this.actions) || this.actions.length === 0)) {
+    if (enabled && this.noResponseTimeout &&
+          (!this.actions || !Array.isArray(this.actions) || this.actions.length === 0)) {
       throw new Error('ActionHookDelayProcessor: no actions specified');
     }
-    else if (enabled && this.actions.some((a) => !a.verb || ![TaskName.Say, TaskName.Play].includes(a.verb))) {
+    else if (enabled && this.actions &&
+          this.actions.some((a) => !a.verb || ![TaskName.Say, TaskName.Play].includes(a.verb))) {
       throw new Error(`ActionHookDelayProcessor: invalid actions specified: ${JSON.stringify(this.actions)}`);
     }
   }

lib/utils/constants.json

@@ -14,6 +14,7 @@
     "Leave": "leave",
     "Lex": "lex",
     "Listen": "listen",
+    "Llm": "llm",
     "Message": "message",
     "Pause": "pause",
     "Play": "play",
@@ -126,6 +127,14 @@
     "NoSpeechDetected": "azure_transcribe::no_speech_detected",
     "VadDetected": "azure_transcribe::vad_detected"
   },
+  "SpeechmaticsTranscriptionEvents": {
+    "Transcription": "speechmatics_transcribe::transcription",
+    "Info": "speechmatics_transcribe::info",
+    "RecognitionStarted": "speechmatics_transcribe::recognition_started",
+    "ConnectFailure": "speechmatics_transcribe::connect_failed",
+    "Connect": "speechmatics_transcribe::connect",
+    "Error": "speechmatics_transcribe::error"
+  },
   "JambonzTranscriptionEvents": {
     "Transcription": "jambonz_transcribe::transcription",
     "ConnectFailure": "jambonz_transcribe::connect_failed",
@@ -158,6 +167,13 @@
     "StandbyEnter": "standby-enter",
     "StandbyExit": "standby-exit"
   },
+  "LlmEvents_OpenAI": {
+    "Error": "error",
+    "Connect": "openai_s2s::connect",
+    "ConnectFailure": "openai_s2s::connect_failed",
+    "Disconnect": "openai_s2s::disconnect",
+    "ServerEvent": "openai_s2s::server_event"
+  },
   "QueueResults": {
     "Bridged": "bridged",
     "Error": "error",
@@ -184,6 +200,8 @@
     "dial:confirm",
     "verb:hook",
     "verb:status",
+    "llm:event",
+    "llm:tool-call",
     "jambonz:error"
   ],
   "RecordState": {

lib/utils/db-utils.js

@@ -91,35 +91,47 @@ const speechMapper = (cred) => {
     else if ('cobalt' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.cobalt_server_uri = o.cobalt_server_uri;
-    } else if ('elevenlabs' === obj.vendor) {
+    }
+    else if ('elevenlabs' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
       obj.model_id = o.model_id;
       obj.options = o.options;
-    } else if ('playht' === obj.vendor) {
+    }
+    else if ('playht' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
       obj.user_id = o.user_id;
       obj.voice_engine = o.voice_engine;
       obj.options = o.options;
-    } else if ('rimelabs' === obj.vendor) {
+    }
+    else if ('rimelabs' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
       obj.model_id = o.model_id;
       obj.options = o.options;
-    } else if ('assemblyai' === obj.vendor) {
+    }
+    else if ('assemblyai' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
-    } else if ('whisper' === obj.vendor) {
+    }
+    else if ('whisper' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
       obj.model_id = o.model_id;
-    }  else if ('verbio' === obj.vendor) {
+    }
+    else if ('verbio' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.client_id = o.client_id;
       obj.client_secret = o.client_secret;
       obj.engine_version = o.engine_version;
-    } else if (obj.vendor.startsWith('custom:')) {
+    }
+    else if ('speechmatics' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = o.api_key;
+      obj.speechmatics_stt_uri = o.speechmatics_stt_uri;
+    }
+    else if (obj.vendor.startsWith('custom:')) {
       const o = JSON.parse(decrypt(credential));
       obj.auth_token = o.auth_token;
       obj.custom_stt_url = o.custom_stt_url;

lib/utils/install-srf-locals.js

@@ -1,5 +1,5 @@
 const Mrf = require('drachtio-fsmrf');
-const ip = require('ip');
+const os = require('os');
 const {
   JAMBONES_MYSQL_HOST,
   JAMBONES_MYSQL_USER,
@@ -18,6 +18,19 @@ const {
 const Registrar = require('@jambonz/mw-registrar');
 const assert = require('assert');
 
+function getLocalIp() {
+  const interfaces = os.networkInterfaces();
+  for (const interfaceName in interfaces) {
+    const interface = interfaces[interfaceName];
+    for (const iface of interface) {
+      if (iface.family === 'IPv4' && !iface.internal) {
+        return iface.address;
+      }
+    }
+  }
+  return '127.0.0.1'; // Fallback to localhost if no suitable interface found
+}
+
 function initMS(logger, wrapper, ms) {
   Object.assign(wrapper, {ms, active: true, connects: 1});
   logger.info(`connected to freeswitch at ${ms.address}`);
@@ -139,7 +152,8 @@ function installSrfLocals(srf, logger) {
     lookupAccountBySid,
     lookupAccountCapacitiesBySid,
     lookupSmppGateways,
-    lookupClientByAccountAndUsername
+    lookupClientByAccountAndUsername,
+    lookupSystemInformation
   } = require('@jambonz/db-helpers')({
     host: JAMBONES_MYSQL_HOST,
     user: JAMBONES_MYSQL_USER,
@@ -194,8 +208,8 @@ function installSrfLocals(srf, logger) {
 
   let localIp;
   try {
-    // Either use the configured IP address or call ip.address() to find it
-    localIp = HTTP_IP || ip.address();
+    // Either use the configured IP address or discover it
+    localIp = HTTP_IP || getLocalIp();
   } catch (err) {
     logger.error({err}, 'installSrfLocals - error detecting local ipv4 address');
   }
@@ -215,6 +229,7 @@ function installSrfLocals(srf, logger) {
       lookupAccountCapacitiesBySid,
       lookupSmppGateways,
       lookupClientByAccountAndUsername,
+      lookupSystemInformation,
       updateCallStatus,
       retrieveCall,
       listCalls,

lib/utils/place-outdial.js

@@ -213,6 +213,8 @@ class SingleDialer extends Emitter {
         },
         cbProvisional: (prov) => {
           const status = {sipStatus: prov.status, sipReason: prov.reason};
+          // Update call-id for sbc outbound INVITE
+          this.callInfo.sbcCallid = prov.get('X-CID');
           if ([180, 183].includes(prov.status) && prov.body) {
             if (status.callStatus !== CallStatus.EarlyMedia) {
               status.callStatus = CallStatus.EarlyMedia;

lib/utils/transcription-utils.js

@@ -1,7 +1,4 @@
-const {
-  TaskName,
-} = require('./constants.json');
-
+const {TaskName} = require('./constants.json');
 const stickyVars = {
   google: [
     'GOOGLE_SPEECH_HINTS',
@@ -51,7 +48,13 @@ const stickyVars = {
   aws: [
     'AWS_VOCABULARY_NAME',
     'AWS_VOCABULARY_FILTER_METHOD',
-    'AWS_VOCABULARY_FILTER_NAME'
+    'AWS_VOCABULARY_FILTER_NAME',
+    'AWS_LANGUAGE_MODEL_NAME',
+    'AWS_ACCESS_KEY_ID',
+    'AWS_SECRET_ACCESS_KEY',
+    'AWS_REGION',
+    'AWS_SECURITY_TOKEN',
+    'AWS_PII_ENTITY_TYPES',
   ],
   nuance: [
     'NUANCE_ACCESS_TOKEN',
@@ -100,6 +103,12 @@ const stickyVars = {
   assemblyai: [
     'ASSEMBLYAI_API_KEY',
     'ASSEMBLYAI_WORD_BOOST'
+  ],
+  speechmatics: [
+    'SPEECHMATICS_API_KEY',
+    'SPEECHMATICS_HOST',
+    'SPEECHMATICS_PATH',
+    'SPEECHMATICS_SPEECH_HINTS',
   ]
 };
 
@@ -152,7 +161,12 @@ const selectDefaultDeepgramModel = (task, language) => {
 
 const optimalGoogleModels = {
   'v1' : {
-    'en-IN':['telephony', 'latest_long']
+    'en-IN':['telephony', 'telephony'],
+    'es-DO':['default', 'default'],
+    'es-MX':['default', 'default'],
+    'en-AU':['telephony', 'telephony'],
+    'en-GB':['telephony', 'telephony'],
+    'en-NZ':['telephony', 'telephony']
   },
   'v2' : {
     'en-IN':['telephony', 'long']
@@ -441,16 +455,41 @@ const normalizeMicrosoft = (evt, channel, language, punctuation = true) => {
 
 const normalizeAws = (evt, channel, language) => {
   const copy = JSON.parse(JSON.stringify(evt));
-  return {
-    language_code: language,
-    channel_tag: channel,
-    is_final: evt[0].is_final,
-    alternatives: evt[0].alternatives,
-    vendor: {
-      name: 'aws',
-      evt: copy
-    }
-  };
+  const isGrpcPayload = Array.isArray(evt);
+  if (isGrpcPayload) {
+    /* legacy grpc api */
+    return {
+      language_code: language,
+      channel_tag: channel,
+      is_final: evt[0].is_final,
+      alternatives: evt[0].alternatives,
+      vendor: {
+        name: 'aws',
+        evt: copy
+      }
+    };
+  }
+  else {
+    /* websocket api */
+    const alternatives = evt.Transcript?.Results[0]?.Alternatives.map((alt) => {
+      const items = alt.Items.filter((item) => item.Type === 'pronunciation' && 'Confidence' in item);
+      const confidence = items.reduce((acc, item) => acc + item.Confidence, 0) / items.length;
+      return {
+        transcript: alt.Transcript,
+        confidence
+      };
+    });
+    return {
+      language_code: language,
+      channel_tag: channel,
+      is_final: evt.Transcript?.Results[0].IsPartial === false,
+      alternatives,
+      vendor: {
+        name: 'aws',
+        evt: copy
+      }
+    };
+  }
 };
 
 const normalizeAssemblyAi = (evt, channel, language) => {
@@ -466,12 +505,37 @@ const normalizeAssemblyAi = (evt, channel, language) => {
       }
     ],
     vendor: {
-      name: 'ASSEMBLYAI',
+      name: 'assemblyai',
       evt: copy
     }
   };
 };
 
+const normalizeSpeechmatics = (evt, channel, language) => {
+  const copy = JSON.parse(JSON.stringify(evt));
+  const is_final = evt.message === 'AddTranscript';
+  const words = evt.results?.filter((r) => r.type === 'word') || [];
+  const confidence = words.length > 0 ?
+    words.reduce((acc, word) => acc + word.alternatives[0].confidence, 0) / words.length :
+    0;
+
+  const alternative = {
+    confidence,
+    transcript: evt.metadata?.transcript
+  };
+  const obj = {
+    language_code: language,
+    channel_tag: channel,
+    is_final,
+    alternatives: [alternative],
+    vendor: {
+      name: 'speechmatics',
+      evt: copy
+    }
+  };
+  return obj;
+};
+
 module.exports = (logger) => {
   const normalizeTranscription = (evt, vendor, channel, language, shortUtterance, punctuation) => {
 
@@ -499,6 +563,8 @@ module.exports = (logger) => {
         return normalizeAssemblyAi(evt, channel, language, shortUtterance);
       case 'verbio':
         return normalizeVerbio(evt, channel, language);
+      case 'speechmatics':
+        return normalizeSpeechmatics(evt, channel, language);
       default:
         if (vendor.startsWith('custom:')) {
           return normalizeCustom(evt, channel, language, vendor);
@@ -572,17 +638,29 @@ module.exports = (logger) => {
       };
     }
     else if (['aws', 'polly'].includes(vendor)) {
+      const {awsOptions = {}} = rOpts;
+      const vocabularyName = awsOptions.vocabularyName || rOpts.vocabularyName;
+      const vocabularyFilterName = awsOptions.vocabularyFilterName || rOpts.vocabularyFilterName;
+      const filterMethod = awsOptions.vocabularyFilterMethod || rOpts.filterMethod;
       opts = {
         ...opts,
-        ...(rOpts.vocabularyName && {AWS_VOCABULARY_NAME: rOpts.vocabularyName}),
-        ...(rOpts.vocabularyFilterName && {AWS_VOCABULARY_FILTER_NAME: rOpts.vocabularyFilterName}),
-        ...(rOpts.filterMethod && {AWS_VOCABULARY_FILTER_METHOD: rOpts.filterMethod}),
+        ...(vocabularyName && {AWS_VOCABULARY_NAME: vocabularyName}),
+        ...(vocabularyFilterName && {AWS_VOCABULARY_FILTER_NAME: vocabularyFilterName}),
+        ...(filterMethod && {AWS_VOCABULARY_FILTER_METHOD: filterMethod}),
         ...(sttCredentials && {
-          ...(sttCredentials.accessKeyId && {AWS_ACCESS_KEY_ID: sttCredentials.accessKeyId}),
-          ...(sttCredentials.secretAccessKey && {AWS_SECRET_ACCESS_KEY: sttCredentials.secretAccessKey}),
+          AWS_ACCESS_KEY_ID: sttCredentials.accessKeyId,
+          AWS_SECRET_ACCESS_KEY: sttCredentials.secretAccessKey,
           AWS_REGION: sttCredentials.region,
-          ...(sttCredentials.sessionToken && {AWS_SESSION_TOKEN: sttCredentials.sessionToken}),
+          AWS_SECURITY_TOKEN: sttCredentials.securityToken
         }),
+        ...(awsOptions.accessKey && {AWS_ACCESS_KEY_ID: awsOptions.accessKey}),
+        ...(awsOptions.secretKey && {AWS_SECRET_ACCESS_KEY: awsOptions.secretKey}),
+        ...(awsOptions.region && {AWS_REGION: awsOptions.region}),
+        ...(awsOptions.securityToken && {AWS_SECURITY_TOKEN: awsOptions.securityToken}),
+        ...(awsOptions.languageModelName && {AWS_LANGUAGE_MODEL_NAME: awsOptions.languageModelName}),
+        ...(awsOptions.piiEntityTypes?.length && {AWS_PII_ENTITY_TYPES: awsOptions.piiEntityTypes.join(',')}),
+        ...(awsOptions.piiIdentifyEntities && {AWS_PII_IDENTIFY_ENTITIES: true}),
+        ...(awsOptions.languageModelName && {AWS_LANGUAGE_MODEL_NAME: awsOptions.languageModelName}),
       };
     }
     else if ('microsoft' === vendor) {
@@ -828,7 +906,8 @@ module.exports = (logger) => {
         ...(cobaltOptions.enableConfusionNetwork && {COBALT_ENABLE_CONFUSION_NETWORK: 1}),
         ...(cobaltOptions.compiledContextData && {COBALT_COMPILED_CONTEXT_DATA: cobaltOptions.compiledContextData}),
       };
-    } else if ('assemblyai' === vendor) {
+    }
+    else if ('assemblyai' === vendor) {
       opts = {
         ...opts,
         ...(sttCredentials.api_key) &&
@@ -836,7 +915,8 @@ module.exports = (logger) => {
         ...(rOpts.hints?.length > 0 &&
           {ASSEMBLYAI_WORD_BOOST: JSON.stringify(rOpts.hints)})
       };
-    } else if ('verbio' === vendor) {
+    }
+    else if ('verbio' === vendor) {
       const {verbioOptions = {}} = rOpts;
       opts = {
         ...opts,
@@ -855,7 +935,16 @@ module.exports = (logger) => {
         ...(verbioOptions.speech_incomplete_timeout &&
           {VERBIO_SPEECH_INCOMPLETE_TIMEOUT: verbioOptions.speech_incomplete_timeout}),
       };
-    } else if (vendor.startsWith('custom:')) {
+    }
+    else if ('speechmatics' === vendor) {
+      opts = {
+        ...opts,
+        ...(sttCredentials.api_key) && {SPEECHMATICS_API_KEY: sttCredentials.api_key},
+        ...(sttCredentials.speechmatics_stt_uri) && {SPEECHMATICS_HOST: sttCredentials.speechmatics_stt_uri},
+        ...(rOpts.hints?.length > 0 && {SPEECHMATICS_SPEECH_HINTS: rOpts.hints.join(',')}),
+      };
+    }
+    else if (vendor.startsWith('custom:')) {
       let {options = {}} = rOpts.customOptions || {};
       const {sampleRate} = rOpts.customOptions || {};
       const {auth_token, custom_stt_url} = sttCredentials;
@@ -923,6 +1012,6 @@ module.exports = (logger) => {
     setChannelVarsForStt,
     setSpeechCredentialsAtRuntime,
     compileSonioxTranscripts,
-    consolidateTranscripts
+    consolidateTranscripts,
   };
 };

lib/utils/ws-requestor.js

@@ -44,7 +44,7 @@ class WsRequestor extends BaseRequestor {
   async request(type, hook, params, httpHeaders = {}) {
     assert(HookMsgTypes.includes(type));
     const url = hook.url || hook;
-    const wantsAck = !['call:status', 'verb:status', 'jambonz:error'].includes(type);
+    const wantsAck = !['call:status', 'verb:status', 'jambonz:error', 'llm:event', 'llm:tool-call'].includes(type);
 
     if (this.maliciousClient) {
       this.logger.info({url: this.url}, 'WsRequestor:request - discarding msg to malicious client');
@@ -132,7 +132,7 @@ class WsRequestor extends BaseRequestor {
       type,
       msgid,
       call_sid: this.call_sid,
-      hook: ['verb:hook', 'session:redirect'].includes(type) ? url : undefined,
+      hook: ['verb:hook', 'session:redirect', 'llm:event', 'llm:tool-call'].includes(type) ? url : undefined,
       data: {...payload},
       ...b3
     };
@@ -392,8 +392,9 @@ class WsRequestor extends BaseRequestor {
     /* messages must be JSON format */
     try {
       const obj = JSON.parse(content);
+      this.logger.debug({obj}, 'WsRequestor:_onMessage - received message');
       //const {type, msgid, command, call_sid = this.call_sid, queueCommand = false, data} = obj;
-      const {type, msgid, command, queueCommand = false, data} = obj;
+      const {type, msgid, command, queueCommand = false, tool_call_id, data} = obj;
       const call_sid = obj.callSid || this.call_sid;
 
       //this.logger.debug({obj}, 'WsRequestor:request websocket: received');
@@ -407,8 +408,8 @@ class WsRequestor extends BaseRequestor {
 
         case 'command':
           assert.ok(command, 'command property not supplied');
-          assert.ok(data, 'data property not supplied');
-          this._recvCommand(msgid, command, call_sid, queueCommand, data);
+          assert.ok(data || command === 'llm:tool-output', 'data property not supplied');
+          this._recvCommand(msgid, command, call_sid, queueCommand, tool_call_id, data);
           break;
 
         default:
@@ -432,10 +433,10 @@ class WsRequestor extends BaseRequestor {
     success && success(data);
   }
 
-  _recvCommand(msgid, command, call_sid, queueCommand, data) {
+  _recvCommand(msgid, command, call_sid, queueCommand, tool_call_id, data) {
     // TODO: validate command
     this.logger.debug({msgid, command, call_sid, queueCommand, data}, 'received command');
-    this.emit('command', {msgid, command, call_sid, queueCommand, data});
+    this.emit('command', {msgid, command, call_sid, queueCommand, tool_call_id, data});
   }
 }
 

package.json

@@ -31,10 +31,10 @@
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/mw-registrar": "^0.2.7",
     "@jambonz/realtimedb-helpers": "^0.8.8",
-    "@jambonz/speech-utils": "^0.1.16",
+    "@jambonz/speech-utils": "^0.1.20",
     "@jambonz/stats-collector": "^0.1.10",
     "@jambonz/time-series": "^0.2.9",
-    "@jambonz/verb-specifications": "^0.0.81",
+    "@jambonz/verb-specifications": "^0.0.83",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/exporter-jaeger": "^1.23.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
@@ -51,7 +51,6 @@
     "drachtio-srf": "^4.5.35",
     "express": "^4.19.2",
     "express-validator": "^7.0.1",
-    "ip": "^2.0.1",
     "moment": "^2.30.1",
     "parse-url": "^9.2.0",
     "pino": "^8.20.0",
@@ -61,7 +60,7 @@
     "short-uuid": "^5.1.0",
     "sinon": "^17.0.1",
     "to-snake-case": "^1.0.0",
-    "undici": "^6.19.4",
+    "undici": "^6.20.0",
     "uuid-random": "^1.3.2",
     "verify-aws-sns-signature": "^0.1.0",
     "ws": "^8.18.0",

test/db/jambones-sql.sql

@@ -1,4 +1,5 @@
 /* SQLEditor (MySQL (2))*/
+
 SET FOREIGN_KEY_CHECKS=0;
 
 DROP TABLE IF EXISTS account_static_ips;
@@ -53,6 +54,8 @@ DROP TABLE IF EXISTS signup_history;
 
 DROP TABLE IF EXISTS smpp_addresses;
 
+DROP TABLE IF EXISTS google_custom_voices;
+
 DROP TABLE IF EXISTS speech_credentials;
 
 DROP TABLE IF EXISTS system_information;
@@ -136,6 +139,9 @@ account_sid CHAR(36) NOT NULL,
 is_active BOOLEAN NOT NULL DEFAULT 1,
 username VARCHAR(64),
 password VARCHAR(1024),
+allow_direct_app_calling BOOLEAN NOT NULL DEFAULT 1,
+allow_direct_queue_calling BOOLEAN NOT NULL DEFAULT 1,
+allow_direct_user_calling BOOLEAN NOT NULL DEFAULT 1,
 PRIMARY KEY (client_sid)
 );
 
@@ -338,11 +344,23 @@ label VARCHAR(64),
 PRIMARY KEY (speech_credential_sid)
 );
 
+CREATE TABLE google_custom_voices
+(
+google_custom_voice_sid CHAR(36) NOT NULL UNIQUE ,
+speech_credential_sid CHAR(36) NOT NULL,
+model VARCHAR(512) NOT NULL,
+reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
+name VARCHAR(64) NOT NULL,
+PRIMARY KEY (google_custom_voice_sid)
+);
+
 CREATE TABLE system_information
 (
 domain_name VARCHAR(255),
 sip_domain_name VARCHAR(255),
-monitoring_domain_name VARCHAR(255)
+monitoring_domain_name VARCHAR(255),
+private_network_cidr VARCHAR(8192),
+log_level ENUM('info', 'debug') NOT NULL DEFAULT 'info'
 );
 
 CREATE TABLE users
@@ -437,11 +455,14 @@ CREATE TABLE sip_gateways
 sip_gateway_sid CHAR(36),
 ipv4 VARCHAR(128) NOT NULL COMMENT 'ip address or DNS name of the gateway.  For gateways providing inbound calling service, ip address is required.',
 netmask INTEGER NOT NULL DEFAULT 32,
-port INTEGER NOT NULL DEFAULT 5060 COMMENT 'sip signaling port',
+port INTEGER COMMENT 'sip signaling port',
 inbound BOOLEAN NOT NULL COMMENT 'if true, whitelist this IP to allow inbound calls from the gateway',
 outbound BOOLEAN NOT NULL COMMENT 'if true, include in least-cost routing when placing calls to the PSTN',
 voip_carrier_sid CHAR(36) NOT NULL,
 is_active BOOLEAN NOT NULL DEFAULT 1,
+send_options_ping BOOLEAN NOT NULL DEFAULT 0,
+use_sips_scheme BOOLEAN NOT NULL DEFAULT 0,
+pad_crypto BOOLEAN NOT NULL DEFAULT 0,
 protocol ENUM('udp','tcp','tls', 'tls/srtp') DEFAULT 'udp' COMMENT 'Outbound call protocol',
 PRIMARY KEY (sip_gateway_sid)
 ) COMMENT='A whitelisted sip gateway used for origination/termination';
@@ -478,11 +499,19 @@ messaging_hook_sid CHAR(36) COMMENT 'webhook to call for inbound SMS/MMS ',
 app_json TEXT,
 speech_synthesis_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_synthesis_language VARCHAR(12) NOT NULL DEFAULT 'en-US',
-speech_synthesis_voice VARCHAR(64),
+speech_synthesis_voice VARCHAR(256),
 speech_synthesis_label VARCHAR(64),
 speech_recognizer_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_recognizer_language VARCHAR(64) NOT NULL DEFAULT 'en-US',
 speech_recognizer_label VARCHAR(64),
+use_for_fallback_speech BOOLEAN DEFAULT false,
+fallback_speech_synthesis_vendor VARCHAR(64),
+fallback_speech_synthesis_language VARCHAR(12),
+fallback_speech_synthesis_voice VARCHAR(256),
+fallback_speech_synthesis_label VARCHAR(64),
+fallback_speech_recognizer_vendor VARCHAR(64),
+fallback_speech_recognizer_language VARCHAR(64),
+fallback_speech_recognizer_label VARCHAR(64),
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
 record_all_calls BOOLEAN NOT NULL DEFAULT false,
 PRIMARY KEY (application_sid)
@@ -525,6 +554,7 @@ siprec_hook_sid CHAR(36),
 record_all_calls BOOLEAN NOT NULL DEFAULT false,
 record_format VARCHAR(16) NOT NULL DEFAULT 'mp3',
 bucket_credential VARCHAR(8192) COMMENT 'credential used to authenticate with storage service',
+enable_debug_log BOOLEAN NOT NULL DEFAULT false,
 PRIMARY KEY (account_sid)
 ) COMMENT='An enterprise that uses the platform for comm services';
 
@@ -619,6 +649,10 @@ ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (ser
 CREATE INDEX account_sid_idx ON speech_credentials (account_sid);
 ALTER TABLE speech_credentials ADD FOREIGN KEY account_sid_idxfk_8 (account_sid) REFERENCES accounts (account_sid);
 
+CREATE INDEX google_custom_voice_sid_idx ON google_custom_voices (google_custom_voice_sid);
+CREATE INDEX speech_credential_sid_idx ON google_custom_voices (speech_credential_sid);
+ALTER TABLE google_custom_voices ADD FOREIGN KEY speech_credential_sid_idxfk (speech_credential_sid) REFERENCES speech_credentials (speech_credential_sid) ON DELETE CASCADE;
+
 CREATE INDEX user_sid_idx ON users (user_sid);
 CREATE INDEX email_idx ON users (email);
 CREATE INDEX phone_idx ON users (phone);
@@ -704,4 +738,5 @@ ALTER TABLE accounts ADD FOREIGN KEY queue_event_hook_sid_idxfk (queue_event_hoo
 ALTER TABLE accounts ADD FOREIGN KEY device_calling_application_sid_idxfk (device_calling_application_sid) REFERENCES applications (application_sid);
 
 ALTER TABLE accounts ADD FOREIGN KEY siprec_hook_sid_idxfk (siprec_hook_sid) REFERENCES applications (application_sid);
-SET FOREIGN_KEY_CHECKS=1;
+
+SET FOREIGN_KEY_CHECKS=1;