google gemini set default model to models/gemini-2.0-flash-live-001

* fixed google s2s mcp initiate wrong functionDeclarations

* populate model from llm verb to google setup message

.gitignore

@@ -42,4 +42,5 @@ ecosystem.config.js
 test/credentials/*.json
 run-tests.sh
 run-coverage.sh
-.vscode
+.vscode
+.env

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
+FROM --platform=linux/amd64 node:20-alpine as base
 
 RUN apk --update --no-cache add --virtual .builds-deps build-base python3
 

README.md

@@ -13,7 +13,7 @@ Configuration is provided via environment variables:
 |AWS_ACCESS_KEY_ID| aws access key id, used for TTS/STT as well SNS notifications|no|
 |AWS_REGION| aws region| no|
 |AWS_SECRET_ACCESS_KEY| aws secret access key, used per above|no|
-|AWS_SNS_TOPIC_ARM| aws sns topic arn that scale-in lifecycle notifications will be published to|no|
+|AWS_SNS_TOPIC_ARN| aws sns topic arn that scale-in lifecycle notifications will be published to|no|
 |DRACHTIO_HOST| ip address of drachtio server (typically '127.0.0.1')|yes|
 |DRACHTIO_PORT| listening port of drachtio server for control connections (typically 9022)|yes|
 |DRACHTIO_SECRET| shared secret|yes|
@@ -72,7 +72,7 @@ module.exports = {
       STATS_PORT: 8125,
       STATS_PROTOCOL: 'tcp',
       STATS_TELEGRAF: 1,
-      AWS_SNS_TOPIC_ARM: 'arn:aws:sns:us-west-1:xxxxxxxxxxx:terraform-20201107200347128600000002',
+      AWS_SNS_TOPIC_ARN: 'arn:aws:sns:us-west-1:xxxxxxxxxxx:terraform-20201107200347128600000002',
       JAMBONES_NETWORK_CIDR: '172.31.0.0/16',
       JAMBONES_MYSQL_HOST: 'aurora-cluster-jambonz.cluster-yyyyyyyyyyy.us-west-1.rds.amazonaws.com',
       JAMBONES_MYSQL_USER: 'admin',

app.js

@@ -25,9 +25,74 @@ const opts = {
 };
 const pino = require('pino');
 const logger = pino(opts, pino.destination({sync: false}));
-const {LifeCycleEvents, FS_UUID_SET_NAME} = require('./lib/utils/constants');
+const {LifeCycleEvents, FS_UUID_SET_NAME, SystemState, FEATURE_SERVER} = require('./lib/utils/constants');
 const installSrfLocals = require('./lib/utils/install-srf-locals');
-installSrfLocals(srf, logger);
+const createHttpListener = require('./lib/utils/http-listener');
+const healthCheck = require('@jambonz/http-health-check');
+
+logger.on('level-change', (lvl, _val, prevLvl, _prevVal, instance) => {
+  if (logger !== instance) {
+    return;
+  }
+  logger.info('system log level %s was changed to %s', prevLvl, lvl);
+});
+
+// Install the srf locals
+installSrfLocals(srf, logger, {
+  onFreeswitchConnect: (wraper) => {
+    // Only connect to drachtio if freeswitch is connected
+    logger.info(`connected to freeswitch at ${wraper.ms.address}, start drachtio server`);
+    if (DRACHTIO_HOST) {
+      srf.connect({host: DRACHTIO_HOST, port: DRACHTIO_PORT, secret: DRACHTIO_SECRET });
+      srf.on('connect', (err, hp) => {
+        const arr = /^(.*)\/(.*)$/.exec(hp.split(',').pop());
+        srf.locals.localSipAddress = `${arr[2]}`;
+        logger.info(`connected to drachtio listening on ${hp}, local sip address is ${srf.locals.localSipAddress}`);
+      });
+    }
+    else {
+      logger.info(`listening for drachtio requests on port ${DRACHTIO_PORT}`);
+      srf.listen({port: DRACHTIO_PORT, secret: DRACHTIO_SECRET});
+    }
+    // Start Http server
+    createHttpListener(logger, srf)
+      .then(({server, app}) => {
+        httpServer = server;
+        healthCheck({app, logger, path: '/', fn: getCount});
+        return {server, app};
+      })
+      .catch((err) => {
+        logger.error(err, 'Error creating http listener');
+      });
+  },
+  onFreeswitchDisconnect: (wraper) => {
+    // check if all freeswitch connections are lost, disconnect drachtio server
+    logger.info(`lost connection to freeswitch at ${wraper.ms.address}`);
+    const ms = srf.locals.getFreeswitch();
+    if (!ms) {
+      logger.info('no freeswitch connections, stopping drachtio server');
+      disconnect();
+    }
+  }
+});
+if (NODE_ENV === 'test') {
+  srf.on('error', (err) => {
+    logger.info(err, 'Error connecting to drachtio');
+  });
+}
+
+// Init services
+const writeSystemAlerts = srf.locals?.writeSystemAlerts;
+if (writeSystemAlerts) {
+  writeSystemAlerts({
+    system_component: FEATURE_SERVER,
+    state : SystemState.Online,
+    fields : {
+      detail: `feature-server with process_id ${process.pid} started`,
+      host: srf.locals?.ipv4
+    }
+  });
+}
 
 const {
   initLocals,
@@ -42,24 +107,6 @@ const {
 const InboundCallSession = require('./lib/session/inbound-call-session');
 const SipRecCallSession = require('./lib/session/siprec-call-session');
 
-if (DRACHTIO_HOST) {
-  srf.connect({host: DRACHTIO_HOST, port: DRACHTIO_PORT, secret: DRACHTIO_SECRET });
-  srf.on('connect', (err, hp) => {
-    const arr = /^(.*)\/(.*)$/.exec(hp.split(',').pop());
-    srf.locals.localSipAddress = `${arr[2]}`;
-    logger.info(`connected to drachtio listening on ${hp}, local sip address is ${srf.locals.localSipAddress}`);
-  });
-}
-else {
-  logger.info(`listening for drachtio requests on port ${DRACHTIO_PORT}`);
-  srf.listen({port: DRACHTIO_PORT, secret: DRACHTIO_SECRET});
-}
-if (NODE_ENV === 'test') {
-  srf.on('error', (err) => {
-    logger.info(err, 'Error connecting to drachtio');
-  });
-}
-
 srf.use('invite', [
   initLocals,
   createRootSpan,
@@ -85,27 +132,27 @@ sessionTracker.on('idle', () => {
   }
 });
 const getCount = () => sessionTracker.count;
-const healthCheck = require('@jambonz/http-health-check');
+
 let httpServer;
 
-const createHttpListener = require('./lib/utils/http-listener');
-createHttpListener(logger, srf)
-  .then(({server, app}) => {
-    httpServer = server;
-    healthCheck({app, logger, path: '/', fn: getCount});
-    return {server, app};
-  })
-  .catch((err) => {
-    logger.error(err, 'Error creating http listener');
-  });
-
-
-setInterval(async() => {
+const monInterval = setInterval(async() => {
   srf.locals.stats.gauge('fs.sip.calls.count', sessionTracker.count);
-  // Checking system log level
-  const systemInformation = await srf.locals.dbHelpers.lookupSystemInformation();
-  if (systemInformation && systemInformation.log_level) {
-    logger.level = systemInformation.log_level;
+  try {
+    const systemInformation = await srf.locals.dbHelpers.lookupSystemInformation();
+    if (systemInformation && systemInformation.log_level) {
+      const envLogLevel = logger.levels.values[JAMBONES_LOGLEVEL.toLowerCase()];
+      const dbLogLevel = logger.levels.values[systemInformation.log_level];
+      const appliedLogLevel = Math.min(envLogLevel, dbLogLevel);
+      if (logger.levelVal !== appliedLogLevel) {
+        logger.level = logger.levels.labels[Math.min(envLogLevel, dbLogLevel)];
+      }
+    }
+  } catch (err) {
+    if (process.env.NODE_ENV === 'test') {
+      clearInterval(monInterval);
+      logger.error('all tests complete');
+    }
+    else logger.error({err}, 'Error checking system log level in database');
   }
 }, 20000);
 
@@ -114,16 +161,29 @@ const disconnect = () => {
     httpServer?.on('close', resolve);
     httpServer?.close();
     srf.disconnect();
+    srf.removeAllListeners();
     srf.locals.mediaservers?.forEach((ms) => ms.disconnect());
   });
 };
-
 process.on('SIGTERM', handle);
+process.on('SIGINT', handle);
 
-function handle(signal) {
+async function handle(signal) {
   const {removeFromSet} = srf.locals.dbHelpers;
   srf.locals.disabled = true;
   logger.info(`got signal ${signal}`);
+  const writeSystemAlerts = srf.locals?.writeSystemAlerts;
+  if (writeSystemAlerts) {
+    // it has to be synchronous call, or else by the time system saves the app terminates
+    await writeSystemAlerts({
+      system_component: FEATURE_SERVER,
+      state : SystemState.Offline,
+      fields : {
+        detail: `feature-server with process_id ${process.pid} stopped, signal ${signal}`,
+        host: srf.locals?.ipv4
+      }
+    });
+  }
   const setName = `${(JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
   const fsServiceUrlSetName = `${(JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
   if (setName && srf.locals.localSipAddress) {

lib/config.js

@@ -93,7 +93,7 @@ const AWS_REGION = process.env.AWS_REGION;
 const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
 const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;
 const AWS_SNS_PORT = parseInt(process.env.AWS_SNS_PORT, 10) || 3001;
-const AWS_SNS_TOPIC_ARM = process.env.AWS_SNS_TOPIC_ARM;
+const AWS_SNS_TOPIC_ARN = process.env.AWS_SNS_TOPIC_ARN;
 const AWS_SNS_PORT_MAX = parseInt(process.env.AWS_SNS_PORT_MAX, 10) || 3005;
 
 const GCP_JSON_KEY = process.env.GCP_JSON_KEY;
@@ -136,6 +136,12 @@ const JAMBONES_DISABLE_DIRECT_P2P_CALL = process.env.JAMBONES_DISABLE_DIRECT_P2P
 const JAMBONES_EAGERLY_PRE_CACHE_AUDIO = parseInt(process.env.JAMBONES_EAGERLY_PRE_CACHE_AUDIO, 10) || 0;
 
 const JAMBONES_USE_FREESWITCH_TIMER_FD = process.env.JAMBONES_USE_FREESWITCH_TIMER_FD;
+const JAMBONES_DIAL_SBC_FOR_REGISTERED_USER = process.env.JAMBONES_DIAL_SBC_FOR_REGISTERED_USER || false;
+const JAMBONES_MEDIA_TIMEOUT_MS = process.env.JAMBONES_MEDIA_TIMEOUT_MS || 0;
+const JAMBONES_MEDIA_HOLD_TIMEOUT_MS = process.env.JAMBONES_MEDIA_HOLD_TIMEOUT_MS || 0;
+// jambonz
+const JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS =
+  process.env.JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS;
 
 module.exports = {
   JAMBONES_MYSQL_HOST,
@@ -186,7 +192,7 @@ module.exports = {
   AWS_ACCESS_KEY_ID,
   AWS_SECRET_ACCESS_KEY,
   AWS_SNS_PORT,
-  AWS_SNS_TOPIC_ARM,
+  AWS_SNS_TOPIC_ARN,
   AWS_SNS_PORT_MAX,
 
   ANCHOR_MEDIA_ALWAYS,
@@ -221,5 +227,9 @@ module.exports = {
   JAMBONZ_RECORD_WS_PASSWORD,
   JAMBONZ_DISABLE_DIAL_PAI_HEADER,
   JAMBONES_DISABLE_DIRECT_P2P_CALL,
-  JAMBONES_USE_FREESWITCH_TIMER_FD
+  JAMBONES_USE_FREESWITCH_TIMER_FD,
+  JAMBONES_DIAL_SBC_FOR_REGISTERED_USER,
+  JAMBONES_MEDIA_TIMEOUT_MS,
+  JAMBONES_MEDIA_HOLD_TIMEOUT_MS,
+  JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS
 };

lib/http-routes/api/create-call.js

@@ -3,7 +3,7 @@ const makeTask = require('../../tasks/make_task');
 const RestCallSession = require('../../session/rest-call-session');
 const CallInfo = require('../../session/call-info');
 const {CallDirection, CallStatus} = require('../../utils/constants');
-const uuidv4 = require('uuid-random');
+const crypto = require('crypto');
 const SipError = require('drachtio-srf').SipError;
 const { validationResult, body } = require('express-validator');
 const { validate } = require('@jambonz/verb-specifications');
@@ -14,6 +14,8 @@ const RootSpan = require('../../utils/call-tracer');
 const dbUtils = require('../../utils/db-utils');
 const { mergeSdpMedia, extractSdpMedia } = require('../../utils/sdp-utils');
 const { createCallSchema, customSanitizeFunction } = require('../schemas/create-call');
+const { selectHostPort } = require('../../utils/network');
+const { JAMBONES_DIAL_SBC_FOR_REGISTERED_USER } = require('../../config');
 
 const removeNullProperties = (obj) => (Object.keys(obj).forEach((key) => obj[key] === null && delete obj[key]), obj);
 const removeNulls = (req, res, next) => {
@@ -65,7 +67,7 @@ router.post('/',
         lookupAppBySid
       }  = srf.locals.dbHelpers;
       const {getSBC, getFreeswitch} = srf.locals;
-      const sbcAddress = getSBC();
+      let sbcAddress = getSBC();
       if (!sbcAddress) throw new Error('no available SBCs for outbound call creation');
       const target = restDial.to;
       const opts = {
@@ -78,7 +80,7 @@ router.post('/',
       const {lookupTeamsByAccount, lookupAccountBySid} = srf.locals.dbHelpers;
       const account = await lookupAccountBySid(req.body.account_sid);
       const accountInfo = await lookupAccountDetails(req.body.account_sid);
-      const callSid = uuidv4();
+      const callSid = crypto.randomUUID();
       const application = req.body.application_sid ? await lookupAppBySid(req.body.application_sid) : null;
       const record_all_calls = account.record_all_calls || (application && application.record_all_calls);
       const recordOutputFormat = account.record_format || 'mp3';
@@ -98,6 +100,7 @@ router.post('/',
         ...(req.body?.application_sid && {'X-Application-Sid': req.body.application_sid}),
         ...(restDial.fromHost && {'X-Preferred-From-Host': restDial.fromHost}),
         ...(record_all_calls && {'X-Record-All-Calls': recordOutputFormat}),
+        ...(target.proxy && {'X-SIP-Proxy': target.proxy}),
         ...target.headers
       };
 
@@ -140,6 +143,16 @@ router.post('/',
         }
       }
 
+      // find handling sbc sip for called user
+      if (JAMBONES_DIAL_SBC_FOR_REGISTERED_USER && target.type === 'user') {
+        const { registrar }  = srf.locals.dbHelpers;
+        const reg = await registrar.query(target.name);
+        if (reg) {
+          sbcAddress = selectHostPort(logger, reg.sbcAddress, 'tcp')[1];
+        }
+        //sbc outbound return 404 Notfound to handle case called user is not reigstered.
+      }
+
       /**
      * trunk isn't specified,
      * check if from-number matches any existing numbers on Jambonz
@@ -196,10 +209,13 @@ router.post('/',
 
       /**
      * create our application object -
-     * not from the database as per an inbound call,
-     * but from the provided params in the request
+     * we merge the inbound call application,
+     * with the provided app params from the request body
      */
-      const app = req.body;
+      const app = {
+        ...application,
+        ...req.body
+      };
 
       /**
      * attach our requestor and notifier objects

lib/middleware.js

@@ -1,5 +1,5 @@
-const uuidv4 = require('uuid-random');
-const {CallDirection, AllowedSipRecVerbs} = require('./utils/constants');
+const crypto = require('crypto');
+const {CallDirection, AllowedSipRecVerbs, WS_CLOSE_CODES} = require('./utils/constants');
 const {parseSiprecPayload} = require('./utils/siprec-utils');
 const CallInfo = require('./session/call-info');
 const HttpRequestor = require('./utils/http-requestor');
@@ -15,6 +15,7 @@ const {
   JAMBONES_DISABLE_DIRECT_P2P_CALL
 } = require('./config');
 const { createJambonzApp } = require('./dynamic-apps');
+const { decrypt } = require('./utils/encrypt-decrypt');
 
 module.exports = function(srf, logger) {
   const {
@@ -45,7 +46,7 @@ module.exports = function(srf, logger) {
       logger.info('getAccountDetails - rejecting call due to missing X-Account-Sid header');
       return res.send(500);
     }
-    const callSid = req.has('X-Retain-Call-Sid') ? req.get('X-Retain-Call-Sid') : uuidv4();
+    const callSid = req.has('X-Retain-Call-Sid') ? req.get('X-Retain-Call-Sid') : crypto.randomUUID();
     const account_sid = req.get('X-Account-Sid');
     req.locals = {callSid, account_sid, callId};
 
@@ -336,7 +337,9 @@ module.exports = function(srf, logger) {
         if (arr) {
           const google_custom_voice_sid = arr[1];
           const [custom_voice] = await lookupGoogleCustomVoice(google_custom_voice_sid);
-          if (custom_voice) {
+          //google voice cloning key has size 200kb, jambonz should not resolve the voice here that the app's calling
+          //webhook will receive big payload, tts-task should resolve the voice later.
+          if (!custom_voice.use_voice_cloning_key) {
             app2.speech_synthesis_voice = {
               reportedUsage: custom_voice.reported_usage,
               model: custom_voice.model
@@ -346,11 +349,10 @@ module.exports = function(srf, logger) {
       }
 
       req.locals.application = app2;
-
       // eslint-disable-next-line no-unused-vars
       const {call_hook, call_status_hook, ...appInfo} = app;  // mask sensitive data like user/pass on webhook
       // eslint-disable-next-line no-unused-vars
-      const {requestor, notifier, ...loggable} = appInfo;
+      const {requestor, notifier, env_vars, ...loggable} = appInfo;
       logger.info({app: loggable}, `retrieved application for incoming call to ${req.locals.calledNumber}`);
       req.locals.callInfo = new CallInfo({
         req,
@@ -415,10 +417,22 @@ module.exports = function(srf, logger) {
             ...(app.fallback_speech_recognizer_language && {fallback_language: app.fallback_speech_recognizer_language})
           }
         };
+        let env_vars;
+        try {
+          if (app.env_vars) {
+            const d_env_vars = JSON.parse(decrypt(app.env_vars));
+            logger.info(`Setting env_vars: ${Object.keys(d_env_vars)}`); // Only log the keys not the values
+            env_vars = d_env_vars;
+          }
+        } catch (error) {
+          logger.info('Unable to set env_vars', error);
+        }
         const params = Object.assign(['POST', 'WS'].includes(app.call_hook.method) ? { sip: req.msg } : {},
           req.locals.callInfo,
           { service_provider_sid: req.locals.service_provider_sid },
-          { defaults });
+          { defaults },
+          { env_vars }
+        );
         logger.debug({ params }, 'sending initial webhook');
         const obj = rootSpan.startChildSpan('performAppWebhook');
         span = obj.span;
@@ -458,7 +472,7 @@ module.exports = function(srf, logger) {
       }).catch((err) => this.logger.info({err}, 'Error generating alert for parsing application'));
       logger.info({err}, `Error retrieving or parsing application: ${err?.message}`);
       res.send(480, {headers: {'X-Reason': err?.message || 'unknown'}});
-      app.requestor.close();
+      app.requestor.close(WS_CLOSE_CODES.GoingAway);
     }
   }
 

lib/session/adulting-call-session.js

@@ -45,8 +45,10 @@ class AdultingCallSession extends CallSession {
     return this.sd.ep;
   }
 
-  /* see note above */
-  set ep(newEp) {}
+  // When adulting session kicked from conference, replaceEndpoint is a must
+  set ep(newEp) {
+    this.sd.ep = newEp;
+  }
 
   get callSid() {
     return this.callInfo.callSid;

lib/session/call-info.js

@@ -1,6 +1,6 @@
 const {CallDirection, CallStatus} = require('../utils/constants');
 const parseUri = require('drachtio-srf').parseUri;
-const uuidv4 = require('uuid-random');
+const crypto = require('crypto');
 const {JAMBONES_API_BASE_URL} = require('../config');
 /**
  * @classdesc Represents the common information for all calls
@@ -57,7 +57,7 @@ class CallInfo {
       // outbound call that is a child of an existing call
       const {req, parentCallInfo, to, callSid} = opts;
       srf = req.srf;
-      this.callSid = callSid || uuidv4();
+      this.callSid = callSid || crypto.randomUUID();
       this.parentCallSid = parentCallInfo.callSid;
       this.accountSid = parentCallInfo.accountSid;
       this.applicationSid = parentCallInfo.applicationSid;

lib/session/confirm-call-session.js

@@ -8,7 +8,7 @@ const CallSession = require('./call-session');
 
  */
 class ConfirmCallSession extends CallSession {
-  constructor({logger, application, dlg, ep, tasks, callInfo, accountInfo, memberId, confName, rootSpan}) {
+  constructor({logger, application, dlg, ep, tasks, callInfo, accountInfo, memberId, confName, rootSpan, req}) {
     super({
       logger,
       application,
@@ -23,6 +23,7 @@ class ConfirmCallSession extends CallSession {
     });
     this.dlg = dlg;
     this.ep = ep;
+    this.req = req;
   }
 
   /**

lib/session/inbound-call-session.js

@@ -70,8 +70,12 @@ class InboundCallSession extends CallSession {
     this._hangup('caller');
   }
 
-  _jambonzHangup() {
-    this.dlg?.destroy();
+  _jambonzHangup(reason) {
+    this.dlg?.destroy({
+      headers: {
+        ...(reason && {'X-Reason': reason})
+      }
+    });
     // kill current task or wakeup the call session.
     this._callReleased();
   }

lib/session/rest-call-session.js

@@ -63,7 +63,7 @@ class RestCallSession extends CallSession {
     this.callInfo.callTerminationBy = terminatedBy;
     const duration = moment().diff(this.dlg.connectTime, 'seconds');
     this.emit('callStatusChange', {callStatus: CallStatus.Completed, duration});
-    this.logger.debug(`RestCallSession: called party hung up by ${terminatedBy}`);
+    this.logger.info(`RestCallSession: called party hung up by ${terminatedBy}`);
     this._callReleased();
   }
 

lib/tasks/conference.js

@@ -83,7 +83,11 @@ class Conference extends Task {
     // reset answer time if we were transferred from another feature server
     if (this.connectTime) dlg.connectTime = this.connectTime;
 
-
+    if (cs.sipRequestWithinDialogHook) {
+      /* remove any existing listener to escape from duplicating events */
+      this._removeSipIndialogRequestListener(this.dlg);
+      this._initSipIndialogRequestListener(cs, dlg);
+    }
     this.ep.on('destroy', this._kicked.bind(this, cs, dlg));
 
     try {
@@ -103,6 +107,7 @@ class Conference extends Task {
 
       this.logger.debug(`Conference:exec - conference ${this.confName} is over`);
       if (this.callMoved !== false) await this.performAction(this.results);
+      this._removeSipIndialogRequestListener(dlg);
     } catch (err) {
       this.logger.info(err, `TaskConference:exec - error in conference ${this.confName}`);
     }
@@ -416,6 +421,20 @@ class Conference extends Task {
     }
   }
 
+  _initSipIndialogRequestListener(cs, dlg) {
+    dlg.on('info', this._onRequestWithinDialog.bind(this, cs));
+    dlg.on('message', this._onRequestWithinDialog.bind(this, cs));
+  }
+
+  _removeSipIndialogRequestListener(dlg) {
+    dlg && dlg.removeAllListeners('message');
+    dlg && dlg.removeAllListeners('info');
+  }
+
+  _onRequestWithinDialog(cs, req, res) {
+    cs._onRequestWithinDialog(req, res);
+  }
+
   /**
    * The conference we have been waiting for has started.
    * It may be on this server or a different one, and we are
@@ -446,7 +465,7 @@ class Conference extends Task {
 
   doConferenceMute(cs, opts) {
     assert (cs.isInConference);
-
+    this.logger.info(`Conference:doConferenceMute ${mute ? 'muting' : 'unmuting'} member`);
     const mute = opts.conf_mute_status === 'mute';
     this.ep.api(`conference ${this.confName} ${mute ? 'mute' : 'unmute'} ${this.memberId}`)
       .catch((err) => this.logger.info({err}, 'Error muting or unmuting participant'));
@@ -549,8 +568,8 @@ class Conference extends Task {
   /**
    * mute or unmute side of the call
    */
-  mute(callSid, doMute) {
-    this.doConferenceMute(this.callSession, {conf_mute_status: doMute});
+  async mute(callSid, doMute) {
+    this.doConferenceMute(this.callSession, {conf_mute_status: doMute ? 'mute' : 'unmute'});
   }
 
   /**
@@ -653,7 +672,8 @@ class Conference extends Task {
         memberId: this.memberId,
         confName: this.confName,
         tasks,
-        rootSpan: cs.rootSpan
+        rootSpan: cs.rootSpan,
+        req: cs.req
       });
       await this._playSession.exec();
       this._playSession = null;

lib/tasks/config.js

@@ -16,7 +16,8 @@ class TaskConfig extends Task {
       'fillerNoise',
       'actionHookDelayAction',
       'boostAudioSignal',
-      'vad'
+      'vad',
+      'ttsStream'
     ].forEach((k) => this[k] = this.data[k] || {});
 
     if ('notifyEvents' in this.data) {
@@ -45,6 +46,12 @@ class TaskConfig extends Task {
       };
       delete this.transcribeOpts.enable;
     }
+    if (this.ttsStream.enable) {
+      this.sayOpts = {
+        verb: 'say',
+        stream: true
+      };
+    }
 
     if (this.data.reset) {
       if (typeof this.data.reset === 'string') this.data.reset = [this.data.reset];
@@ -75,6 +82,7 @@ class TaskConfig extends Task {
   get hasVad() { return Object.keys(this.vad).length; }
   get hasFillerNoise() { return Object.keys(this.fillerNoise).length; }
   get hasReferHook() { return Object.keys(this.data).includes('referHook'); }
+  get hasTtsStream() { return Object.keys(this.ttsStream).length; }
 
   get summary() {
     const phrase = [];
@@ -106,6 +114,9 @@ class TaskConfig extends Task {
     if (this.onHoldMusic) phrase.push(`onHoldMusic: ${this.onHoldMusic}`);
     if ('boostAudioSignal' in this.data) phrase.push(`setGain ${this.data.boostAudioSignal}`);
     if (this.hasReferHook) phrase.push('set referHook');
+    if (this.hasTtsStream) {
+      phrase.push(`${this.ttsStream.enable ? 'enable' : 'disable'} ttsStream`);
+    }
     return `${this.name}{${phrase.join(',')}}`;
   }
 
@@ -176,18 +187,20 @@ class TaskConfig extends Task {
         : cs.speechRecognizerVendor;
       cs.speechRecognizerLabel = this.recognizer.label === 'default'
         ? cs.speechRecognizerLabel : this.recognizer.label;
-      cs.speechRecognizerLanguage = this.recognizer.language !== 'default'
+      cs.speechRecognizerLanguage = this.recognizer.language !== undefined && this.recognizer.language !== 'default'
         ? this.recognizer.language
         : cs.speechRecognizerLanguage;
 
       //fallback
-      cs.fallbackSpeechRecognizerVendor = this.recognizer.fallbackVendor !== 'default'
+      cs.fallbackSpeechRecognizerVendor = this.recognizer.fallbackVendor !== undefined &&
+        this.recognizer.fallbackVendor !== 'default'
         ? this.recognizer.fallbackVendor
         : cs.fallbackSpeechRecognizerVendor;
       cs.fallbackSpeechRecognizerLabel = this.recognizer.fallbackLabel === 'default' ?
         cs.fallbackSpeechRecognizerLabel :
         this.recognizer.fallbackLabel;
-      cs.fallbackSpeechRecognizerLanguage = this.recognizer.fallbackLanguage !== 'default'
+      cs.fallbackSpeechRecognizerLanguage = this.recognizer.fallbackLanguage !== undefined &&
+        this.recognizer.fallbackLanguage !== 'default'
         ? this.recognizer.fallbackLanguage
         : cs.fallbackSpeechRecognizerLanguage;
 
@@ -305,6 +318,22 @@ class TaskConfig extends Task {
     if (this.hasReferHook) {
       cs.referHook = this.data.referHook;
     }
+
+    if (this.ttsStream.enable && this.sayOpts) {
+      this.sayOpts.synthesizer = this.hasSynthesizer ? this.synthesizer : {
+        vendor: cs.speechSynthesisVendor,
+        language: cs.speechSynthesisLanguage,
+        voice: cs.speechSynthesisVoice,
+        ...(cs.speechSynthesisLabel && {
+          label: cs.speechSynthesisLabel
+        })
+      };
+      this.logger.info({opts: this.gatherOpts}, 'Config: enabling ttsStream');
+      cs.enableBackgroundTtsStream(this.sayOpts);
+    } else if (!this.ttsStream.enable) {
+      this.logger.info('Config: disabling ttsStream');
+      cs.disableTtsStream();
+    }
   }
 
   async kill(cs) {

lib/tasks/dequeue.js

@@ -3,8 +3,7 @@ const {TaskName, TaskPreconditions, DequeueResults, BONG_TONE} = require('../uti
 const Emitter = require('events');
 const bent = require('bent');
 const assert = require('assert');
-
-const sleepFor = (ms) => new Promise((resolve) => setTimeout(() => resolve(), ms));
+const { sleepFor } = require('../utils/helpers');
 
 const getUrl = (cs) => `${cs.srf.locals.serviceUrl}/v1/dequeue/${cs.callSid}`;
 

lib/tasks/dial.js

@@ -6,6 +6,7 @@ const {
   TaskName,
   TaskPreconditions,
   MAX_SIMRINGS,
+  MediaPath,
   KillReason
 } = require('../utils/constants');
 const assert = require('assert');
@@ -17,9 +18,13 @@ const dbUtils = require('../utils/db-utils');
 const parseDecibels = require('../utils/parse-decibels');
 const debug = require('debug')('jambonz:feature-server');
 const {parseUri} = require('drachtio-srf');
-const {ANCHOR_MEDIA_ALWAYS, JAMBONZ_DISABLE_DIAL_PAI_HEADER} = require('../config');
+const {ANCHOR_MEDIA_ALWAYS,
+  JAMBONZ_DISABLE_DIAL_PAI_HEADER,
+  JAMBONES_DIAL_SBC_FOR_REGISTERED_USER} = require('../config');
 const { isOnhold, isOpusFirst } = require('../utils/sdp-utils');
 const { normalizeJambones } = require('@jambonz/verb-specifications');
+const { selectHostPort } = require('../utils/network');
+const { sleepFor } = require('../utils/helpers');
 
 function parseDtmfOptions(logger, dtmfCapture) {
   let parentDtmfCollector, childDtmfCollector;
@@ -82,8 +87,6 @@ function filterAndLimit(logger, tasks) {
   return unique;
 }
 
-const sleepFor = (ms) => new Promise((resolve) => setTimeout(() => resolve(), ms));
-
 class TaskDial extends Task {
   constructor(logger, opts) {
     super(logger, opts);
@@ -105,6 +108,7 @@ class TaskDial extends Task {
     this.proxy = this.data.proxy;
     this.tag = this.data.tag;
     this.boostAudioSignal = this.data.boostAudioSignal;
+    this._mediaPath = MediaPath.FullMedia;
 
     if (this.dtmfHook) {
       const {parentDtmfCollector, childDtmfCollector} = parseDtmfOptions(logger, this.data.dtmfCapture || {});
@@ -116,8 +120,9 @@ class TaskDial extends Task {
       }
     }
 
-    if (this.data.listen) {
-      this.listenTask = makeTask(logger, {'listen': this.data.listen}, this);
+    const listenData = this.data.listen || this.data.stream;
+    if (listenData) {
+      this.listenTask = makeTask(logger, {'listen': listenData }, this);
     }
     if (this.data.transcribe) {
       this.transcribeTask = makeTask(logger, {'transcribe' : this.data.transcribe}, this);
@@ -152,17 +157,21 @@ class TaskDial extends Task {
 
   get canReleaseMedia() {
     const keepAnchor = this.data.anchorMedia ||
-    this.cs.isBackGroundListen ||
-    this.cs.onHoldMusic ||
-    ANCHOR_MEDIA_ALWAYS ||
-    this.listenTask ||
-    this.dubTasks ||
-    this.transcribeTask ||
-    this.startAmd;
+      this.cs.isBackGroundListen ||
+      this.cs.onHoldMusic ||
+      ANCHOR_MEDIA_ALWAYS ||
+      this.listenTask ||
+      this.dubTasks ||
+      this.transcribeTask ||
+      this.startAmd;
 
     return !keepAnchor;
   }
 
+  get shouldExitMediaPathEntirely() {
+    return this.data.exitMediaPath;
+  }
+
   get summary() {
     if (this.target.length === 1) {
       const target = this.target[0];
@@ -183,6 +192,16 @@ class TaskDial extends Task {
 
   async exec(cs) {
     await super.exec(cs);
+
+    if (this.data.anchorMedia && this.data.exitMediaPath) {
+      this.logger.info('Dial:exec - incompatible anchorMedia and exitMediaPath are both set, will obey anchorMedia');
+      delete this.data.exitMediaPath;
+    }
+    if (!this.canReleaseMedia && this.data.exitMediaPath) {
+      this.logger.info(
+        'Dial:exec - exitMediaPath is set so features such as transcribe and record will not work on this call');
+    }
+
     try {
       if (this.listenTask) {
         const {span, ctx} = this.startChildSpan(`nested:${this.listenTask.summary}`);
@@ -210,10 +229,10 @@ class TaskDial extends Task {
               try {
                 await this.epOther.play(this.dialMusic);
               } catch (err) {
-                this.logger.error(err, `TaskDial:exec error playing ${this.dialMusic}`);
+                this.logger.error(err, `TaskDial:exec error playing dialMusic ${this.dialMusic}`);
                 await sleepFor(1000);
               }
-            } while (!this.killed || !this.bridged);
+            } while (!this.killed && !this.bridged && this._mediaPath === MediaPath.FullMedia);
           })();
         }
       }
@@ -254,7 +273,9 @@ class TaskDial extends Task {
     this._removeDtmfDetection(this.dlg);
     await this._killOutdials();
     if (this.sd) {
-      this.sd.kill();
+      const byeReasonHeader = this.killReason === KillReason.MediaTimeout ? 'Media Timeout' : undefined;
+      this.sd.kill(byeReasonHeader);
+      this.sd.ep?.removeListener('destroy', this._handleMediaTimeout.bind(this));
       this.sd.removeAllListeners();
       this.sd = null;
     }
@@ -300,7 +321,7 @@ class TaskDial extends Task {
       if (!cs.callGone && this.epOther) {
 
         /* if we can release the media back to the SBC, do so now */
-        if (this.canReleaseMedia) this._releaseMedia(cs, this.sd);
+        if (this.canReleaseMedia) this._releaseMedia(cs, this.sd, this.shouldExitMediaPathEntirely);
         else this.epOther.bridge(this.ep);
       }
     } catch (err) {
@@ -342,6 +363,12 @@ class TaskDial extends Task {
         const by = parseUri(req.getParsedHeader('Referred-By').uri);
         const referredBy = req.get('Referred-By');
         const userAgent = req.get('User-Agent');
+        const customHeaders = Object.keys(req.headers)
+          .filter((h) => h.toLowerCase().startsWith('x-'))
+          .reduce((acc, h) => {
+            acc[h] = req.get(h);
+            return acc;
+          }, {});
         this.logger.info({to}, 'refer to parsed');
         const json = await cs.requestor.request('verb:hook', this.referHook, {
           ...(callInfo.toJSON()),
@@ -352,7 +379,8 @@ class TaskDial extends Task {
             ...(userAgent && {sip_user_agent: userAgent}),
             ...(by && {referred_by_user: by.scheme === 'tel' ? by.number : by.user}),
             referring_call_sid,
-            referred_call_sid
+            referred_call_sid,
+            ...customHeaders
           }
         }, httpHeaders);
         if (json && Array.isArray(json)) {
@@ -378,6 +406,9 @@ class TaskDial extends Task {
             this.logger.info(err, 'Dial:handleRefer - error setting new application after receiving REFER');
           }
         }
+        //caller and callee legs are briged together, accept refer with 202 will release callee leg endpoint
+        //that makes freeswitch release endpoint for caller leg.
+        if (this.ep) this.ep.unbridge();
         res.send(202);
         this.logger.info('DialTask:handleRefer - sent 202 Accepted');
       } catch (err) {
@@ -467,7 +498,7 @@ class TaskDial extends Task {
     dlg && dlg.removeAllListeners('info');
   }
 
-  async _onRequestWithinDialog(cs, req, res) {
+  _onRequestWithinDialog(cs, req, res) {
     cs._onRequestWithinDialog(req, res);
   }
 
@@ -490,8 +521,8 @@ class TaskDial extends Task {
     const {req, callInfo, direction, srf} = cs;
     const {getSBC} = srf.locals;
     const {lookupTeamsByAccount, lookupAccountBySid} = srf.locals.dbHelpers;
-    const {lookupCarrier, lookupCarrierByPhoneNumber} = dbUtils(this.logger, cs.srf);
-    const sbcAddress = this.proxy || getSBC();
+    const {lookupCarrier, lookupCarrierByPhoneNumber, lookupVoipCarrierBySid} = dbUtils(this.logger, cs.srf);
+    let sbcAddress = this.proxy || getSBC();
     const teamsInfo = {};
     let fqdn;
 
@@ -500,19 +531,27 @@ class TaskDial extends Task {
       'X-Account-Sid': cs.accountSid,
       ...(req && req.has('X-CID') && {'X-CID': req.get('X-CID')}),
       ...(direction === 'outbound' && callInfo.sbcCallid && {'X-CID': callInfo.sbcCallid}),
-      ...(req && req.has('P-Asserted-Identity') && !JAMBONZ_DISABLE_DIAL_PAI_HEADER &&
-        {'P-Asserted-Identity': req.get('P-Asserted-Identity')}),
+      ...(!JAMBONZ_DISABLE_DIAL_PAI_HEADER && req && {
+        ...(req.has('P-Asserted-Identity') && {'P-Asserted-Identity': req.get('P-Asserted-Identity')}),
+        ...(req.has('Privacy') && {'Privacy': req.get('Privacy')}),
+      }),
       ...(req && req.has('X-Voip-Carrier-Sid') && {'X-Voip-Carrier-Sid': req.get('X-Voip-Carrier-Sid')}),
       // Put headers at the end to make sure opt.headers override all default behavior.
       ...this.headers
     };
 
+    // default to inband dtmf if not specified
+    this.inbandDtmfEnabled = cs.inbandDtmfEnabled;
+    // get calling user from From header
+    const parsedFrom = req.getParsedHeader('from');
+    const fromUri = parseUri(parsedFrom.uri);
     const opts = {
       headers: this.headers,
       proxy: `sip:${sbcAddress}`,
-      callingNumber: this.callerId || req.callingNumber,
+      callingNumber: this.callerId || fromUri.user,
       ...(this.callerName && {callingName: this.callerName}),
-      opusFirst: isOpusFirst(this.cs.ep.remote.sdp)
+      opusFirst: isOpusFirst(this.cs.ep.remote.sdp),
+      isVideoCall: this.cs.ep.remote.sdp.includes('m=video')
     };
 
     const t = this.target.find((t) => t.type === 'teams');
@@ -556,6 +595,15 @@ class TaskDial extends Task {
             this.logger.error({err}, 'Error looking up account by sid');
           }
         }
+        // find handling sbc sip for called user
+        if (JAMBONES_DIAL_SBC_FOR_REGISTERED_USER && t.type === 'user') {
+          const { registrar }  = srf.locals.dbHelpers;
+          const reg = await registrar.query(t.name);
+          if (reg) {
+            sbcAddress = selectHostPort(this.logger, reg.sbcAddress, 'tcp')[1];
+          }
+          //sbc outbound return 404 Notfound to handle case called user is not reigstered.
+        }
         if (t.type === 'phone' && t.trunk) {
           const voip_carrier_sid = await lookupCarrier(cs.accountSid, t.trunk);
           this.logger.info(`Dial:_attemptCalls: selected ${voip_carrier_sid} for requested carrier: ${t.trunk}`);
@@ -572,10 +620,17 @@ class TaskDial extends Task {
           const str = this.callerId || req.callingNumber || '';
           const callingNumber = str.startsWith('+') ? str.substring(1) : str;
           const voip_carrier_sid = await lookupCarrierByPhoneNumber(cs.accountSid, callingNumber);
+          const req_voip_carrier_sid = req.has('X-Voip-Carrier-Sid') ? req.get('X-Voip-Carrier-Sid') : null;
           if (voip_carrier_sid) {
             this.logger.info(
               `Dial:_attemptCalls: selected voip_carrier_sid ${voip_carrier_sid} for callingNumber: ${callingNumber}`);
             opts.headers['X-Requested-Carrier-Sid'] = voip_carrier_sid;
+
+            // Checking if outbound carrier is different from inbound carrier and has dtmf type tones
+            if (voip_carrier_sid !== req_voip_carrier_sid) {
+              const [voipCarrier] = await lookupVoipCarrierBySid(voip_carrier_sid);
+              this.inbandDtmfEnabled = voipCarrier?.dtmf_type === 'tones';
+            }
           }
         }
 
@@ -732,7 +787,7 @@ class TaskDial extends Task {
             // Offhold, time to release media
             const newSdp = await this.ep.modify(req.body);
             await res.send(200, {body: newSdp});
-            await this._releaseMedia(this.cs, this.sd);
+            await this._releaseMedia(this.cs, this.sd, this.shouldExitMediaPathEntirely);
             this.isOutgoingLegHold = false;
           } else {
             this.logger.debug('Dial: _onReinvite receive unhold Request, update media server');
@@ -816,10 +871,14 @@ class TaskDial extends Task {
 
     if (this.parentDtmfCollector) this._installDtmfDetection(cs, cs.dlg);
     if (this.childDtmfCollector) this._installDtmfDetection(cs, this.dlg);
-    if (cs.sipRequestWithinDialogHook) this._initSipIndialogRequestListener(cs, this.dlg);
+    if (cs.sipRequestWithinDialogHook) {
+      /* remove any existing listener to escape from duplicating events */
+      this._removeSipIndialogRequestListener(this.dlg);
+      this._initSipIndialogRequestListener(cs, this.dlg);
+    }
 
     if (this.transcribeTask) this.transcribeTask.exec(cs, {ep: this.epOther, ep2:this.ep});
-    if (this.listenTask) this.listenTask.exec(cs, {ep: this.epOther});
+    if (this.listenTask) this.listenTask.exec(cs, {ep: this.listenTask.channel === 2 ? this.ep : this.epOther});
     if (this.startAmd) {
       try {
         this.startAmd(cs, this.ep, this, this.data.amd);
@@ -841,7 +900,17 @@ class TaskDial extends Task {
     }
 
     /* if we can release the media back to the SBC, do so now */
-    if (this.canReleaseMedia) setTimeout(this._releaseMedia.bind(this, cs, sd), 200);
+    if (this.canReleaseMedia || this.shouldExitMediaPathEntirely) {
+      setTimeout(this._releaseMedia.bind(this, cs, sd, this.shouldExitMediaPathEntirely), 200);
+    }
+
+    this.sd.ep.once('destroy', this._handleMediaTimeout.bind(this));
+  }
+
+  _handleMediaTimeout(evt) {
+    if (evt?.reason === 'MEDIA_TIMEOUT' && this.sd && this.bridged) {
+      this.kill(this.cs, KillReason.MediaTimeout);
+    }
   }
 
   _bridgeEarlyMedia(sd) {
@@ -853,22 +922,57 @@ class TaskDial extends Task {
     }
   }
 
+  /* public api */
+  async updateMediaPath(desiredPath) {
+    this.logger.info(`Dial:updateMediaPath - ${this._mediaPath} => ${desiredPath}`);
+    switch (desiredPath) {
+      case MediaPath.NoMedia:
+        assert(this._mediaPath !== MediaPath.NoMedia, 'updateMediaPath: already no-media');
+        await this._releaseMedia(this.cs, this.sd, true);
+        break;
+
+      case MediaPath.PartialMedia:
+        assert(this._mediaPath !== MediaPath.PartialMedia, 'updateMediaPath: already partial-media');
+        if (this._mediaPath === MediaPath.FullMedia) {
+          await this._releaseMedia(this.cs, this.sd, false);
+        }
+        else {
+          // to go from no-media to partial-media we need to go through full-media first
+          await this.reAnchorMedia(this.cs, this.sd);
+          await this._releaseMedia(this.cs, this.sd, false);
+        }
+        assert(!this.epOther, 'updateMediaPath: epOther should be null');
+        assert(!this.ep, 'updateMediaPath: ep should be null');
+
+        break;
+      case MediaPath.FullMedia:
+        assert(this._mediaPath !== MediaPath.FullMedia, 'updateMediaPath: already full-media');
+        await this.reAnchorMedia(this.cs, this.sd);
+        break;
+
+      default:
+        assert(false, `updateMediaPath: invalid path request ${desiredPath}`);
+    }
+  }
+
   /**
    * Release the media from freeswitch
    * @param {*} cs
    * @param {*} sd
    */
-  async _releaseMedia(cs, sd) {
+  async _releaseMedia(cs, sd, releaseEntirely = false) {
     assert(cs.ep && sd.ep);
 
     try {
       // Wait until we got new SDP from B leg to ofter to A Leg
       const aLegSdp = cs.ep.remote.sdp;
-      await sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp);
+      await sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp, releaseEntirely);
       const bLegSdp = sd.dlg.remote.sdp;
-      await cs.releaseMediaToSBC(bLegSdp);
+      await cs.releaseMediaToSBC(bLegSdp, releaseEntirely);
       this.epOther = null;
-      this.logger.info('Dial:_releaseMedia - successfully released media from freewitch');
+      this._mediaPath = releaseEntirely ? MediaPath.NoMedia : MediaPath.PartialMedia;
+      this.logger.info(
+        `Dial:_releaseMedia - successfully released media from freewitch, media path is now ${this._mediaPath}`);
     } catch (err) {
       this.logger.info({err}, 'Dial:_releaseMedia error');
     }
@@ -878,8 +982,14 @@ class TaskDial extends Task {
     if (cs.ep && sd.ep) return;
 
     this.logger.info('Dial:reAnchorMedia - re-anchoring media to freewitch');
-    await Promise.all([sd.reAnchorMedia(), cs.reAnchorMedia()]);
+    await Promise.all([sd.reAnchorMedia(this._mediaPath), cs.reAnchorMedia(this._mediaPath)]);
     this.epOther = cs.ep;
+
+    this.epOther.bridge(this.ep);
+
+    this._mediaPath = MediaPath.FullMedia;
+    this.logger.info(
+      `Dial:_releaseMedia - successfully re-anchored media to freewitch, media path is now ${this._mediaPath}`);
   }
 
   // Handle RE-INVITE hold from caller leg.
@@ -898,11 +1008,12 @@ class TaskDial extends Task {
         }
         this._onHoldHook(req);
       } else if (!isOnhold(req.body)) {
-        if (this.epOther && this.ep && this.isIncomingLegHold && this.canReleaseMedia) {
+        if (this.epOther && this.ep && this.isIncomingLegHold &&
+          (this.canReleaseMedia || this.shouldExitMediaPathEntirely)) {
           // Offhold, time to release media
           const newSdp = await this.epOther.modify(req.body);
           await res.send(200, {body: newSdp});
-          await this._releaseMedia(this.cs, this.sd);
+          await this._releaseMedia(this.cs, this.sd, this.shouldExitMediaPathEntirely);
           isHandled = true;
         }
         this.isIncomingLegHold = false;
@@ -961,7 +1072,8 @@ class TaskDial extends Task {
               callInfo: this.cs.callInfo,
               accountInfo: this.cs.accountInfo,
               tasks,
-              rootSpan: this.cs.rootSpan
+              rootSpan: this.cs.rootSpan,
+              req: this.cs.req
             });
             await this._onHoldSession.exec();
             this._onHoldSession = null;

lib/tasks/enqueue.js

@@ -369,7 +369,8 @@ class TaskEnqueue extends Task {
         callInfo: cs.callInfo,
         accountInfo: cs.accountInfo,
         tasks: tasksToRun,
-        rootSpan: cs.rootSpan
+        rootSpan: cs.rootSpan,
+        req: cs.req
       });
       await this._playSession.exec();
       this._playSession = null;

lib/tasks/gather.js

@@ -11,6 +11,8 @@ const {
   NvidiaTranscriptionEvents,
   JambonzTranscriptionEvents,
   AssemblyAiTranscriptionEvents,
+  VoxistTranscriptionEvents,
+  OpenAITranscriptionEvents,
   VadDetection,
   VerbioTranscriptionEvents,
   SpeechmaticsTranscriptionEvents
@@ -24,6 +26,7 @@ const makeTask = require('./make_task');
 const assert = require('assert');
 const SttTask = require('./stt-task');
 const { SpeechCredentialError } = require('../utils/error');
+const SPEECHMATICS_DEFAULT_ASR_TIMEOUT = 1200;
 
 class TaskGather extends SttTask {
   constructor(logger, opts, parentTask) {
@@ -81,6 +84,7 @@ class TaskGather extends SttTask {
     this._bufferedTranscripts = [];
     this.partialTranscriptsCount = 0;
     this.bugname_prefix = 'gather_';
+
   }
 
   get name() { return TaskName.Gather; }
@@ -108,6 +112,12 @@ class TaskGather extends SttTask {
     return this.fillerNoise.startDelaySecs;
   }
 
+  get isStreamingTts() { return this.sayTask && this.sayTask.isStreamingTts; }
+
+  getTtsVendorData() {
+    if (this.sayTask) return this.sayTask.getTtsVendorData(this.cs);
+  }
+
   get summary() {
     let s = `${this.name}{`;
     if (this.input.length === 2) s += 'inputs=[speech,digits],';
@@ -140,7 +150,6 @@ class TaskGather extends SttTask {
   async handling(cs, {ep}) {
     this.logger.debug({options: this.data}, 'Gather:exec');
     await super.exec(cs, {ep});
-    const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
 
     this.fillerNoise = {
       ...(cs.fillerNoise || {}),
@@ -156,12 +165,23 @@ class TaskGather extends SttTask {
       const {hints, hintsBoost} = cs.globalSttHints;
       const setOfHints = new Set((this.data.recognizer.hints || [])
         .concat(hints)
-        .filter((h) => typeof h === 'string' && h.length > 0));
+        // allow for hints to be an array of object
+        .filter((h) => (typeof h === 'string' && h.length > 0) || (typeof h === 'object')));
       this.data.recognizer.hints = [...setOfHints];
       if (!this.data.recognizer.hintsBoost && hintsBoost) this.data.recognizer.hintsBoost = hintsBoost;
       this.logger.debug({hints: this.data.recognizer.hints, hintsBoost: this.data.recognizer.hintsBoost},
         'Gather:exec - applying global sttHints');
     }
+
+    // specials case for speechmatics: they dont do endpointing so we need to enable continuous ASR
+    if (this.vendor === 'speechmatics' && !this.isContinuousAsr) {
+      const maxDelay = this.recognizer?.speechmaticsOptions?.transcription_config?.max_delay;
+      if (maxDelay) this.asrTimeout = Math.min(SPEECHMATICS_DEFAULT_ASR_TIMEOUT, maxDelay * 1000);
+      else this.asrTimeout = SPEECHMATICS_DEFAULT_ASR_TIMEOUT;
+      this.isContinuousAsr = true;
+      this.logger.debug(`Gather:exec - auto-enabling continuous ASR for speechmatics w/ timeout ${this.asrTimeout}`);
+    }
+
     if (!this.isContinuousAsr && cs.isContinuousAsr) {
       this.isContinuousAsr = true;
       this.asrTimeout = cs.asrTimeout * 1000;
@@ -210,7 +230,6 @@ class TaskGather extends SttTask {
             return;
           }
           this._startTranscribing(ep);
-          return updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid);
         } catch (e) {
           await this._startFallback(cs, ep, {error: e});
         }
@@ -222,6 +241,7 @@ class TaskGather extends SttTask {
         const {span, ctx} = this.startChildSpan(`nested:${this.sayTask.summary}`);
         const process = () => {
           this.logger.debug('Gather: nested say task completed');
+          this.playComplete = true;
           if (!this.listenDuringPrompt) {
             startDtmfListener();
           }
@@ -252,6 +272,7 @@ class TaskGather extends SttTask {
         const {span, ctx} = this.startChildSpan(`nested:${this.playTask.summary}`);
         const process = () => {
           this.logger.debug('Gather: nested play task completed');
+          this.playComplete = true;
           if (!this.listenDuringPrompt) {
             startDtmfListener();
           }
@@ -290,8 +311,6 @@ class TaskGather extends SttTask {
         await this._setSpeechHandlers(cs, ep);
         if (!this.resolved && !this.killed) {
           this._startTranscribing(ep);
-          updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid)
-            .catch(() => {/*already logged error */});
         }
         else {
           this.logger.info('Gather:exec - task was killed or resolved quickly, not starting transcription');
@@ -346,6 +365,13 @@ class TaskGather extends SttTask {
       this._killAudio(cs);
       this.emit('dtmf', evt);
     }
+    if (this.isContinuousAsr && evt.dtmf === this.asrDtmfTerminationDigit && this._bufferedTranscripts.length > 0) {
+      this.logger.info(`continuousAsr triggered with dtmf ${this.asrDtmfTerminationDigit}`);
+      this._clearAsrTimer();
+      this._clearTimer();
+      this._startFinalAsrTimer();
+      return;
+    }
     if (evt.dtmf === this.finishOnKey && this.input.includes('digits')) {
       resolved = true;
       this._resolve('dtmf-terminator-key');
@@ -368,13 +394,6 @@ class TaskGather extends SttTask {
         this._resolve('dtmf-num-digits');
       }
     }
-    else if (this.isContinuousAsr && evt.dtmf === this.asrDtmfTerminationDigit) {
-      this.logger.info(`continuousAsr triggered with dtmf ${this.asrDtmfTerminationDigit}`);
-      this._clearAsrTimer();
-      this._clearTimer();
-      this._startFinalAsrTimer();
-      return;
-    }
     if (!resolved && this.interDigitTimeout > 0 && this.digitBuffer.length >= this.minDigits) {
       /* start interDigitTimer */
       const ms = this.interDigitTimeout * 1000;
@@ -516,6 +535,17 @@ class TaskGather extends SttTask {
           this._onVendorConnectFailure.bind(this, cs, ep));
         break;
 
+      case 'voxist':
+        this.bugname = `${this.bugname_prefix}voxist_transcribe`;
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.Transcription,
+          this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(
+          ep, VoxistTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.ConnectFailure,
+          this._onVendorConnectFailure.bind(this, cs, ep));
+        break;
+
       case 'speechmatics':
         this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
         this.addCustomEventListener(
@@ -533,6 +563,31 @@ class TaskGather extends SttTask {
 
         break;
 
+      case 'openai':
+        this.bugname = `${this.bugname_prefix}openai_transcribe`;
+        this.addCustomEventListener(
+          ep, OpenAITranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(
+          ep, OpenAITranscriptionEvents.SpeechStarted, this._onOpenAISpeechStarted.bind(this, cs, ep));
+        this.addCustomEventListener(
+          ep, OpenAITranscriptionEvents.SpeechStopped, this._onOpenAISpeechStopped.bind(this, cs, ep));
+        this.addCustomEventListener(ep, OpenAITranscriptionEvents.Connect,
+          this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, OpenAITranscriptionEvents.ConnectFailure,
+          this._onVendorConnectFailure.bind(this, cs, ep));
+        this.addCustomEventListener(ep, OpenAITranscriptionEvents.Error,
+          this._onOpenAIErrror.bind(this, cs, ep));
+
+        /* openai delta transcripts are useful only for minBargeinWordCount eval */
+        if (this.minBargeinWordCount > 1) {
+          this.openaiPartials = [];
+          opts.OPENAI_WANT_PARTIALS = 1;
+          this.addCustomEventListener(
+            ep, OpenAITranscriptionEvents.PartialTranscript, this._onOpenAIPartialTranscript.bind(this, cs, ep));
+        }
+        this.modelSupportsConversationTracking = opts.OPENAI_MODEL !== 'whisper-1';
+        break;
+
       default:
         if (this.vendor.startsWith('custom:')) {
           this.bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
@@ -564,6 +619,25 @@ class TaskGather extends SttTask {
       bugname: this.bugname
     }, 'Gather:_startTranscribing');
 
+
+    /* special feature for openai: we can provide a prompt that includes recent conversation history */
+    let prompt;
+    if (this.vendor === 'openai') {
+      if (this.modelSupportsConversationTracking) {
+        prompt = this.formatOpenAIPrompt(this.cs, {
+          prompt: this.data.recognizer?.openaiOptions?.prompt,
+          hintsTemplate: this.data.recognizer?.openaiOptions?.promptTemplates?.hintsTemplate,
+          // eslint-disable-next-line max-len
+          conversationHistoryTemplate: this.data.recognizer?.openaiOptions?.promptTemplates?.conversationHistoryTemplate,
+          hints: this.data.recognizer?.hints,
+        });
+        this.logger.debug({prompt}, 'Gather:_startTranscribing - created an openai prompt');
+      }
+      else if (this.data.recognizer?.hints?.length > 0) {
+        prompt = this.data.recognizer?.hints.join(', ');
+      }
+    }
+
     /**
      * Note: we don't need to ask deepgram for interim results, because they
      * already send us words as they are finalized (is_final=true) even before
@@ -575,6 +649,7 @@ class TaskGather extends SttTask {
       interim: this.interim,
       bugname: this.bugname,
       hostport: this.hostport,
+      prompt
     }).catch((err) => {
       const {writeAlerts, AlertType} = this.cs.srf.locals;
       this.logger.error(err, 'TaskGather:_startTranscribing error');
@@ -610,12 +685,21 @@ class TaskGather extends SttTask {
   }
 
   _startAsrTimer() {
-    if (this.vendor === 'deepgram') return; // no need
+    // Deepgram has a case that UtteranceEnd is not sent to cover the last word end time.
+    // So we need to wait for the asrTimeout to be sure that the last word is sent.
+    // if (this.vendor === 'deepgram') return; // no need
     assert(this.isContinuousAsr);
     this._clearAsrTimer();
     this._asrTimer = setTimeout(() => {
       this.logger.debug('_startAsrTimer - asr timer went off');
       const evt = this.consolidateTranscripts(this._bufferedTranscripts, 1, this.language, this.vendor);
+
+      /* special case for speechmatics - keep listening if we dont have any transcripts */
+      if (this.vendor === 'speechmatics' && this._bufferedTranscripts.length === 0) {
+        this.logger.debug('Gather:_startAsrTimer - speechmatics, no transcripts yet, keep listening');
+        this._startAsrTimer();
+        return;
+      }
       this._resolve(this._bufferedTranscripts.length > 0 ? 'speech' : 'timeout', evt);
     }, this.asrTimeout);
     this.logger.debug(`_startAsrTimer: set for ${this.asrTimeout}ms`);
@@ -726,6 +810,7 @@ class TaskGather extends SttTask {
         this._fillerNoiseOn = false;  // in a race, if we just started audio it may sneak through here
         this.ep.api('uuid_break', this.ep.uuid)
           .catch((err) => this.logger.info(err, 'Error killing audio'));
+        cs.clearTtsStream();
       }
       return;
     }
@@ -747,7 +832,11 @@ class TaskGather extends SttTask {
     const bugname = fsEvent.getHeader('media-bugname');
     const finished = fsEvent.getHeader('transcription-session-finished');
     this.logger.debug({evt, bugname, finished, vendor: this.vendor}, 'Gather:_onTranscription raw transcript');
-    if (bugname && this.bugname !== bugname) return;
+    if (bugname && this.bugname !== bugname) {
+      this.logger.debug(
+        `Gather:_onTranscription - ignoring transcript from ${bugname} because our bug is ${this.bugname}`);
+      return;
+    }
     if (finished === 'true') return;
 
     if (this.vendor === 'ibm' && evt?.state === 'listening') return;
@@ -757,10 +846,17 @@ class TaskGather extends SttTask {
         this.logger.debug('Gather:_onTranscription - got UtteranceEnd event from deepgram but no buffered transcripts');
       }
       else {
-        this.logger.debug('Gather:_onTranscription - got UtteranceEnd event from deepgram, return buffered transcript');
-        evt = this.consolidateTranscripts(this._bufferedTranscripts, 1, this.language, this.vendor);
-        this._bufferedTranscripts = [];
-        this._resolve('speech', evt);
+        const utteranceTime = evt.last_word_end;
+        // eslint-disable-next-line max-len
+        if (utteranceTime && this._dgTimeOfLastUnprocessedWord && utteranceTime < this._dgTimeOfLastUnprocessedWord && utteranceTime != -1) {
+          this.logger.debug('Gather:_onTranscription - got UtteranceEnd with unprocessed words, continue listening');
+        }
+        else {
+          this.logger.debug('Gather:_onTranscription - got UtteranceEnd from deepgram, return buffered transcript');
+          evt = this.consolidateTranscripts(this._bufferedTranscripts, 1, this.language, this.vendor);
+          this._bufferedTranscripts = [];
+          this._resolve('speech', evt);
+        }
       }
       return;
     }
@@ -771,7 +867,7 @@ class TaskGather extends SttTask {
 
     evt = this.normalizeTranscription(evt, this.vendor, 1, this.language,
       this.shortUtterance, this.data.recognizer.punctuation);
-    this.logger.debug({evt, bugname, finished, vendor: this.vendor}, 'Gather:_onTranscription normalized transcript');
+    //this.logger.debug({evt, bugname, finished, vendor: this.vendor}, 'Gather:_onTranscription normalized transcript');
 
     if (evt.alternatives.length === 0) {
       this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, continue listening');
@@ -779,8 +875,6 @@ class TaskGather extends SttTask {
     }
     const confidence = evt.alternatives[0].confidence;
     const minConfidence = this.data.recognizer?.minConfidence;
-    this.logger.debug({evt},
-      `TaskGather:_onTranscription - confidence (${confidence}), minConfidence (${minConfidence})`);
     if (confidence && minConfidence && confidence < minConfidence) {
       this.logger.info({evt},
         'TaskGather:_onTranscription - Transcript confidence ' +
@@ -834,7 +928,7 @@ class TaskGather extends SttTask {
         const t = evt.alternatives[0].transcript;
         if (t) {
           /* remove trailing punctuation */
-          if (/[,;:\.!\?]$/.test(t)) {
+          if (this.vendor !== 'speechmatics' && /[,;:\.!\?]$/.test(t)) {
             this.logger.debug('TaskGather:_onTranscription - removing trailing punctuation');
             evt.alternatives[0].transcript = t.slice(0, -1);
           }
@@ -850,7 +944,10 @@ class TaskGather extends SttTask {
         this._startAsrTimer();
 
         /* some STT engines will keep listening after a final response, so no need to restart */
-        if (!['soniox', 'aws', 'microsoft', 'deepgram'].includes(this.vendor)) this._startTranscribing(ep);
+        if (!['soniox', 'aws', 'microsoft', 'deepgram', 'speechmatics'].includes(this.vendor) &&
+          !this.vendor.startsWith('custom')) {
+          this._startTranscribing(ep);
+        }
       }
       else {
         /* this was removed to fix https://github.com/jambonz/jambonz-feature-server/issues/783 */
@@ -893,8 +990,21 @@ class TaskGather extends SttTask {
         if (originalEvent.is_final && evt.alternatives[0].transcript !== '') {
           this.logger.debug({evt}, 'Gather:_onTranscription - buffering a completed (partial) deepgram transcript');
           this._bufferedTranscripts.push(evt);
+          this._dgTimeOfLastUnprocessedWord = null;
+        }
+        if (evt.alternatives[0].transcript === '') {
+          emptyTranscript = true;
+        }
+        else if (!originalEvent.is_final) {
+          /* Deepgram: we have unprocessed words-save last word end time so we can later compare to UtteranceEnd */
+          const words = originalEvent.channel.alternatives[0].words;
+          if (words?.length > 0) {
+            this._dgTimeOfLastUnprocessedWord = words.slice(-1)[0].end;
+            this.logger.debug(
+              `TaskGather:_onTranscription - saving word end time: ${this._dgTimeOfLastUnprocessedWord}`);
+          }
+
         }
-        if (evt.alternatives[0].transcript === '') emptyTranscript = true;
       }
 
       if (!emptyTranscript) {
@@ -924,7 +1034,7 @@ class TaskGather extends SttTask {
         }
       }
       // If transcription received, reset timeout timer.
-      if (this._timeoutTimer) {
+      if (this._timeoutTimer && !emptyTranscript) {
         this._startTimer();
       }
       /* restart asr timer if we get a partial transcript (only if the asr timer is already running) */
@@ -969,7 +1079,6 @@ class TaskGather extends SttTask {
         bugname: this.bugname
       })
         .catch((err) => this.logger.error({err}, `Error stopping transcription for primary vendor ${this.vendor}`));
-      const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
       try {
         this.logger.debug('gather:_startFallback');
         this.notifyError({ msg: 'ASR error',
@@ -978,7 +1087,6 @@ class TaskGather extends SttTask {
         this._speechHandlersSet = false;
         await this._setSpeechHandlers(cs, ep);
         this._startTranscribing(ep);
-        updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid);
         return true;
       } catch (error) {
         this.logger.info({error}, `There is error while falling back to ${this.fallbackVendor}`);
@@ -1032,6 +1140,33 @@ class TaskGather extends SttTask {
     this._onVendorError(cs, _ep, {error: JSON.stringify(e)});
   }
 
+  async _onOpenAIErrror(cs, _ep, evt) {
+    // eslint-disable-next-line no-unused-vars
+    const {message, ...e} = evt;
+    this._onVendorError(cs, _ep, {error: JSON.stringify(e)});
+  }
+
+  async _onOpenAISpeechStarted(cs, _ep, evt) {
+    this.logger.debug({evt}, 'TaskGather:_onOpenAISpeechStarted');
+  }
+
+  async _onOpenAISpeechStopped(cs, _ep, evt) {
+    this.logger.debug({evt}, 'TaskGather:_onOpenAISpeechStopped');
+  }
+
+  async _onOpenAIPartialTranscript(cs, _ep, evt) {
+    if (!this.playComplete) {
+      const words = evt.delta.split(' ').filter((w) => /[A-Za-z0-0]/.test(w));
+      this.openaiPartials.push(...words);
+      this.logger.debug({words, partials: this.openaiPartials, evt}, 'TaskGather:_onOpenAIPartialTranscript - words');
+      if (this.openaiPartials.length >= this.minBargeinWordCount) {
+        this.logger.debug({partials: this.openaiPartials}, 'killing audio due to speech (openai)');
+        this._killAudio(cs);
+        this.notifyStatus({event: 'speech-bargein-detected', words: this.openaiPartials});
+      }
+    }
+  }
+
   async _onVendorError(cs, _ep, evt) {
     super._onVendorError(cs, _ep, evt);
     if (!(await this._startFallback(cs, _ep, evt))) {
@@ -1141,6 +1276,7 @@ class TaskGather extends SttTask {
         }
       }
       else if (reason.startsWith('speech')) {
+        this.cs.emit('userSaid', evt.alternatives[0].transcript);
         if (this.parentTask) this.parentTask.emit('transcription', evt);
         else {
           this.emit('transcription', evt);
@@ -1166,13 +1302,12 @@ class TaskGather extends SttTask {
         if (this.parentTask) this.parentTask.emit('stt-low-confidence', evt);
         else {
           this.emit('stt-low-confidence', evt);
-          returnedVerbs = await this.performAction({reason: 'stt-low-confidence'});
+          returnedVerbs = await this.performAction({speech:evt, reason: 'stt-low-confidence'});
         }
       }
     } catch (err) {  /*already logged error*/ }
 
     // Gather got response from hook, cancel actionHookDelay processing
-    this.logger.debug('TaskGather:_resolve - checking ahd');
     if (this.cs.actionHookDelayProcessor) {
       if (returnedVerbs) {
         this.logger.debug('TaskGather:_resolve - got response from action hook, cancelling actionHookDelay');

lib/tasks/listen.js

@@ -17,7 +17,7 @@ class TaskListen extends Task {
 
     [
       'action', 'auth', 'method', 'url', 'finishOnKey', 'maxLength', 'metadata', 'mixType', 'passDtmf', 'playBeep',
-      'sampleRate', 'timeout', 'transcribe', 'wsAuth', 'disableBidirectionalAudio'
+      'sampleRate', 'timeout', 'transcribe', 'wsAuth', 'disableBidirectionalAudio', 'channel'
     ].forEach((k) => this[k] = this.data[k]);
 
     this.mixType = this.mixType || 'mono';
@@ -221,7 +221,7 @@ class TaskListen extends Task {
     }
   }
   _onConnect(ep) {
-    this.logger.debug('TaskListen:_onConnect');
+    this.logger.info('TaskListen:_onConnect');
   }
   _onConnectFailure(ep, evt) {
     this.logger.info(evt, 'TaskListen:_onConnectFailure');

lib/tasks/llm/index.js

@@ -1,6 +1,11 @@
 const Task = require('../task');
 const {TaskPreconditions} = require('../../utils/constants');
 const TaskLlmOpenAI_S2S = require('./llms/openai_s2s');
+const TaskLlmVoiceAgent_S2S = require('./llms/voice_agent_s2s');
+const TaskLlmUltravox_S2S = require('./llms/ultravox_s2s');
+const TaskLlmElevenlabs_S2S = require('./llms/elevenlabs_s2s');
+const TaskLlmGoogle_S2S = require('./llms/google_s2s');
+const LlmMcpService = require('../../utils/llm-mcp');
 
 class TaskLlm extends Task {
   constructor(logger, opts) {
@@ -15,6 +20,8 @@ class TaskLlm extends Task {
 
     // delegate to the specific llm model
     this.llm = this.createSpecificLlm();
+    // MCP
+    this.mcpServers = this.data.mcpServers || [];
   }
 
   get name() { return this.llm.name ; }
@@ -25,14 +32,32 @@ class TaskLlm extends Task {
 
   get ep() { return this.cs.ep; }
 
+  get mcpService() {
+    return this.llmMcpService;
+  }
+
+  get isMcpEnabled() {
+    return this.mcpServers.length > 0;
+  }
+
   async exec(cs, {ep}) {
     await super.exec(cs, {ep});
+
+    // create the MCP service if we have MCP servers
+    if (this.isMcpEnabled) {
+      this.llmMcpService = new LlmMcpService(this.logger, this.mcpServers);
+      await this.llmMcpService.init();
+    }
     await this.llm.exec(cs, {ep});
   }
 
   async kill(cs) {
     super.kill(cs);
     await this.llm.kill(cs);
+    // clean up MCP clients
+    if (this.isMcpEnabled) {
+      await this.mcpService.close();
+    }
   }
 
   createSpecificLlm() {
@@ -40,10 +65,26 @@ class TaskLlm extends Task {
     switch (this.vendor) {
       case 'openai':
       case 'microsoft':
-        if (this.model.startsWith('gpt-4o-realtime')) {
-          llm = new TaskLlmOpenAI_S2S(this.logger, this.data, this);
-        }
+        llm = new TaskLlmOpenAI_S2S(this.logger, this.data, this);
         break;
+
+      case 'voiceagent':
+      case 'deepgram':
+        llm = new TaskLlmVoiceAgent_S2S(this.logger, this.data, this);
+        break;
+
+      case 'ultravox':
+        llm = new TaskLlmUltravox_S2S(this.logger, this.data, this);
+        break;
+
+      case 'elevenlabs':
+        llm = new TaskLlmElevenlabs_S2S(this.logger, this.data, this);
+        break;
+
+      case 'google':
+        llm = new TaskLlmGoogle_S2S(this.logger, this.data, this);
+        break;
+
       default:
         throw new Error(`Unsupported vendor ${this.vendor} for LLM`);
     }
@@ -67,8 +108,15 @@ class TaskLlm extends Task {
     await this.cs?.requestor.request('llm:event', this.eventHook, data);
   }
 
+
   async sendToolHook(tool_call_id, data) {
-    await this.cs?.requestor.request('llm:tool-call', this.toolHook, {tool_call_id, ...data});
+    const tool_response = await this.cs?.requestor.request('llm:tool-call', this.toolHook, {tool_call_id, ...data});
+    // if the toolHook was a websocket it will return undefined, otherwise it should return an object
+    if (typeof tool_response != 'undefined') {
+      tool_response.type = 'client_tool_result';
+      tool_response.invocation_id = tool_call_id;
+      this.processToolOutput(tool_call_id, tool_response);
+    }
   }
 
   async processToolOutput(tool_call_id, data) {

lib/tasks/llm/llms/elevenlabs_s2s.js

@@ -0,0 +1,327 @@
+const Task = require('../../task');
+const TaskName = 'Llm_Elevenlabs_s2s';
+const {LlmEvents_Elevenlabs} = require('../../../utils/constants');
+const {request} = require('undici');
+const ClientEvent = 'client.event';
+const SessionDelete = 'session.delete';
+
+const elevenlabs_server_events = [
+  'conversation_initiation_metadata',
+  'user_transcript',
+  'agent_response',
+  'client_tool_call'
+];
+
+const expandWildcards = (events) => {
+  const expandedEvents = [];
+
+  events.forEach((evt) => {
+    if (evt.endsWith('.*')) {
+      const prefix = evt.slice(0, -2); // Remove the wildcard ".*"
+      const matchingEvents = elevenlabs_server_events.filter((e) => e.startsWith(prefix));
+      expandedEvents.push(...matchingEvents);
+    } else {
+      expandedEvents.push(evt);
+    }
+  });
+
+  return expandedEvents;
+};
+
+class TaskLlmElevenlabs_S2S extends Task {
+  constructor(logger, opts, parentTask) {
+    super(logger, opts, parentTask);
+    this.parent = parentTask;
+
+    this.vendor = this.parent.vendor;
+    this.auth = this.parent.auth;
+
+    const {agent_id, api_key} = this.auth || {};
+    if (!agent_id) throw new Error('auth.agent_id is required for Elevenlabs S2S');
+
+    this.agent_id = agent_id;
+    this.api_key = api_key;
+    this.actionHook = this.data.actionHook;
+    this.eventHook = this.data.eventHook;
+    this.toolHook = this.data.toolHook;
+    const {
+      conversation_initiation_client_data,
+      input_sample_rate = 16000,
+      output_sample_rate = 16000
+    } = this.data.llmOptions;
+    this.conversation_initiation_client_data = conversation_initiation_client_data;
+    this.input_sample_rate = input_sample_rate;
+    this.output_sample_rate = output_sample_rate;
+    this.results = {
+      completionReason: 'normal conversation end'
+    };
+
+    /**
+     * only one of these will have items,
+     * if includeEvents, then these are the events to include
+     * if excludeEvents, then these are the events to exclude
+     */
+    this.includeEvents = [];
+    this.excludeEvents = [];
+
+    /* default to all events if user did not specify */
+    this._populateEvents(this.data.events || elevenlabs_server_events);
+
+    this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
+    this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
+  }
+
+  get name() { return TaskName; }
+
+  async getSignedUrl() {
+    if (!this.api_key) {
+      return {
+        host: 'api.elevenlabs.io',
+        path: `/v1/convai/conversation?agent_id=${this.agent_id}`,
+      };
+    }
+
+    const {statusCode, body} = await request(
+      `https://api.elevenlabs.io/v1/convai/conversation/get_signed_url?agent_id=${this.agent_id}`, {
+        method: 'GET',
+        headers: {
+          'xi-api-key': this.api_key
+        },
+      }
+    );
+    const data = await body.json();
+    if (statusCode !== 200 || !data?.signed_url) {
+      this.logger.error({statusCode, data}, 'Elevenlabs Error registering call');
+      throw new Error(`Elevenlabs  Error registering call: ${data.message}`);
+    }
+
+    const url = new URL(data.signed_url);
+    return {
+      host: url.hostname,
+      path: url.pathname + url.search,
+    };
+  }
+
+  async _api(ep, args) {
+    const res = await ep.api('uuid_elevenlabs_s2s', `^^|${args.join('|')}`);
+    if (!res.body?.startsWith('+OK')) {
+      throw new Error({args}, `Error calling uuid_elevenlabs_s2s: ${res.body}`);
+    }
+  }
+
+  async exec(cs, {ep}) {
+    await super.exec(cs);
+    await this._startListening(cs, ep);
+
+    await this.awaitTaskDone();
+
+    /* note: the parent llm verb started the span, which is why this is necessary */
+    await this.parent.performAction(this.results);
+
+    this._unregisterHandlers();
+  }
+
+  async kill(cs) {
+    super.kill(cs);
+
+    this._api(cs.ep, [cs.ep.uuid, SessionDelete])
+      .catch((err) => this.logger.info({err}, 'TaskLlmElevenlabs_S2S:kill - error deleting session'));
+
+    this.notifyTaskDone();
+  }
+
+  /**
+   * Send function call output to the Elevenlabs server in the form of conversation.item.create
+   * per https://elevenlabs.io/docs/conversational-ai/api-reference/conversational-ai/websocket
+   */
+  async processToolOutput(ep, tool_call_id, rawData) {
+    try {
+      const {data} = rawData;
+      this.logger.debug({tool_call_id, data}, 'TaskLlmElevenlabs_S2S:processToolOutput');
+
+      if (!data.type || data.type !== 'client_tool_result') {
+        this.logger.info({data},
+          'TaskLlmElevenlabs_S2S:processToolOutput - invalid tool output, must be client_tool_result');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+      }
+    } catch (err) {
+      this.logger.info({err}, 'TaskLlmElevenlabs_S2S:processToolOutput');
+    }
+  }
+
+  /**
+   * Send a session.update to the Elevenlabs server
+   * Note: creating and deleting conversation items also supported as well as interrupting the assistant
+   */
+  async processLlmUpdate(ep, data, _callSid) {
+    this.logger.debug({data, _callSid}, 'TaskLlmElevenlabs_S2S:processLlmUpdate, ignored');
+  }
+
+  async _startListening(cs, ep) {
+    this._registerHandlers(ep);
+
+    try {
+      const {host, path} = await this.getSignedUrl();
+      const args = this.conversation_initiation_client_data ?
+        [ep.uuid, 'session.create', this.input_sample_rate, this.output_sample_rate, host, path] :
+        [ep.uuid, 'session.create', this.input_sample_rate, this.output_sample_rate, host, path, 'no_initial_config'];
+      await this._api(ep, args);
+    } catch (err) {
+      this.logger.error({err}, 'TaskLlmElevenlabs_S2S:_startListening');
+      this.notifyTaskDone();
+    }
+  }
+
+  async _sendClientEvent(ep, obj) {
+    let ok = true;
+    this.logger.debug({obj}, 'TaskLlmElevenlabs_S2S:_sendClientEvent');
+    try {
+      const args = [ep.uuid, ClientEvent, JSON.stringify(obj)];
+      await this._api(ep, args);
+    } catch (err) {
+      ok = false;
+      this.logger.error({err}, 'TaskLlmElevenlabs_S2S:_sendClientEvent - Error');
+    }
+    return ok;
+  }
+
+  async _sendInitialMessage(ep) {
+    if (this.conversation_initiation_client_data) {
+      if (!await this._sendClientEvent(ep, {
+        type: 'conversation_initiation_client_data',
+        ...this.conversation_initiation_client_data
+      })) {
+        this.notifyTaskDone();
+      }
+    }
+  }
+
+  _registerHandlers(ep) {
+    this.addCustomEventListener(ep, LlmEvents_Elevenlabs.Connect, this._onConnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Elevenlabs.ConnectFailure, this._onConnectFailure.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Elevenlabs.Disconnect, this._onDisconnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Elevenlabs.ServerEvent, this._onServerEvent.bind(this, ep));
+  }
+
+  _unregisterHandlers() {
+    this.removeCustomEventListeners();
+  }
+
+  _onError(ep, evt) {
+    this.logger.info({evt}, 'TaskLlmElevenlabs_S2S:_onError');
+    this.notifyTaskDone();
+  }
+
+  _onConnect(ep) {
+    this.logger.debug('TaskLlmElevenlabs_S2S:_onConnect');
+    this._sendInitialMessage(ep);
+  }
+  _onConnectFailure(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmElevenlabs_S2S:_onConnectFailure');
+    this.results = {completionReason: 'connection failure'};
+    this.notifyTaskDone();
+  }
+  _onDisconnect(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmElevenlabs_S2S:_onConnectFailure');
+    this.results = {completionReason: 'disconnect from remote end'};
+    this.notifyTaskDone();
+  }
+  async _onServerEvent(ep, evt) {
+    let endConversation = false;
+    const type = evt.type;
+    this.logger.info({evt}, 'TaskLlmElevenlabs_S2S:_onServerEvent');
+
+    if (type === 'error') {
+      endConversation = true;
+      this.results = {
+        completionReason: 'server error',
+        error: evt.error
+      };
+    }
+
+    /* tool calls */
+    else if (type === 'client_tool_call') {
+      this.logger.debug({evt}, 'TaskLlmElevenlabs_S2S:_onServerEvent - function_call');
+      const {tool_name: name, tool_call_id: call_id, parameters: args} = evt.client_tool_call;
+
+      const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+      if (mcpTools.some((tool) => tool.name === name)) {
+        this.logger.debug({name, args}, 'TaskLlmElevenlabs_S2S:_onServerEvent - calling mcp tool');
+        try {
+          const res = await this.parent.mcpService.callMcpTool(name, args);
+          this.logger.debug({res}, 'TaskLlmElevenlabs_S2S:_onServerEvent - function_call - mcp result');
+          this.processToolOutput(ep, call_id, {
+            data: {
+              type: 'client_tool_result',
+              tool_call_id: call_id,
+              result: res.content?.length ? res.content[0] : res.content,
+              is_error: false
+            }
+          });
+          return;
+        }
+        catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmElevenlabs_S2S - error calling mcp tool');
+          this.results = {
+            completionReason: 'client error calling mcp function',
+            error: err
+          };
+          endConversation = true;
+        }
+      } else if (!this.toolHook) {
+        this.logger.warn({evt}, 'TaskLlmElevenlabs_S2S:_onServerEvent - no toolHook defined!');
+      }
+      else {
+        try {
+          await this.parent.sendToolHook(call_id, {name, args});
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmElevenlabs_S2S - error calling function');
+          this.results = {
+            completionReason: 'client error calling function',
+            error: err
+          };
+          endConversation = true;
+        }
+      }
+    }
+
+    /* check whether we should notify on this event */
+    if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
+      this.parent.sendEventHook(evt)
+        .catch((err) => this.logger.info({err},
+          'TaskLlmElevenlabs_S2S:_onServerEvent - error sending event hook'));
+    }
+
+    if (endConversation) {
+      this.logger.info({results: this.results},
+        'TaskLlmElevenlabs_S2S:_onServerEvent - ending conversation due to error');
+      this.notifyTaskDone();
+    }
+  }
+
+  _populateEvents(events) {
+    if (events.includes('all')) {
+      /* work by excluding specific events */
+      const exclude = events
+        .filter((evt) => evt.startsWith('-'))
+        .map((evt) => evt.slice(1));
+      if (exclude.length === 0) this.includeEvents = elevenlabs_server_events;
+      else this.excludeEvents = expandWildcards(exclude);
+    }
+    else {
+      /* work by including specific events */
+      const include = events
+        .filter((evt) => !evt.startsWith('-'));
+      this.includeEvents = expandWildcards(include);
+    }
+
+    this.logger.debug({
+      includeEvents: this.includeEvents,
+      excludeEvents: this.excludeEvents
+    }, 'TaskLlmElevenlabs_S2S:_populateEvents');
+  }
+}
+
+module.exports = TaskLlmElevenlabs_S2S;

lib/tasks/llm/llms/google_s2s.js

@@ -0,0 +1,319 @@
+const Task = require('../../task');
+const TaskName = 'Llm_Google_s2s';
+const {LlmEvents_Google} = require('../../../utils/constants');
+const ClientEvent = 'client.event';
+const SessionDelete = 'session.delete';
+
+const google_server_events = [
+  'error',
+  'session.created',
+  'session.updated',
+];
+
+const expandWildcards = (events) => {
+  const expandedEvents = [];
+
+  events.forEach((evt) => {
+    if (evt.endsWith('.*')) {
+      const prefix = evt.slice(0, -2); // Remove the wildcard ".*"
+      const matchingEvents = google_server_events.filter((e) => e.startsWith(prefix));
+      expandedEvents.push(...matchingEvents);
+    } else {
+      expandedEvents.push(evt);
+    }
+  });
+
+  return expandedEvents;
+};
+
+class TaskLlmGoogle_S2S extends Task {
+  constructor(logger, opts, parentTask) {
+    super(logger, opts, parentTask);
+    this.parent = parentTask;
+
+    this.vendor = this.parent.vendor;
+    this.vendor = this.parent.vendor;
+    this.model = this.parent.model || 'models/gemini-2.0-flash-live-001';
+    this.auth = this.parent.auth;
+    this.connectionOptions = this.parent.connectOptions;
+
+    const {apiKey} = this.auth || {};
+    if (!apiKey) throw new Error('auth.apiKey is required for Google S2S');
+
+    this.apiKey = apiKey;
+
+    this.actionHook = this.data.actionHook;
+    this.eventHook = this.data.eventHook;
+    this.toolHook = this.data.toolHook;
+
+    const {setup} = this.data.llmOptions;
+
+    if (typeof setup !== 'object') {
+      throw new Error('llmOptions with an initial setup is required for Google S2S');
+    }
+    this.setup = {
+      ...setup,
+      model: this.model,
+      // make sure output is always audio
+      generationConfig: {
+        ...(setup.generationConfig || {}),
+        responseModalities: 'audio'
+      }
+    };
+
+    this.results = {
+      completionReason: 'normal conversation end'
+    };
+
+    /**
+     * only one of these will have items,
+     * if includeEvents, then these are the events to include
+     * if excludeEvents, then these are the events to exclude
+     */
+    this.includeEvents = [];
+    this.excludeEvents = [];
+
+    /* default to all events if user did not specify */
+    this._populateEvents(this.data.events || google_server_events);
+
+    this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
+    this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
+  }
+
+  get name() { return TaskName; }
+
+  async _api(ep, args) {
+    const res = await ep.api('uuid_google_s2s', `^^|${args.join('|')}`);
+    if (!res.body?.startsWith('+OK')) {
+      throw new Error({args}, `Error calling uuid_openai_s2s: ${res.body}`);
+    }
+  }
+
+  async exec(cs, {ep}) {
+    await super.exec(cs);
+
+    await this._startListening(cs, ep);
+
+    await this.awaitTaskDone();
+
+    /* note: the parent llm verb started the span, which is why this is necessary */
+    await this.parent.performAction(this.results);
+
+    this._unregisterHandlers();
+  }
+
+  async kill(cs) {
+    super.kill(cs);
+
+    this._api(cs.ep, [cs.ep.uuid, SessionDelete])
+      .catch((err) => this.logger.info({err}, 'TaskLlmGoogle_S2S:kill - error deleting session'));
+
+    this.notifyTaskDone();
+  }
+
+  _populateEvents(events) {
+    if (events.includes('all')) {
+      /* work by excluding specific events */
+      const exclude = events
+        .filter((evt) => evt.startsWith('-'))
+        .map((evt) => evt.slice(1));
+      if (exclude.length === 0) this.includeEvents = google_server_events;
+      else this.excludeEvents = expandWildcards(exclude);
+    }
+    else {
+      /* work by including specific events */
+      const include = events
+        .filter((evt) => !evt.startsWith('-'));
+      this.includeEvents = expandWildcards(include);
+    }
+
+    this.logger.debug({
+      includeEvents: this.includeEvents,
+      excludeEvents: this.excludeEvents
+    }, 'TaskLlmGoogle_S2S:_populateEvents');
+  }
+
+  async _startListening(cs, ep) {
+    this._registerHandlers(ep);
+
+    try {
+      const args = [ep.uuid, 'session.create', this.apiKey];
+      await this._api(ep, args);
+    } catch (err) {
+      this.logger.error({err}, 'TaskLlmGoogle_S2S:_startListening');
+      this.notifyTaskDone();
+    }
+  }
+
+  async _sendClientEvent(ep, obj) {
+    let ok = true;
+    this.logger.debug({obj}, 'TaskLlmGoogle_S2S:_sendClientEvent');
+    try {
+      const args = [ep.uuid, ClientEvent, JSON.stringify(obj)];
+      await this._api(ep, args);
+    } catch (err) {
+      ok = false;
+      this.logger.error({err}, 'TaskLlmGoogle_S2S:_sendClientEvent - Error');
+    }
+    return ok;
+  }
+
+  async _sendInitialMessage(ep) {
+    const setup = this.setup;
+    const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+    if (mcpTools && mcpTools.length > 0) {
+      const convertedTools = [
+        {
+          functionDeclarations: mcpTools.map((tool) => {
+            if (tool.inputSchema) {
+              delete tool.inputSchema.additionalProperties;
+              delete tool.inputSchema['$schema'];
+            }
+            return {
+              name: tool.name,
+              description: tool.description,
+              parameters: tool.inputSchema,
+            };
+          })
+        }
+      ];
+      // merge with any existing tools
+      setup.tools = [...convertedTools, ...(this.setup.tools || [])];
+    }
+    if (!await this._sendClientEvent(ep, {
+      setup,
+    })) {
+      this.logger.debug(this.setup, 'TaskLlmGoogle_S2S:_sendInitialMessage - sending session.update');
+      this.notifyTaskDone();
+    }
+  }
+
+  _registerHandlers(ep) {
+    this.addCustomEventListener(ep, LlmEvents_Google.Connect, this._onConnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Google.ConnectFailure, this._onConnectFailure.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Google.Disconnect, this._onDisconnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Google.ServerEvent, this._onServerEvent.bind(this, ep));
+  }
+
+  _unregisterHandlers() {
+    this.removeCustomEventListeners();
+  }
+
+  _onError(ep, evt) {
+    this.logger.info({evt}, 'TaskLlmGoogle_S2S:_onError');
+    this.notifyTaskDone();
+  }
+
+  _onConnect(ep) {
+    this.logger.debug('TaskLlmGoogle_S2S:_onConnect');
+    this._sendInitialMessage(ep);
+  }
+  _onConnectFailure(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmGoogle_S2S:_onConnectFailure');
+    this.results = {completionReason: 'connection failure'};
+    this.notifyTaskDone();
+  }
+  _onDisconnect(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmGoogle_S2S:_onConnectFailure');
+    this.results = {completionReason: 'disconnect from remote end'};
+    this.notifyTaskDone();
+  }
+
+  async _onServerEvent(ep, evt) {
+    let endConversation = false;
+    this.logger.debug({evt}, 'TaskLlmGoogle_S2S:_onServerEvent');
+    const {toolCall /**toolCallCancellation*/}  = evt;
+
+    if (toolCall) {
+      this.logger.debug({toolCall}, 'TaskLlmGoogle_S2S:_onServerEvent - toolCall');
+      if (!this.toolHook) {
+        this.logger.info({evt}, 'TaskLlmGoogle_S2S:_onServerEvent - no toolHook defined!');
+      }
+      else {
+        const {functionCalls} = toolCall;
+        const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+        const functionResponses = [];
+        if (mcpTools && mcpTools.length > 0) {
+          for (const functionCall of functionCalls) {
+            const {name, args, id} = functionCall;
+            const tool = mcpTools.find((tool) => tool.name === name);
+            if (tool) {
+              const response = await this.parent.mcpService.callMcpTool(name, args);
+              functionResponses.push({
+                response: {
+                  output: response,
+                },
+                id
+              });
+            }
+          }
+        }
+
+        if (functionResponses && functionResponses.length > 0) {
+          this.logger.debug({functionResponses}, 'TaskLlmGoogle_S2S:_onServerEvent - function_call - mcp result');
+          this.processToolOutput(ep, 'tool_call_id', {
+            toolResponse: {
+              functionResponses
+            }
+          });
+        } else {
+          try {
+            await this.parent.sendToolHook('function_call_id', {type: 'toolCall', functionCalls});
+          } catch (err) {
+            this.logger.info({err, evt}, 'TaskLlmGoogle_S2S - error calling function');
+            this.results = {
+              completionReason: 'client error calling function',
+              error: err
+            };
+            endConversation = true;
+          }
+        }
+      }
+    }
+
+    this._sendLlmEvent('llm_event', evt);
+
+    if (endConversation) {
+      this.logger.info({results: this.results},
+        'TaskLlmGoogle_S2S:_onServerEvent - ending conversation due to error');
+      this.notifyTaskDone();
+    }
+  }
+
+  _sendLlmEvent(type, evt) {
+    /* check whether we should notify on this event */
+    if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
+      this.parent.sendEventHook(evt)
+        .catch((err) => this.logger.info({err}, 'TaskLlmGoogle_S2S:_onServerEvent - error sending event hook'));
+    }
+  }
+
+  async processLlmUpdate(ep, data, _callSid) {
+    try {
+      this.logger.debug({data, _callSid}, 'TaskLlmGoogle_S2S:processLlmUpdate');
+
+      await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+    } catch (err) {
+      this.logger.info({err, data}, 'TaskLlmGoogle_S2S:processLlmUpdate - Error processing LLM update');
+    }
+  }
+
+  async processToolOutput(ep, tool_call_id, data) {
+    try {
+      this.logger.debug({tool_call_id, data}, 'TaskLlmGoogle_S2S:processToolOutput');
+      const {toolResponse} = data;
+
+      if (!toolResponse) {
+        this.logger.info({data},
+          'TaskLlmGoogle_S2S:processToolOutput - invalid tool output, must be functionResponses');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+      }
+    } catch (err) {
+      this.logger.info({err, data}, 'TaskLlmGoogle_S2S:processToolOutput - Error processing tool output');
+    }
+  }
+}
+
+module.exports = TaskLlmGoogle_S2S;

lib/tasks/llm/llms/openai_s2s.js

@@ -59,7 +59,7 @@ class TaskLlmOpenAI_S2S extends Task {
     this.parent = parentTask;
 
     this.vendor = this.parent.vendor;
-    this.model = this.parent.model;
+    this.model = this.parent.model || 'gpt-4o-realtime-preview-2024-12-17';
     this.auth = this.parent.auth;
     this.connectionOptions = this.parent.connectOptions;
 
@@ -120,9 +120,9 @@ class TaskLlmOpenAI_S2S extends Task {
 
     switch (this.vendor) {
       case 'openai':
-        return 'v1/realtime?model=gpt-4o-realtime-preview-2024-10-01';
+        return `v1/realtime?model=${this.model}`;
       case 'microsoft':
-        return 'openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-1001&';
+        return `openai/realtime?api-version=2024-10-01-preview&deployment=${this.model}`;
     }
   }
 
@@ -235,6 +235,23 @@ class TaskLlmOpenAI_S2S extends Task {
 
     /* send immediate session.update if present */
     else if (this.session_update) {
+      if (this.parent.isMcpEnabled) {
+        this.logger.debug('TaskLlmOpenAI_S2S:_sendInitialMessage - mcp enabled');
+        const tools = await this.parent.mcpService.getAvailableMcpTools();
+        if (tools && tools.length > 0 && this.session_update) {
+          const convertedTools = tools.map((tool) => ({
+            name: tool.name,
+            type: 'function',
+            description: tool.description,
+            parameters: tool.inputSchema
+          }));
+
+          this.session_update.tools = [
+            ...convertedTools,
+            ...(this.session_update.tools || [])
+          ];
+        }
+      }
       obj = {type: 'session.update', session: this.session_update};
       this.logger.debug({obj}, 'TaskLlmOpenAI_S2S:_sendInitialMessage - sending session.update');
       if (!await this._sendClientEvent(ep, obj)) {
@@ -299,13 +316,37 @@ class TaskLlmOpenAI_S2S extends Task {
     /* tool calls */
     else if (type === 'response.output_item.done' && evt.item?.type === 'function_call') {
       this.logger.debug({evt}, 'TaskLlmOpenAI_S2S:_onServerEvent - function_call');
-      if (!this.toolHook) {
+      const {name, call_id} = evt.item;
+      const args = JSON.parse(evt.item.arguments);
+
+      const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+      if (mcpTools.some((tool) => tool.name === name)) {
+        this.logger.debug({call_id, name, args}, 'TaskLlmOpenAI_S2S:_onServerEvent - calling mcp tool');
+        try {
+          const res = await this.parent.mcpService.callMcpTool(name, args);
+          this.logger.debug({res}, 'TaskLlmOpenAI_S2S:_onServerEvent - function_call - mcp result');
+          this.processToolOutput(ep, call_id, {
+            type: 'conversation.item.create',
+            item: {
+              type: 'function_call_output',
+              call_id,
+              output: res.content[0]?.text || 'There is no output from the function call',
+            }
+          });
+          return;
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmOpenAI_S2S - error calling function');
+          this.results = {
+            completionReason: 'client error calling mcp function',
+            error: err
+          };
+          endConversation = true;
+        }
+      }
+      else if (!this.toolHook) {
         this.logger.warn({evt}, 'TaskLlmOpenAI_S2S:_onServerEvent - no toolHook defined!');
       }
       else {
-        const {name, call_id} = evt.item;
-        const args = JSON.parse(evt.item.arguments);
-
         try {
           await this.parent.sendToolHook(call_id, {name, args});
         } catch (err) {

lib/tasks/llm/llms/ultravox_s2s.js

@@ -0,0 +1,344 @@
+const Task = require('../../task');
+const TaskName = 'Llm_Ultravox_s2s';
+const {request} = require('undici');
+const {LlmEvents_Ultravox} = require('../../../utils/constants');
+
+const ultravox_server_events = [
+  'createCall',
+  'pong',
+  'state',
+  'transcript',
+  'conversationText',
+  'clientToolInvocation',
+  'playbackClearBuffer',
+];
+
+const ClientEvent = 'client.event';
+
+const expandWildcards = (events) => {
+  // no-op for deepgram
+  return events;
+};
+
+const SessionDelete = 'session.delete';
+
+class TaskLlmUltravox_S2S extends Task {
+  constructor(logger, opts, parentTask) {
+    super(logger, opts, parentTask);
+    this.parent = parentTask;
+
+    this.vendor = this.parent.vendor;
+    this.model = this.parent.model || 'fixie-ai/ultravox';
+    this.auth = this.parent.auth;
+    this.connectionOptions = this.parent.connectOptions;
+
+    const {apiKey} = this.auth || {};
+    if (!apiKey) throw new Error('auth.apiKey is required for Vendor: Ultravox');
+    this.apiKey = apiKey;
+    this.actionHook = this.data.actionHook;
+    this.eventHook = this.data.eventHook;
+    this.toolHook = this.data.toolHook;
+
+    this.results = {
+      completionReason: 'normal conversation end'
+    };
+
+    /**
+     * only one of these will have items,
+     * if includeEvents, then these are the events to include
+     * if excludeEvents, then these are the events to exclude
+     */
+    this.includeEvents = [];
+    this.excludeEvents = [];
+
+    /* default to all events if user did not specify */
+    this._populateEvents(this.data.events || ultravox_server_events);
+
+    this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
+    this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
+  }
+
+  get name() { return TaskName; }
+
+  async _api(ep, args) {
+    const res = await ep.api('uuid_ultravox_s2s', `^^|${args.join('|')}`);
+    if (!res.body?.startsWith('+OK')) {
+      throw new Error(`Error calling uuid_ultravox_s2s: ${JSON.stringify(res.body)}`);
+    }
+  }
+
+  /**
+   * Converts a JSON Schema to the dynamic parameters format used in the Ultravox API
+   * @param {Object} jsonSchema - A JSON Schema object defining parameters
+   * @param {string} locationDefault - Default location value for parameters (default: 'PARAMETER_LOCATION_BODY')
+   * @returns {Array} Array of dynamic parameters objects
+   */
+  transformSchemaToParameters(jsonSchema, locationDefault = 'PARAMETER_LOCATION_BODY') {
+    if (jsonSchema.properties) {
+      const required = jsonSchema.required || [];
+
+      return Object.entries(jsonSchema.properties).map(([name]) => {
+        return {
+          name,
+          location: locationDefault,
+          required: required.includes(name)
+        };
+      });
+    }
+
+    return [];
+  }
+
+  async createCall() {
+    const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+    if (mcpTools && mcpTools.length > 0) {
+      const convertedTools = mcpTools.map((tool) => {
+        return {
+          temporaryTool: {
+            modelToolName: tool.name,
+            description: tool.description,
+            dynamicParameters: this.transformSchemaToParameters(tool.inputSchema),
+            // use client tool that ultravox call tool via freeswitch module.
+            client: {}
+          }
+        };
+      }
+      );
+      // merge with any existing tools
+      this.data.llmOptions.selectedTools = [
+        ...convertedTools,
+        ...(this.data.llmOptions.selectedTools || [])
+      ];
+    }
+
+    const payload = {
+      ...this.data.llmOptions,
+      model: this.model,
+      medium: {
+        ...(this.data.llmOptions.medium || {}),
+        serverWebSocket: {
+          inputSampleRate: 8000,
+          outputSampleRate: 8000,
+        }
+      }
+    };
+    const {statusCode, body} = await request('https://api.ultravox.ai/api/calls', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'X-API-Key': this.apiKey
+      },
+      body: JSON.stringify(payload)
+    });
+    const data = await body.json();
+    if (statusCode !== 201 || !data?.joinUrl) {
+      this.logger.info({statusCode, data}, 'Ultravox Error registering call');
+      throw new Error(`Ultravox Error registering call:${statusCode} - ${data.detail}`);
+    }
+    this.logger.debug({joinUrl: data.joinUrl}, 'Ultravox Call registered');
+    return data;
+  }
+
+  _unregisterHandlers() {
+    this.removeCustomEventListeners();
+  }
+
+  _registerHandlers(ep) {
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.Connect, this._onConnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.ConnectFailure, this._onConnectFailure.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.Disconnect, this._onDisconnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_Ultravox.ServerEvent, this._onServerEvent.bind(this, ep));
+  }
+
+  async _startListening(cs, ep) {
+    this._registerHandlers(ep);
+
+    try {
+      const data = await this.createCall();
+      const {joinUrl} = data;
+      // split the joinUrl into host and path
+      const {host, pathname, search} = new URL(joinUrl);
+      const args = [ep.uuid, 'session.create', host, pathname + search];
+      await this._api(ep, args);
+      // Notify the application that the session has been created with detail information
+      this._sendLlmEvent('createCall', {
+        type: 'createCall',
+        ...data
+      });
+    } catch (err) {
+      this.logger.info({err}, 'TaskLlmUltraVox_S2S:_startListening - Error sending createCall');
+      this.results = {completionReason: `connection failure - ${err}`};
+      this.notifyTaskDone();
+    }
+  }
+
+  async exec(cs, {ep}) {
+    await super.exec(cs);
+
+    await this._startListening(cs, ep);
+
+    await this.awaitTaskDone();
+
+    /* note: the parent llm verb started the span, which is why this is necessary */
+    await this.parent.performAction(this.results);
+
+    this._unregisterHandlers();
+  }
+
+  async kill(cs) {
+    super.kill(cs);
+
+    this._api(cs.ep, [cs.ep.uuid, SessionDelete])
+      .catch((err) => this.logger.info({err}, 'TaskLlmUltravox_S2S:kill - error deleting session'));
+
+    this.notifyTaskDone();
+  }
+
+  _onConnect(ep) {
+    this.logger.info('TaskLlmUltravox_S2S:_onConnect');
+  }
+  _onConnectFailure(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmUltravox_S2S:_onConnectFailure');
+    this.results = {completionReason: 'connection failure'};
+    this.notifyTaskDone();
+  }
+  _onDisconnect(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmUltravox_S2S:_onConnectFailure');
+    this.results = {completionReason: 'disconnect from remote end'};
+    this.notifyTaskDone();
+  }
+
+  async _onServerEvent(_ep, evt) {
+    let endConversation = false;
+    const type = evt.type;
+    this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent');
+
+    /* server errors of some sort */
+    if (type === 'error') {
+      endConversation = true;
+      this.results = {
+        completionReason: 'server error',
+        error: evt.error
+      };
+    }
+
+    /* tool calls */
+    else if (type === 'client_tool_invocation') {
+      this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent - function_call');
+      const {toolName: name, invocationId: call_id, parameters: args} = evt;
+
+      const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+      if (mcpTools.some((tool) => tool.name === name)) {
+        this.logger.debug({
+          name,
+          input: args
+        }, 'TaskLlmUltravox_S2S:_onServerEvent - function_call - mcp tool');
+        try {
+          const res = await this.parent.mcpService.callMcpTool(name, args);
+          this.logger.debug({res}, 'TaskLlmUltravox_S2S:_onServerEvent - function_call - mcp result');
+          this.processToolOutput(_ep, call_id, {
+            type: 'client_tool_result',
+            invocation_id: call_id,
+            result: res.content
+          });
+          return;
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmUltravox_S2S - error calling mcp tool');
+          this.results = {
+            completionReason: 'client error calling mcp function',
+            error: err
+          };
+          endConversation = true;
+        }
+      } else if (!this.toolHook) {
+        this.logger.info({evt}, 'TaskLlmUltravox_S2S:_onServerEvent - no toolHook defined!');
+      }
+      else {
+        try {
+          await this.parent.sendToolHook(call_id, {name, args});
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmUltravox_S2S - error calling function');
+          this.results = {
+            completionReason: 'client error calling function',
+            error: err
+          };
+          endConversation = true;
+        }
+      }
+    }
+
+    this._sendLlmEvent(type, evt);
+
+    if (endConversation) {
+      this.logger.info({results: this.results},
+        'TaskLlmUltravox_S2S:_onServerEvent - ending conversation due to error');
+      this.notifyTaskDone();
+    }
+  }
+
+  _sendLlmEvent(type, evt) {
+    /* check whether we should notify on this event */
+    if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
+      this.parent.sendEventHook(evt)
+        .catch((err) => this.logger.info({err}, 'TaskLlmUltravox_S2S:_onServerEvent - error sending event hook'));
+    }
+  }
+
+  async processLlmUpdate(ep, data, _callSid) {
+    try {
+      this.logger.debug({data, _callSid}, 'TaskLlmUltravox_S2S:processLlmUpdate');
+
+      if (!data.type || ![
+        'input_text_message'
+      ].includes(data.type)) {
+        this.logger.info({data},
+          'TaskLlmUltravox_S2S:processLlmUpdate - invalid mid-call request, only input_text_message supported');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+      }
+    } catch (err) {
+      this.logger.info({err, data}, 'TaskLlmUltravox_S2S:processLlmUpdate - Error processing LLM update');
+    }
+  }
+
+  async processToolOutput(ep, tool_call_id, data) {
+    try {
+      this.logger.debug({tool_call_id, data}, 'TaskLlmUltravox_S2S:processToolOutput');
+
+      if (!data.type || data.type !== 'client_tool_result') {
+        this.logger.info({data},
+          'TaskLlmUltravox_S2S:processToolOutput - invalid tool output, must be client_tool_result');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+      }
+    } catch (err) {
+      this.logger.info({err, data}, 'TaskLlmUltravox_S2S:processToolOutput - Error processing tool output');
+    }
+  }
+
+  _populateEvents(events) {
+    if (events.includes('all')) {
+      /* work by excluding specific events */
+      const exclude = events
+        .filter((evt) => evt.startsWith('-'))
+        .map((evt) => evt.slice(1));
+      if (exclude.length === 0) this.includeEvents = ultravox_server_events;
+      else this.excludeEvents = expandWildcards(exclude);
+    }
+    else {
+      /* work by including specific events */
+      const include = events
+        .filter((evt) => !evt.startsWith('-'));
+      this.includeEvents = expandWildcards(include);
+    }
+
+    this.logger.debug({
+      includeEvents: this.includeEvents,
+      excludeEvents: this.excludeEvents
+    }, 'TaskLlmUltravox_S2S:_populateEvents');
+  }
+}
+
+module.exports = TaskLlmUltravox_S2S;

lib/tasks/llm/llms/voice_agent_s2s.js

@@ -0,0 +1,352 @@
+const Task = require('../../task');
+const TaskName = 'Llm_VoiceAgent_s2s';
+const {LlmEvents_VoiceAgent} = require('../../../utils/constants');
+const ClientEvent = 'client.event';
+const SessionDelete = 'session.delete';
+
+const va_server_events = [
+  'Error',
+  'Welcome',
+  'SettingsApplied',
+  'ConversationText',
+  'UserStartedSpeaking',
+  'EndOfThought',
+  'AgentThinking',
+  'FunctionCallRequest',
+  'FunctionCalling',
+  'AgentStartedSpeaking',
+  'AgentAudioDone',
+];
+
+const expandWildcards = (events) => {
+  // no-op for deepgram
+  return events;
+};
+
+class TaskLlmVoiceAgent_S2S extends Task {
+  constructor(logger, opts, parentTask) {
+    super(logger, opts, parentTask);
+    this.parent = parentTask;
+
+    this.vendor = this.parent.vendor;
+    this.model = this.parent.model || 'voice-agent';
+    this.auth = this.parent.auth;
+    this.connectionOptions = this.parent.connectOptions;
+
+    const {apiKey} = this.auth || {};
+    if (!apiKey) throw new Error('auth.apiKey is required for VoiceAgent S2S');
+
+    this.apiKey = apiKey;
+    this.authType = 'bearer';
+    this.actionHook = this.data.actionHook;
+    this.eventHook = this.data.eventHook;
+    this.toolHook = this.data.toolHook;
+    const {Settings} = this.data.llmOptions;
+
+    if (typeof Settings !== 'object') {
+      throw new Error('llmOptions with an initial Settings is required for VoiceAgent S2S');
+    }
+
+    // eslint-disable-next-line no-unused-vars
+    const {audio, ...rest} = Settings;
+    const cfg = this.Settings = rest;
+
+    if (!cfg.agent) throw new Error('llmOptions.Settings.agent is required for VoiceAgent S2S');
+    if (!cfg.agent.think) {
+      throw new Error('llmOptions.Settings.agent.think is required for VoiceAgent S2S');
+    }
+    if (!cfg.agent.think.provider?.model) {
+      throw new Error('llmOptions.Settings.agent.think.provider.model is required for VoiceAgent S2S');
+    }
+    if (!cfg.agent.think.provider?.type) {
+      throw new Error('llmOptions.Settings.agent.think.provider.type is required for VoiceAgent S2S');
+    }
+
+    this.results = {
+      completionReason: 'normal conversation end'
+    };
+
+    /**
+     * only one of these will have items,
+     * if includeEvents, then these are the events to include
+     * if excludeEvents, then these are the events to exclude
+     */
+    this.includeEvents = [];
+    this.excludeEvents = [];
+
+    /* default to all events if user did not specify */
+    this._populateEvents(this.data.events || va_server_events);
+
+    this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
+    this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
+  }
+
+  get name() { return TaskName; }
+
+  get host() {
+    const {host} = this.connectionOptions || {};
+    return host || 'agent.deepgram.com';
+  }
+
+  get path() {
+    const {path} = this.connectionOptions || {};
+    if (path) return path;
+
+    return '/v1/agent/converse';
+  }
+
+  async _api(ep, args) {
+    const res = await ep.api('uuid_voice_agent_s2s', `^^|${args.join('|')}`);
+    if (!res.body?.startsWith('+OK')) {
+      throw new Error(`Error calling uuid_voice_agent_s2s: ${JSON.stringify(res.body)}`);
+    }
+  }
+
+  async exec(cs, {ep}) {
+    await super.exec(cs);
+
+    await this._startListening(cs, ep);
+
+    await this.awaitTaskDone();
+
+    /* note: the parent llm verb started the span, which is why this is necessary */
+    await this.parent.performAction(this.results);
+
+    this._unregisterHandlers();
+  }
+
+  async kill(cs) {
+    super.kill(cs);
+
+    this._api(cs.ep, [cs.ep.uuid, SessionDelete])
+      .catch((err) => this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:kill - error deleting session'));
+
+    this.notifyTaskDone();
+  }
+
+  /**
+   * Send function call response to the VoiceAgent server
+   */
+  async processToolOutput(ep, tool_call_id, data) {
+    try {
+      const {data:response} = data;
+      this.logger.debug({tool_call_id, response}, 'TaskLlmVoiceAgent_S2S:processToolOutput');
+
+      if (!response.type || response.type !== 'FunctionCallResponse') {
+        this.logger.info({response},
+          'TaskLlmVoiceAgent_S2S:processToolOutput - invalid tool output, must be FunctionCallResponse');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(response)]);
+      }
+    } catch (err) {
+      this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:processToolOutput');
+    }
+  }
+
+  /**
+   * Send a session.update to the VoiceAgent server
+   * Note: creating and deleting conversation items also supported as well as interrupting the assistant
+   */
+  async processLlmUpdate(ep, data, _callSid) {
+    try {
+      this.logger.debug({data, _callSid}, 'TaskLlmVoiceAgent_S2S:processLlmUpdate');
+
+      if (!data.type || ![
+        'UpdateInstructions',
+        'UpdateSpeak',
+        'InjectAgentMessage',
+      ].includes(data.type)) {
+        this.logger.info({data}, 'TaskLlmVoiceAgent_S2S:processLlmUpdate - invalid mid-call request');
+      }
+      else {
+        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
+      }
+    } catch (err) {
+      this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:processLlmUpdate');
+    }
+  }
+
+  async _startListening(cs, ep) {
+    this._registerHandlers(ep);
+
+    try {
+      const args = [ep.uuid, 'session.create', this.host, this.path, this.authType, this.apiKey];
+      await this._api(ep, args);
+    } catch (err) {
+      this.logger.error({err}, `TaskLlmVoiceAgent_S2S:_startListening: ${JSON.stringify(err)}`);
+      this.notifyTaskDone();
+    }
+  }
+
+  async _sendClientEvent(ep, obj) {
+    let ok = true;
+    this.logger.debug({obj}, 'TaskLlmVoiceAgent_S2S:_sendClientEvent');
+    try {
+      const args = [ep.uuid, ClientEvent, JSON.stringify(obj)];
+      await this._api(ep, args);
+    } catch (err) {
+      ok = false;
+      this.logger.error({err}, 'TaskLlmVoiceAgent_S2S:_sendClientEvent - Error');
+    }
+    return ok;
+  }
+
+  async _sendInitialMessage(ep) {
+    const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+    if (mcpTools && mcpTools.length > 0 && this.Settings.agent?.think) {
+      const convertedTools = mcpTools.map((tool) => ({
+        name: tool.name,
+        description: tool.description,
+        parameters: tool.inputSchema
+      }));
+
+      this.Settings.agent.think.functions = [
+        ...convertedTools,
+        ...(this.Settings.agent.think?.functions || [])
+      ];
+    }
+    if (!await this._sendClientEvent(ep, this.Settings)) {
+      this.notifyTaskDone();
+    }
+  }
+
+  _registerHandlers(ep) {
+    this.addCustomEventListener(ep, LlmEvents_VoiceAgent.Connect, this._onConnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_VoiceAgent.ConnectFailure, this._onConnectFailure.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_VoiceAgent.Disconnect, this._onDisconnect.bind(this, ep));
+    this.addCustomEventListener(ep, LlmEvents_VoiceAgent.ServerEvent, this._onServerEvent.bind(this, ep));
+  }
+
+  _unregisterHandlers() {
+    this.removeCustomEventListeners();
+  }
+
+  _onError(_ep, evt) {
+    this.logger.info({evt}, 'TaskLlmVoiceAgent_S2S:_onError');
+    this.notifyTaskDone();
+  }
+
+  _onConnect(ep) {
+    this.logger.debug('TaskLlmVoiceAgent_S2S:_onConnect');
+    this._sendInitialMessage(ep);
+  }
+  _onConnectFailure(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmVoiceAgent_S2S:_onConnectFailure');
+    this.results = {completionReason: 'connection failure'};
+    this.notifyTaskDone();
+  }
+  _onDisconnect(_ep, evt) {
+    this.logger.info(evt, 'TaskLlmVoiceAgent_S2S:_onConnectFailure');
+    this.results = {completionReason: 'disconnect from remote end'};
+    this.notifyTaskDone();
+  }
+  async _onServerEvent(_ep, evt) {
+    let endConversation = false;
+    const type = evt.type;
+    this.logger.info({evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent');
+
+    /* check for failures, such as rate limit exceeded, that should terminate the conversation */
+    if (type === 'response.done' && evt.response.status === 'failed') {
+      endConversation = true;
+      this.results = {
+        completionReason: 'server failure',
+        error: evt.response.status_details?.error
+      };
+    }
+
+    /* server errors of some sort */
+    else if (type === 'error') {
+      endConversation = true;
+      this.results = {
+        completionReason: 'server error',
+        error: evt.error
+      };
+    }
+
+    /* tool calls */
+    else if (type === 'FunctionCallRequest') {
+      this.logger.debug({evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - function_call');
+
+      const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+      if (!this.toolHook && mcpTools.length === 0) {
+        this.logger.warn({evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - no toolHook defined!');
+      } else {
+        const {functions} = evt;
+        const handledFunctions = [];
+
+        try {
+          if (mcpTools && mcpTools.length > 0) {
+            for (const func of functions) {
+              const {name, arguments: args, id} = func;
+              const tool = mcpTools.find((tool) => tool.name === name);
+              if (tool) {
+                handledFunctions.push(name);
+                const response = await this.parent.mcpService.callMcpTool(name, JSON.parse(args));
+                this.logger.debug({response}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - function_call - mcp result');
+                this.processToolOutput(_ep, id, {
+                  data: {
+                    type: 'FunctionCallResponse',
+                    id,
+                    name,
+                    content: response.length > 0 ? response[0].text : 'There is no output from the function call'
+                  }
+                });
+              }
+            }
+          }
+
+          for (const func of functions) {
+            const {name, arguments: args, id} = func;
+            if (!handledFunctions.includes(name)) {
+              await this.parent.sendToolHook(id, {name, args: JSON.parse(args)});
+            }
+          }
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - error calling function');
+          this.results = {
+            completionReason: 'client error calling function',
+            error: err
+          };
+          endConversation = true;
+        }
+      }
+    }
+
+    /* check whether we should notify on this event */
+    if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
+      this.parent.sendEventHook(evt)
+        .catch((err) => this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - error sending event hook'));
+    }
+
+    if (endConversation) {
+      this.logger.info({results: this.results},
+        'TaskLlmVoiceAgent_S2S:_onServerEvent - ending conversation due to error');
+      this.notifyTaskDone();
+    }
+  }
+
+  _populateEvents(events) {
+    if (events.includes('all')) {
+      /* work by excluding specific events */
+      const exclude = events
+        .filter((evt) => evt.startsWith('-'))
+        .map((evt) => evt.slice(1));
+      if (exclude.length === 0) this.includeEvents = va_server_events;
+      else this.excludeEvents = expandWildcards(exclude);
+    }
+    else {
+      /* work by including specific events */
+      const include = events
+        .filter((evt) => !evt.startsWith('-'));
+      this.includeEvents = expandWildcards(include);
+    }
+
+    this.logger.debug({
+      includeEvents: this.includeEvents,
+      excludeEvents: this.excludeEvents
+    }, 'TaskLlmVoiceAgent_S2S:_populateEvents');
+  }
+}
+
+module.exports = TaskLlmVoiceAgent_S2S;

lib/tasks/make_task.js

@@ -84,6 +84,7 @@ function makeTask(logger, obj, parent) {
       const TaskTranscribe = require('./transcribe');
       return new TaskTranscribe(logger, data, parent);
     case TaskName.Listen:
+    case TaskName.Stream:
       const TaskListen = require('./listen');
       return new TaskListen(logger, data, parent);
     case TaskName.Redirect:

lib/tasks/message.js

@@ -1,7 +1,7 @@
 const Task = require('./task');
 const {TaskName, TaskPreconditions} = require('../utils/constants');
 const bent = require('bent');
-const uuidv4 = require('uuid-random');
+const crypto = require('crypto');
 const {K8S} = require('../config');
 class TaskMessage extends Task {
   constructor(logger, opts) {
@@ -9,7 +9,7 @@ class TaskMessage extends Task {
     this.preconditions = TaskPreconditions.None;
 
     this.payload = {
-      message_sid: this.data.message_sid || uuidv4(),
+      message_sid: this.data.message_sid || crypto.randomUUID(),
       carrier: this.data.carrier,
       to: this.data.to,
       from: this.data.from,

lib/tasks/play.js

@@ -1,6 +1,6 @@
 const Task = require('./task');
 const {TaskName, TaskPreconditions} = require('../utils/constants');
-
+const { PlayFileNotFoundError } = require('../utils/error');
 class TaskPlay extends Task {
   constructor(logger, opts) {
     super(logger, opts);
@@ -26,6 +26,7 @@ class TaskPlay extends Task {
     let playbackSeconds = 0;
     let playbackMilliseconds = 0;
     let completed = !(this.timeoutSecs > 0 || this.loop);
+    cs.playingAudio = true;
     if (this.timeoutSecs > 0) {
       timeout = setTimeout(async() => {
         completed = true;
@@ -39,6 +40,22 @@ class TaskPlay extends Task {
     try {
       this.notifyStatus({event: 'start-playback'});
       while (!this.killed && (this.loop === 'forever' || this.loop--) && this.ep.connected) {
+        /* Listen for playback-start event and set up a one-time listener for uuid_break
+         * that will kill the audio playback if the taskIds match. This ensures that
+         * we only kill the currently playing audio and not audio from other tasks.
+         * As we are using stickyEventEmitter, even if the event is emitted before the listener is registered,
+         * the listener will receive the most recent event.
+         */
+        ep.once('playback-start', (evt) => {
+          this.logger.debug({evt}, 'Play got playback-start');
+          this.cs.stickyEventEmitter.once('uuid_break', (t) => {
+            if (t?.taskId === this.taskId) {
+              this.logger.debug(`Play got kill-playback, executing uuid_break, taskId: ${t?.taskId}`);
+              this.ep.api('uuid_break', this.ep.uuid).catch((err) => this.logger.info(err, 'Error killing audio'));
+              this.notifyStatus({event: 'kill-playback'});
+            }
+          });
+        });
         if (cs.isInConference) {
           const {memberId, confName, confUuid} = cs;
           if (Array.isArray(this.url)) {
@@ -66,23 +83,35 @@ class TaskPlay extends Task {
         }
       }
     } catch (err) {
-      if (timeout) clearTimeout(timeout);
-      this.logger.info(err, `TaskPlay:exec - error playing ${this.url}`);
+      this.logger.info(`TaskPlay:exec - error playing ${this.url}: ${err.message}`);
+      this.playComplete = true;
+      if (err.message === 'File Not Found') {
+        const {writeAlerts, AlertType} = cs.srf.locals;
+        await this.performAction({status: 'fail', reason: 'playFailed'}, !(this.parentTask || cs.isConfirmCallSession));
+        this.emit('playDone');
+        writeAlerts({
+          account_sid: cs.accountSid,
+          alert_type: AlertType.PLAY_FILENOTFOUND,
+          url: this.url,
+          target_sid: cs.callSid
+        });
+        throw new PlayFileNotFoundError(this.url);
+      }
     }
     this.emit('playDone');
   }
 
   async kill(cs) {
     super.kill(cs);
-    if (this.ep.connected && !this.playComplete) {
+    if (this.ep?.connected && !this.playComplete) {
       this.logger.debug('TaskPlay:kill - killing audio');
       if (cs.isInConference) {
         const {memberId, confName} = cs;
         this.killPlayToConfMember(this.ep, memberId, confName);
       }
       else {
-        this.notifyStatus({event: 'kill-playback'});
-        this.ep.api('uuid_break', this.ep.uuid).catch((err) => this.logger.info(err, 'Error killing audio'));
+        //this.ep.api('uuid_break', this.ep.uuid).catch((err) => this.logger.info(err, 'Error killing audio'));
+        cs.stickyEventEmitter.emit('uuid_break', this);
       }
     }
   }

lib/tasks/redirect.js

@@ -1,5 +1,8 @@
 const Task = require('./task');
 const {TaskName} = require('../utils/constants');
+const WsRequestor = require('../utils/ws-requestor');
+const URL = require('url');
+const HttpRequestor = require('../utils/http-requestor');
 
 /**
  * Redirects to a new application
@@ -13,6 +16,32 @@ class TaskRedirect extends Task {
 
   async exec(cs) {
     await super.exec(cs);
+
+    if (cs.requestor instanceof WsRequestor && cs.application.requestor._isAbsoluteUrl(this.actionHook)) {
+      this.logger.info(`Task:performAction redirecting to ${this.actionHook}, requires new ws connection`);
+      try {
+        this.cs.requestor.close();
+        const requestor = new WsRequestor(this.logger, cs.accountSid, {url: this.actionHook}, this.webhook_secret) ;
+        this.cs.application.requestor = requestor;
+      } catch (err) {
+        this.logger.info(err, `Task:performAction error redirecting to ${this.actionHook}`);
+      }
+    } else if (cs.application.requestor._isAbsoluteUrl(this.actionHook)) {
+      const baseUrl =  this.cs.application.requestor.baseUrl;
+      const newUrl = URL.parse(this.actionHook);
+      const newBaseUrl = newUrl.protocol + '//' + newUrl.host;
+      if (baseUrl != newBaseUrl) {
+        try {
+          this.logger.info(`Task:redirect updating base url to ${newBaseUrl}`);
+          const newRequestor = new HttpRequestor(this.logger, cs.accountSid, {url: this.actionHook},
+            cs.accountInfo.account.webhook_secret);
+          this.cs.requestor.removeAllListeners();
+          this.cs.application.requestor = newRequestor;
+        } catch (err) {
+          this.logger.info(err, `Task:redirect error updating base url to ${this.actionHook}`);
+        }
+      }
+    }
     await this.performAction();
   }
 }

lib/tasks/rest_dial.js

@@ -12,6 +12,7 @@ class TaskRestDial extends Task {
 
     this.from = this.data.from;
     this.callerName = this.data.callerName;
+    this.timeLimit = this.data.timeLimit;
     this.fromHost = this.data.fromHost;
     this.to = this.data.to;
     this.call_hook = this.data.call_hook;
@@ -66,6 +67,9 @@ class TaskRestDial extends Task {
     const cs = this.callSession;
     cs.setDialog(dlg);
     cs.referHook = this.referHook;
+    if (this.timeLimit) {
+      cs.startMaxCallDurationTimer(this.timeLimit);
+    }
     this.logger.debug('TaskRestDial:_onConnect - call connected');
     if (this.sipRequestWithinDialogHook) this._initSipRequestWithinDialogHandler(cs, dlg);
     try {
@@ -77,11 +81,13 @@ class TaskRestDial extends Task {
           synthesizer: {
             vendor: cs.speechSynthesisVendor,
             language: cs.speechSynthesisLanguage,
-            voice: cs.speechSynthesisVoice
+            voice: cs.speechSynthesisVoice,
+            label: cs.speechSynthesisLabel,
           },
           recognizer: {
             vendor:  cs.speechRecognizerVendor,
-            language: cs.speechRecognizerLanguage
+            language: cs.speechRecognizerLanguage,
+            label: cs.speechRecognizerLabel,
           }
         }
       };

lib/tasks/say.js

@@ -1,3 +1,4 @@
+const assert = require('assert');
 const TtsTask = require('./tts-task');
 const {TaskName, TaskPreconditions} = require('../utils/constants');
 const pollySSMLSplit = require('polly-ssml-split');
@@ -35,24 +36,40 @@ class TaskSay extends TtsTask {
     super(logger, opts, parentTask);
     this.preconditions = TaskPreconditions.Endpoint;
 
-    this.text = (Array.isArray(this.data.text) ? this.data.text : [this.data.text])
-      .map((t) => breakLengthyTextIfNeeded(this.logger, t))
-      .flat();
+    assert.ok((typeof this.data.text === 'string' || Array.isArray(this.data.text)) || this.data.stream === true,
+      'Say: either text or stream:true is required');
 
-    this.loop = this.data.loop || 1;
-    this.isHandledByPrimaryProvider = true;
+
+    if (this.data.stream === true) {
+      this._isStreamingTts = true;
+      this.closeOnStreamEmpty = this.data.closeOnStreamEmpty !== false;
+    }
+    else {
+      this._isStreamingTts = false;
+      this.text = (Array.isArray(this.data.text) ? this.data.text : [this.data.text])
+        .map((t) => breakLengthyTextIfNeeded(this.logger, t))
+        .flat();
+
+      this.loop = this.data.loop || 1;
+      this.isHandledByPrimaryProvider = true;
+    }
   }
 
   get name() { return TaskName.Say; }
 
   get summary() {
-    for (let i = 0; i < this.text.length; i++) {
-      if (this.text[i].startsWith('silence_stream')) continue;
-      return `${this.name}{text=${this.text[i].slice(0, 15)}${this.text[i].length > 15 ? '...' : ''}}`;
+    if (this.isStreamingTts) return `${this.name} streaming`;
+    else {
+      for (let i = 0; i < this.text.length; i++) {
+        if (this.text[i].startsWith('silence_stream')) continue;
+        return `${this.name}{text=${this.text[i].slice(0, 15)}${this.text[i].length > 15 ? '...' : ''}}`;
+      }
+      return `${this.name}{${this.text[0]}}`;
     }
-    return `${this.name}{${this.text[0]}}`;
   }
 
+  get isStreamingTts() { return this._isStreamingTts; }
+
   _validateURL(urlString) {
     try {
       new URL(urlString);
@@ -63,14 +80,19 @@ class TaskSay extends TtsTask {
   }
 
   async exec(cs, obj) {
+    if (this.isStreamingTts && !cs.appIsUsingWebsockets) {
+      throw new Error('Say: streaming say verb requires applications to use the websocket API');
+    }
+
     try {
-      await this.handling(cs, obj);
+      if (this.isStreamingTts) await this.handlingStreaming(cs, obj);
+      else await this.handling(cs, obj);
       this.emit('playDone');
     } catch (error) {
       if (error instanceof SpeechCredentialError) {
         // if say failed due to speech credentials, alarm is writtern and error notification is sent
         // finished this say to move to next task.
-        this.logger.info('Say failed due to SpeechCredentialError, finished!');
+        this.logger.info({error}, 'Say failed due to SpeechCredentialError, finished!');
         this.emit('playDone');
         return;
       }
@@ -78,6 +100,35 @@ class TaskSay extends TtsTask {
     }
   }
 
+  async handlingStreaming(cs, {ep}) {
+    const {vendor, language, voice, label} = this.getTtsVendorData(cs);
+    const  credentials = cs.getSpeechCredentials(vendor, 'tts', label);
+    if (!credentials) {
+      throw new SpeechCredentialError(
+        `No text-to-speech service credentials for ${vendor} with labels: ${label} have been configured`);
+    }
+    this.ep = ep;
+    try {
+
+      await this.setTtsStreamingChannelVars(vendor, language, voice, credentials, ep);
+
+      await cs.startTtsStream();
+
+      cs.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_open'})
+        .catch((err) => this.logger.info({err}, 'TaskSay:handlingStreaming - Error sending'));
+    } catch (err) {
+      this.logger.info({err}, 'TaskSay:handlingStreaming - Error setting channel vars');
+      cs.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_closed'})
+        .catch((err) => this.logger.info({err}, 'TaskSay:handlingStreaming - Error sending'));
+
+      //TODO: send tts:streaming-event with error?
+      this.notifyTaskDone();
+    }
+
+    await this.awaitTaskDone();
+    this.logger.info('TaskSay:handlingStreaming - done');
+  }
+
   async handling(cs, {ep}) {
     const {srf, accountSid:account_sid, callSid:target_sid} = cs;
     const {writeAlerts, AlertType} = srf.locals;
@@ -96,7 +147,7 @@ class TaskSay extends TtsTask {
     let voice =  this.synthesizer.voice && this.synthesizer.voice !== 'default' ?
       this.synthesizer.voice :
       cs.speechSynthesisVoice;
-    let label = this.taskInlcudeSynthesizer ? this.synthesizer.label : cs.speechSynthesisLabel;
+    let label = this.taskIncludeSynthesizer ? this.synthesizer.label : cs.speechSynthesisLabel;
 
     const fallbackVendor = this.synthesizer.fallbackVendor && this.synthesizer.fallbackVendor !== 'default' ?
       this.synthesizer.fallbackVendor :
@@ -107,7 +158,7 @@ class TaskSay extends TtsTask {
     const fallbackVoice =  this.synthesizer.fallbackVoice && this.synthesizer.fallbackVoice !== 'default' ?
       this.synthesizer.fallbackVoice :
       cs.fallbackSpeechSynthesisVoice;
-    const fallbackLabel = this.taskInlcudeSynthesizer ?
+    const fallbackLabel = this.taskIncludeSynthesizer ?
       this.synthesizer.fallbackLabel : cs.fallbackSpeechSynthesisLabel;
 
     if (cs.hasFallbackTts) {
@@ -162,7 +213,7 @@ class TaskSay extends TtsTask {
           ep.once('playback-start', (evt) => {
             this.logger.debug({evt}, 'Say got playback-start');
             if (this.otelSpan) {
-              this._addStreamingTtsAttributes(this.otelSpan, evt);
+              this._addStreamingTtsAttributes(this.otelSpan, evt, vendor);
               this.otelSpan.end();
               this.otelSpan = null;
               if (evt.variable_tts_cache_filename) {
@@ -172,7 +223,19 @@ class TaskSay extends TtsTask {
           });
           ep.once('playback-stop', (evt) => {
             this.logger.debug({evt}, 'Say got playback-stop');
-            if (evt.variable_tts_error) {
+            this.notifyStatus({event: 'stop-playback'});
+            this.notifiedPlayBackStop = true;
+            const tts_error = evt.variable_tts_error;
+            let response_code = 200;
+            // Check if any property ends with _response_code
+            for (const [key, value] of Object.entries(evt)) {
+              if (key.endsWith('_response_code')) {
+                response_code = parseInt(value, 10) || 200;
+                break;
+              }
+            }
+
+            if (tts_error) {
               writeAlerts({
                 account_sid,
                 alert_type: AlertType.TTS_FAILURE,
@@ -181,7 +244,7 @@ class TaskSay extends TtsTask {
                 target_sid
               }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
             }
-            if (evt.variable_tts_cache_filename && !this.killed) {
+            if (!tts_error && response_code < 300 && evt.variable_tts_cache_filename && !this.killed) {
               const text = parseTextFromSayString(this.text[segment]);
               addFileToCache(evt.variable_tts_cache_filename, {
                 account_sid,
@@ -189,12 +252,15 @@ class TaskSay extends TtsTask {
                 language,
                 voice,
                 engine,
-                text
+                model: this.model || this.model_id,
+                text,
+                instructions: this.instructions
               }).catch((err) => this.logger.info({err}, 'Error adding file to cache'));
             }
 
             if (this._playResolve) {
-              evt.variable_tts_error ? this._playReject(new Error(evt.variable_tts_error)) : this._playResolve();
+              (tts_error || response_code >= 300) ? this._playReject(new Error(evt.variable_tts_error)) :
+                this._playResolve();
             }
           });
           // wait for playback-stop event received to confirm if the playback is successful
@@ -240,8 +306,13 @@ class TaskSay extends TtsTask {
       if (cs.isInConference) {
         const {memberId, confName} = cs;
         this.killPlayToConfMember(this.ep, memberId, confName);
-      }
-      else {
+      } else if (this.isStreamingTts) {
+        this.logger.debug('TaskSay:kill - clearing TTS stream for streaming audio');
+        cs.clearTtsStream();
+      } else {
+        if (!this.notifiedPlayBackStop) {
+          this.notifyStatus({event: 'stop-playback'});
+        }
         this.notifyStatus({event: 'kill-playback'});
         this.ep.api('uuid_break', this.ep.uuid);
       }
@@ -253,26 +324,39 @@ class TaskSay extends TtsTask {
         this._playResolve = null;
       }
     }
+    this.notifyTaskDone();
   }
 
-  _addStreamingTtsAttributes(span, evt) {
+  _addStreamingTtsAttributes(span, evt, vendor) {
     const attrs = {'tts.cached': false};
     for (const [key, value] of Object.entries(evt)) {
       if (key.startsWith('variable_tts_')) {
         let newKey = key.substring('variable_tts_'.length)
           .replace('whisper_', 'whisper.')
+          .replace('nvidia_', 'nvidia.')
           .replace('deepgram_', 'deepgram.')
           .replace('playht_', 'playht.')
+          .replace('cartesia_', 'cartesia.')
           .replace('rimelabs_', 'rimelabs.')
           .replace('verbio_', 'verbio.')
           .replace('elevenlabs_', 'elevenlabs.');
         if (spanMapping[newKey]) newKey = spanMapping[newKey];
         attrs[newKey] = value;
+        if (key === 'variable_tts_time_to_first_byte_ms' && value) {
+          this.cs.srf.locals.stats.histogram('tts.response_time', value, [`vendor:${vendor}`]);
+        }
       }
     }
     delete attrs['cache_filename']; //no value in adding this to the span
     span.setAttributes(attrs);
   }
+
+  notifyTtsStreamIsEmpty() {
+    if (this.isStreamingTts && this.closeOnStreamEmpty) {
+      this.logger.info('TaskSay:notifyTtsStreamIsEmpty - stream is empty, killing task');
+      this.notifyTaskDone();
+    }
+  }
 }
 
 const spanMapping = {
@@ -309,6 +393,11 @@ const spanMapping = {
   'playht.name_lookup_time_ms': 'name_lookup_ms',
   'playht.connect_time_ms': 'connect_ms',
   'playht.final_response_time_ms': 'final_response_ms',
+  // Cartesia
+  'cartesia.request_id': 'cartesia.req_id',
+  'cartesia.name_lookup_time_ms': 'name_lookup_ms',
+  'cartesia.connect_time_ms': 'connect_ms',
+  'cartesia.final_response_time_ms': 'final_response_ms',
   // Rimelabs
   'rimelabs.name_lookup_time_ms': 'name_lookup_ms',
   'rimelabs.connect_time_ms': 'connect_ms',

lib/tasks/sip_decline.js

@@ -18,6 +18,11 @@ class TaskSipDecline extends Task {
     super.exec(cs);
     res.send(this.data.status, this.data.reason, {
       headers: this.headers
+    }, (err) => {
+      if (!err) {
+        // Call was successfully declined
+        cs._callReleased();
+      }
     });
     cs.emit('callStatusChange', {
       callStatus: CallStatus.Failed,

lib/tasks/sip_refer.js

@@ -12,6 +12,7 @@ class TaskSipRefer extends Task {
 
     this.referTo = this.data.referTo;
     this.referredBy = this.data.referredBy;
+    this.referredByDisplayName = this.data.referredByDisplayName;
     this.headers = this.data.headers || {};
     this.eventHook = this.data.eventHook;
   }
@@ -94,7 +95,10 @@ class TaskSipRefer extends Task {
         }
         if (status >= 200) {
           this.referSpan.setAttributes({'refer.finalNotify': status});
-          await this.performAction({refer_status: 202, final_referred_call_status: status});
+          await this.performAction({refer_status: 202, final_referred_call_status: status})
+            .catch((err) => {
+              this.logger.error(err, 'TaskSipRefer:exec - error performing action finalNotify');
+            });
           this.notifyTaskDone();
         }
       }
@@ -102,7 +106,7 @@ class TaskSipRefer extends Task {
   }
 
   _normalizeReferHeaders(cs, dlg) {
-    let {referTo, referredBy} = this;
+    let {referTo, referredBy, referredByDisplayName} = this;
 
     /* get IP address of the SBC to use as hostname if needed */
     const {host} = parseUri(dlg.remote.uri);
@@ -117,9 +121,12 @@ class TaskSipRefer extends Task {
       referredBy = cs.req?.callingNumber || dlg.local.uri;
       this.logger.info({referredBy}, 'setting referredby');
     }
+    if (!referredByDisplayName) {
+      referredByDisplayName = cs.req?.callingName;
+    }
     if (!referredBy.startsWith('<') && !referredBy.startsWith('sip') && !referredBy.startsWith('"')) {
       /* they may have only provided a phone number/user */
-      referredBy = `sip:${referredBy}@${host}`;
+      referredBy = `${referredByDisplayName ? `"${referredByDisplayName}"` : ''}<sip:${referredBy}@${host}>`;
     }
     return {referTo, referredBy};
   }

lib/tasks/stt-task.js

@@ -5,6 +5,30 @@ const { TaskPreconditions, CobaltTranscriptionEvents } = require('../utils/const
 const { SpeechCredentialError } = require('../utils/error');
 const {JAMBONES_AWS_TRANSCRIBE_USE_GRPC} = require('../config');
 
+/**
+ * "Please insert turns here: {{turns:4}}"
+// -> { processed: 'Please insert turns here: {{turns}}', turns: 4 }
+
+processTurnString("Please insert turns here: {{turns}}"));
+// -> { processed: 'Please insert turns here: {{turns}}', turns: null }
+ */
+const processTurnString = (input) => {
+  const regex = /\{\{turns(?::(\d+))?\}\}/;
+  const match = input.match(regex);
+
+  if (!match) {
+    return {
+      processed: input,
+      turns: null
+    };
+  }
+
+  const turns = match[1] ? parseInt(match[1], 10) : null;
+  const processed = input.replace(regex, '{{turns}}');
+
+  return { processed, turns };
+};
+
 class SttTask extends Task {
 
   constructor(logger, data, parentTask) {
@@ -219,7 +243,8 @@ class SttTask extends Task {
           roleArn
         });
       this.logger.debug({roleArn}, `(roleArn) got aws access token ${servedFromCache ? 'from cache' : ''}`);
-      credentials = {...credentials, accessKeyId, secretAccessKey, sessionToken};
+      // from role ARN, we will get SessionToken, but feature server use it as securityToken.
+      credentials = {...credentials, accessKeyId, secretAccessKey, securityToken: sessionToken};
     }
     else if (vendor === 'verbio' && credentials.client_id && credentials.client_secret) {
       const {access_token, servedFromCache} = await getVerbioAccessToken(credentials);
@@ -229,9 +254,13 @@ class SttTask extends Task {
     }
     else if (vendor == 'aws' && !JAMBONES_AWS_TRANSCRIBE_USE_GRPC) {
       /* get AWS access token */
-      const {accessKeyId, secretAccessKey, securityToken, region } = credentials;
+      const {speech_credential_sid, accessKeyId, secretAccessKey, securityToken, region } = credentials;
       if (!securityToken) {
-        const { servedFromCache, ...newCredentials} = await getAwsAuthToken({accessKeyId, secretAccessKey, region});
+        const { servedFromCache, ...newCredentials} = await getAwsAuthToken({
+          speech_credential_sid,
+          accessKeyId,
+          secretAccessKey,
+          region});
         this.logger.debug({newCredentials}, `got aws security token ${servedFromCache ? 'from cache' : ''}`);
         credentials = {...newCredentials, region};
       }
@@ -285,6 +314,57 @@ class SttTask extends Task {
     });
   }
 
+  formatOpenAIPrompt(cs, {prompt, hintsTemplate, conversationHistoryTemplate, hints}) {
+    let conversationHistoryPrompt, hintsPrompt;
+
+    /* generate conversation history from template */
+    if (conversationHistoryTemplate) {
+      const {processed, turns} = processTurnString(conversationHistoryTemplate);
+      this.logger.debug({processed, turns}, 'SttTask: processed conversation history template');
+      conversationHistoryPrompt = cs.getFormattedConversation(turns || 4);
+      //this.logger.debug({conversationHistoryPrompt}, 'SttTask: conversation history');
+      if (conversationHistoryPrompt) {
+        conversationHistoryPrompt = processed.replace('{{turns}}', `\n${conversationHistoryPrompt}\nuser: `);
+      }
+    }
+
+    /* generate hints from template */
+    if (hintsTemplate && Array.isArray(hints) && hints.length > 0) {
+      hintsPrompt = hintsTemplate.replace('{{hints}}', hints);
+    }
+
+    /* combine into final prompt */
+    let finalPrompt = prompt || '';
+    if (hintsPrompt) {
+      finalPrompt = `${finalPrompt}\n${hintsPrompt}`;
+    }
+    if (conversationHistoryPrompt) {
+      finalPrompt = `${finalPrompt}\n${conversationHistoryPrompt}`;
+    }
+
+    this.logger.debug({
+      finalPrompt,
+      hints,
+      hintsPrompt,
+      conversationHistoryTemplate,
+      conversationHistoryPrompt
+    }, 'SttTask: formatted OpenAI prompt');
+    return finalPrompt?.trimStart();
+  }
+
+  /* some STT engines will keep listening after a final response, so no need to restart */
+  doesVendorContinueListeningAfterFinalTranscript(vendor) {
+    return (vendor.startsWith('custom:') || [
+      'soniox',
+      'aws',
+      'microsoft',
+      'deepgram',
+      'google',
+      'speechmatics',
+      'openai',
+    ].includes(vendor));
+  }
+
   _onCompileContext(ep, key, evt) {
     const {addKey} = this.cs.srf.locals.dbHelpers;
     this.logger.debug({evt}, `received cobalt compile context event, will cache under ${key}`);

lib/tasks/task.js

@@ -1,5 +1,5 @@
 const Emitter = require('events');
-const uuidv4 = require('uuid-random');
+const crypto = require('crypto');
 const {TaskPreconditions} = require('../utils/constants');
 const { normalizeJambones } = require('@jambonz/verb-specifications');
 const WsRequestor = require('../utils/ws-requestor');
@@ -19,6 +19,7 @@ class Task extends Emitter {
     this.data = data;
     this.actionHook = this.data.actionHook;
     this.id = data.id;
+    this.taskId = crypto.randomUUID();
 
     this._killInProgress = false;
     this._completionPromise = new Promise((resolve) => this._completionResolver = resolve);
@@ -272,7 +273,7 @@ class Task extends Emitter {
   }
 
   async transferCallToFeatureServer(cs, sipAddress, opts) {
-    const uuid = uuidv4();
+    const uuid = crypto.randomUUID();
     const {addKey} = cs.srf.locals.dbHelpers;
     const obj = Object.assign({}, cs.application);
     delete obj.requestor;

lib/tasks/transcribe.js

@@ -13,6 +13,8 @@ const {
   JambonzTranscriptionEvents,
   TranscribeStatus,
   AssemblyAiTranscriptionEvents,
+  VoxistTranscriptionEvents,
+  OpenAITranscriptionEvents,
   VerbioTranscriptionEvents,
   SpeechmaticsTranscriptionEvents
 } = require('../utils/constants.json');
@@ -27,8 +29,8 @@ class TaskTranscribe extends SttTask {
     super(logger, opts, parentTask);
 
     this.transcriptionHook = this.data.transcriptionHook;
+    this.translationHook = this.data.translationHook;
     this.earlyMedia = this.data.earlyMedia === true || (parentTask && parentTask.earlyMedia);
-
     if (this.data.recognizer) {
       this.interim = !!this.data.recognizer.interim;
       this.separateRecognitionPerChannel = this.data.recognizer.separateRecognitionPerChannel;
@@ -100,11 +102,10 @@ class TaskTranscribe extends SttTask {
         ...this.data.recognizer.nuanceOptions
       };
     }
-    const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
 
     if (cs.hasGlobalSttHints) {
       const {hints, hintsBoost} = cs.globalSttHints;
-      this.data.recognizer.hints = this.data.recognizer.hints.concat(hints);
+      this.data.recognizer.hints = this.data.recognizer?.hints?.concat(hints);
       if (!this.data.recognizer.hintsBoost && hintsBoost) this.data.recognizer.hintsBoost = hintsBoost;
       this.logger.debug({hints: this.data.recognizer.hints, hintsBoost: this.data.recognizer.hintsBoost},
         'Transcribe:exec - applying global sttHints');
@@ -117,9 +118,6 @@ class TaskTranscribe extends SttTask {
       if (this.transcribing2) {
         await this._startTranscribing(cs, ep2, 2);
       }
-
-      updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid)
-        .catch(() => {/*already logged error */});
     } catch (err) {
       if (!(await this._startFallback(cs, ep, {error: err}))) {
         this.logger.info(err, 'TaskTranscribe:exec - error');
@@ -303,10 +301,23 @@ class TaskTranscribe extends SttTask {
           this._onVendorConnectFailure.bind(this, cs, ep, channel));
         break;
 
+      case 'voxist':
+        this.bugname = `${this.bugname_prefix}voxist_transcribe`;
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.Transcription,
+          this._onTranscription.bind(this, cs, ep, channel));
+        this.addCustomEventListener(ep,
+          VoxistTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
+        this.addCustomEventListener(ep, VoxistTranscriptionEvents.ConnectFailure,
+          this._onVendorConnectFailure.bind(this, cs, ep, channel));
+        break;
+
       case 'speechmatics':
         this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
         this.addCustomEventListener(
-          ep, SpeechmaticsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+          ep, SpeechmaticsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep, channel));
+        this.addCustomEventListener(
+          ep, SpeechmaticsTranscriptionEvents.Translation, this._onTranslation.bind(this, cs, ep, channel));
         this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Info,
           this._onSpeechmaticsInfo.bind(this, cs, ep));
         this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.RecognitionStarted,
@@ -319,6 +330,20 @@ class TaskTranscribe extends SttTask {
           this._onSpeechmaticsError.bind(this, cs, ep));
         break;
 
+      case 'openai':
+        this.bugname = `${this.bugname_prefix}openai_transcribe`;
+        this.addCustomEventListener(
+          ep, OpenAITranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep, channel));
+        this.addCustomEventListener(ep, OpenAITranscriptionEvents.Connect,
+          this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, OpenAITranscriptionEvents.ConnectFailure,
+          this._onVendorConnectFailure.bind(this, cs, ep));
+        this.addCustomEventListener(ep, OpenAITranscriptionEvents.Error,
+          this._onOpenAIErrror.bind(this, cs, ep));
+
+        this.modelSupportsConversationTracking = opts.OPENAI_MODEL !== 'whisper-1';
+        break;
+
       default:
         if (this.vendor.startsWith('custom:')) {
           this.bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
@@ -354,6 +379,25 @@ class TaskTranscribe extends SttTask {
   async _transcribe(ep) {
     this.logger.debug(
       `TaskTranscribe:_transcribe - starting transcription vendor ${this.vendor} bugname ${this.bugname}`);
+
+    /* special feature for openai: we can provide a prompt that includes recent conversation history */
+    let prompt;
+    if (this.vendor === 'openai') {
+      if (this.modelSupportsConversationTracking) {
+        prompt = this.formatOpenAIPrompt(this.cs, {
+          prompt: this.data.recognizer?.openaiOptions?.prompt,
+          hintsTemplate: this.data.recognizer?.openaiOptions?.promptTemplates?.hintsTemplate,
+          // eslint-disable-next-line max-len
+          conversationHistoryTemplate: this.data.recognizer?.openaiOptions?.promptTemplates?.conversationHistoryTemplate,
+          hints: this.data.recognizer?.hints,
+        });
+        this.logger.debug({prompt}, 'Gather:_startTranscribing - created an openai prompt');
+      }
+      else if (this.data.recognizer?.hints?.length > 0) {
+        prompt = this.data.recognizer?.hints.join(', ');
+      }
+    }
+
     await ep.startTranscription({
       vendor: this.vendor,
       interim: this.interim ? true : false,
@@ -445,8 +489,9 @@ class TaskTranscribe extends SttTask {
         this._startAsrTimer(channel);
 
         /* some STT engines will keep listening after a final response, so no need to restart */
-        if (!['soniox', 'aws', 'microsoft', 'deepgram', 'google']
-          .includes(this.vendor)) this._startTranscribing(cs, ep, channel);
+        if (!this.doesVendorContinueListeningAfterFinalTranscript(this.vendor)) {
+          this._startTranscribing(cs, ep, channel);
+        }
       }
       else {
         if (this.vendor === 'soniox') {
@@ -469,9 +514,7 @@ class TaskTranscribe extends SttTask {
         this.logger.debug({evt}, 'TaskTranscribe:_onTranscription - sending final transcript');
         this._resolve(channel, evt);
 
-        /* some STT engines will keep listening after a final response, so no need to restart */
-        if (!['soniox', 'aws', 'microsoft', 'deepgram', 'google'].includes(this.vendor) &&
-          !this.vendor.startsWith('custom:')) {
+        if (!this.doesVendorContinueListeningAfterFinalTranscript(this.vendor)) {
           this.logger.debug('TaskTranscribe:_onTranscription - restarting transcribe');
           this._startTranscribing(cs, ep, channel);
         }
@@ -496,6 +539,47 @@ class TaskTranscribe extends SttTask {
     }
   }
 
+  async _onTranslation(_cs, _ep, channel, evt, _fsEvent) {
+    this.logger.debug({evt}, 'TaskTranscribe:_onTranslation');
+    if (this.translationHook && evt.results?.length > 0) {
+      try {
+        const b3 = this.getTracingPropagation();
+        const httpHeaders = b3 && {b3};
+        const payload = {
+          ...this.cs.callInfo,
+          ...httpHeaders,
+          translation: {
+            channel,
+            language: evt.language,
+            translation: evt.results[0].content
+          }
+        };
+
+        this.logger.debug({payload}, 'sending translationHook');
+        const json = await this.cs.requestor.request('verb:hook', this.translationHook, payload);
+        this.logger.info({json}, 'completed translationHook');
+        if (json && Array.isArray(json) && !this.parentTask) {
+          const makeTask = require('./make_task');
+          const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
+          if (tasks && tasks.length > 0) {
+            this.logger.info({tasks: tasks}, `${this.name} replacing application with ${tasks.length} tasks`);
+            this.cs.replaceApplication(tasks);
+          }
+        }
+      } catch (err) {
+        this.logger.info(err, 'TranscribeTask:_onTranslation error');
+      }
+      if (this.parentTask) {
+        this.parentTask.emit('translation', evt);
+      }
+    }
+    if (this.killed) {
+      this.logger.debug('TaskTranscribe:_onTranslation exiting after receiving final transcription');
+      this._clearTimer();
+      this.notifyTaskDone();
+    }
+  }
+
   async _resolve(channel, evt) {
     if (evt.is_final) {
       /* we've got a final transcript, so end the otel child span for this channel */
@@ -569,12 +653,21 @@ class TaskTranscribe extends SttTask {
   }
 
   _onMaxDurationExceeded(cs, ep, channel) {
-    this.logger.debug(`TaskTranscribe:_onMaxDurationExceeded on channel ${channel}`);
+    this.restartDueToError(ep, channel, 'Max duration exceeded');
+  }
+
+  _onMaxBufferExceeded(cs, ep, channel) {
+    this.restartDueToError(ep, channel, 'Max buffer exceeded');
+  }
+
+  restartDueToError(ep, channel, reason) {
+    this.logger.debug(`TaskTranscribe:${reason} on channel ${channel}`);
     if (this.paused) return;
+
     if (this.childSpan[channel - 1] && this.childSpan[channel - 1].span) {
       this.childSpan[channel - 1].span.setAttributes({
         channel,
-        'stt.resolve': 'max duration exceeded',
+        'stt.resolve': reason,
         'stt.label': this.label || 'None',
       });
       this.childSpan[channel - 1].span.end();
@@ -601,7 +694,6 @@ class TaskTranscribe extends SttTask {
         bugname: this.bugname
       })
         .catch((err) => this.logger.error({err}, `Error stopping transcription for primary vendor ${this.vendor}`));
-      const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
       try {
         this.notifyError({ msg: 'ASR error',
           details:`STT Vendor ${this.vendor} error: ${evt.error || evt.reason}`, failover: 'in progress'});
@@ -612,7 +704,6 @@ class TaskTranscribe extends SttTask {
         }
         this[`_speechHandlersSet_${channel}`] = false;
         this._startTranscribing(cs, _ep, channel);
-        updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid);
         return true;
       } catch (error) {
         this.notifyError({ msg: 'ASR error',
@@ -633,6 +724,14 @@ class TaskTranscribe extends SttTask {
       return;
     }
     this.logger.info({evt}, 'TaskTranscribe:_onJambonzError');
+    if (this.vendor === 'microsoft' &&
+      evt.error?.includes('Due to service inactivity, the client buffer exceeded maximum size. Resetting the buffer')) {
+      let channel = 1;
+      if (this.ep !== _ep) {
+        channel = 2;
+      }
+      return this._onMaxBufferExceeded(cs, _ep, channel);
+    }
     if (this.paused) return;
     const {writeAlerts, AlertType} = cs.srf.locals;
 
@@ -677,7 +776,13 @@ class TaskTranscribe extends SttTask {
     this.logger.debug({evt}, 'TaskGather:_onSpeechmaticsInfo');
   }
 
-  async _onSpeechmaticsErrror(cs, _ep, evt) {
+  async _onSpeechmaticsError(cs, _ep, evt) {
+    // eslint-disable-next-line no-unused-vars
+    const {message, ...e} = evt;
+    this._onVendorError(cs, _ep, {error: JSON.stringify(e)});
+  }
+
+  async _onOpenAIErrror(cs, _ep, evt) {
     // eslint-disable-next-line no-unused-vars
     const {message, ...e} = evt;
     this._onVendorError(cs, _ep, {error: JSON.stringify(e)});

lib/tasks/tts-task.js

@@ -1,6 +1,7 @@
 const Task = require('./task');
 const { TaskPreconditions } = require('../utils/constants');
 const { SpeechCredentialError } = require('../utils/error');
+const dbUtils = require('../utils/db-utils');
 
 class TtsTask extends Task {
 
@@ -12,14 +13,15 @@ class TtsTask extends Task {
 
     this.earlyMedia = this.data.earlyMedia === true || (parentTask && parentTask.earlyMedia);
     /**
-     * Task use taskInlcudeSynthesizer to identify
-     * if taskInlcudeSynthesizer === true, use label from verb.synthesizer, even it's empty
-     * if taskInlcudeSynthesizer === false, use label from application.synthesizer
+     * Task use taskIncludeSynthesizer to identify
+     * if taskIncludeSynthesizer === true, use label from verb.synthesizer, even it's empty
+     * if taskIncludeSynthesizer === false, use label from application.synthesizer
      */
-    this.taskInlcudeSynthesizer = !!this.data.synthesizer;
+    this.taskIncludeSynthesizer = !!this.data.synthesizer;
     this.synthesizer = this.data.synthesizer || {};
     this.disableTtsCache = this.data.disableTtsCache;
     this.options = this.synthesizer.options || {};
+    this.instructions = this.data.instructions;
   }
 
   async exec(cs) {
@@ -41,11 +43,108 @@ class TtsTask extends Task {
         }
       }
     }
+
+    const fullText = Array.isArray(this.text) ? this.text.join(' ') : this.text;
+    if (fullText.length > 0) {
+      cs.emit('botSaid', fullText);
+    }
+  }
+
+  getTtsVendorData(cs) {
+    const vendor = this.synthesizer.vendor && this.synthesizer.vendor !== 'default' ?
+      this.synthesizer.vendor :
+      cs.speechSynthesisVendor;
+    const language = this.synthesizer.language && this.synthesizer.language !== 'default' ?
+      this.synthesizer.language :
+      cs.speechSynthesisLanguage ;
+    const voice =  this.synthesizer.voice && this.synthesizer.voice !== 'default' ?
+      this.synthesizer.voice :
+      cs.speechSynthesisVoice;
+    const label = this.taskIncludeSynthesizer ? this.synthesizer.label : cs.speechSynthesisLabel;
+    return {vendor, language, voice, label};
+  }
+
+  async setTtsStreamingChannelVars(vendor, language, voice, credentials, ep) {
+    const {api_key, model_id, custom_tts_streaming_url, auth_token} = credentials;
+    let obj;
+
+    this.logger.debug({credentials},
+      `setTtsStreamingChannelVars: vendor: ${vendor}, language: ${language}, voice: ${voice}`);
+
+    switch (vendor) {
+      case 'deepgram':
+        obj = {
+          DEEPGRAM_API_KEY: api_key,
+          DEEPGRAM_TTS_STREAMING_MODEL: voice
+        };
+        break;
+      case 'cartesia':
+        obj = {
+          CARTESIA_API_KEY: api_key,
+          CARTESIA_TTS_STREAMING_MODEL_ID: model_id,
+          CARTESIA_TTS_STREAMING_VOICE_ID: voice,
+          CARTESIA_TTS_STREAMING_LANGUAGE: language || 'en',
+        };
+        break;
+      case 'elevenlabs':
+        const {stability, similarity_boost, use_speaker_boost, style, speed} = this.options.voice_settings || {};
+        obj = {
+          ELEVENLABS_API_KEY: api_key,
+          ELEVENLABS_TTS_STREAMING_MODEL_ID: model_id,
+          ELEVENLABS_TTS_STREAMING_VOICE_ID: voice,
+          // 20/12/2024 - only eleven_turbo_v2_5 support multiple language
+          ...(['eleven_turbo_v2_5'].includes(model_id) && {ELEVENLABS_TTS_STREAMING_LANGUAGE: language}),
+          ...(stability && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_STABILITY: stability}),
+          ...(similarity_boost && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_SIMILARITY_BOOST: similarity_boost}),
+          ...(use_speaker_boost && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_USE_SPEAKER_BOOST: use_speaker_boost}),
+          ...(style && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_STYLE: style}),
+          // speed has value 0.7 to 1.2, 1.0 is default, make sure we send the value event it's 0
+          ...(speed !== null && speed !== undefined && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_SPEED: `${speed}`}),
+          ...(this.options.pronunciation_dictionary_locators &&
+            Array.isArray(this.options.pronunciation_dictionary_locators) && {
+            ELEVENLABS_TTS_STREAMING_PRONUNCIATION_DICTIONARY_LOCATORS:
+            JSON.stringify(this.options.pronunciation_dictionary_locators)
+          }),
+        };
+        break;
+      case 'rimelabs':
+        const {
+          pauseBetweenBrackets, phonemizeBetweenBrackets, inlineSpeedAlpha, speedAlpha, reduceLatency
+        } = this.options;
+        obj = {
+          RIMELABS_API_KEY: api_key,
+          RIMELABS_TTS_STREAMING_MODEL_ID: model_id,
+          RIMELABS_TTS_STREAMING_VOICE_ID: voice,
+          RIMELABS_TTS_STREAMING_LANGUAGE: language || 'en',
+          ...(pauseBetweenBrackets && {RIMELABS_TTS_STREAMING_PAUSE_BETWEEN_BRACKETS: pauseBetweenBrackets}),
+          ...(phonemizeBetweenBrackets &&
+            {RIMELABS_TTS_STREAMING_PHONEMIZE_BETWEEN_BRACKETS: phonemizeBetweenBrackets}),
+          ...(inlineSpeedAlpha && {RIMELABS_TTS_STREAMING_INLINE_SPEED_ALPHA: inlineSpeedAlpha}),
+          ...(speedAlpha && {RIMELABS_TTS_STREAMING_SPEED_ALPHA: speedAlpha}),
+          ...(reduceLatency && {RIMELABS_TTS_STREAMING_REDUCE_LATENCY: reduceLatency})
+        };
+        break;
+      default:
+        if (vendor.startsWith('custom:')) {
+          const use_tls = custom_tts_streaming_url.startsWith('wss://');
+          obj = {
+            CUSTOM_TTS_STREAMING_HOST: custom_tts_streaming_url.replace(/^(ws|wss):\/\//, ''),
+            CUSTOM_TTS_STREAMING_API_KEY: auth_token,
+            CUSTOM_TTS_STREAMING_VOICE_ID: voice,
+            CUSTOM_TTS_STREAMING_LANGUAGE: language || 'en',
+            CUSTOM_TTS_STREAMING_USE_TLS: use_tls
+          };
+        } else {
+          throw new Error(`vendor ${vendor} is not supported for tts streaming yet`);
+        }
+    }
+    this.logger.debug({vendor, credentials, obj}, 'setTtsStreamingChannelVars');
+
+    await ep.set(obj);
   }
 
   async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label, preCache = false}) {
     const {srf, accountSid:account_sid} = cs;
-    const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, srf);
     const {writeAlerts, AlertType, stats} = srf.locals;
     const {synthAudio} = srf.locals.dbHelpers;
     const engine = this.synthesizer.engine || cs.synthesizer?.engine || 'neural';
@@ -57,15 +156,14 @@ class TtsTask extends Task {
         `No text-to-speech service credentials for ${vendor} with labels: ${label} have been configured`);
     }
     /* parse Nuance voices into name and model */
-    let model;
     if (vendor === 'nuance' && voice) {
       const arr = /([A-Za-z-]*)\s+-\s+(enhanced|standard)/.exec(voice);
       if (arr) {
         voice = arr[1];
-        model = arr[2];
+        this.model = arr[2];
       }
     } else if (vendor === 'deepgram') {
-      model = voice;
+      this.model = voice;
     }
 
     /* allow for microsoft custom region voice and api_key to be specified as an override */
@@ -95,8 +193,24 @@ class TtsTask extends Task {
     } else if (vendor === 'playht') {
       credentials = credentials || {};
       credentials.voice_engine = this.options.voice_engine || credentials.voice_engine;
+    } else if (vendor === 'google' && typeof voice === 'string' && voice.startsWith('custom_')) {
+      const {lookupGoogleCustomVoice} = dbUtils(this.logger, cs.srf);
+      const arr = /custom_(.*)/.exec(voice);
+      if (arr) {
+        const google_custom_voice_sid = arr[1];
+        const [custom_voice] = await lookupGoogleCustomVoice(google_custom_voice_sid);
+        if (custom_voice.use_voice_cloning_key) {
+          voice = {
+            voice_cloning_key: custom_voice.voice_cloning_key,
+          };
+        }
+      }
+    } else if (vendor === 'cartesia') {
+      credentials.model_id = this.options.model_id || credentials.model_id;
     }
 
+    this.model_id = credentials.model_id;
+
     /**
      * note on cache_speech_handles.  This was found to be risky.
      * It can cause a crash in the following sequence on a single call:
@@ -117,7 +231,8 @@ class TtsTask extends Task {
     // If vendor is changed from the previous one, then reset the cache_speech_handles flag
     //cs.currentTtsVendor = vendor;
 
-    if (!preCache && !this._disableTracing) this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
+    if (!preCache && !this._disableTracing)
+      this.logger.debug({vendor, language, voice, model: this.model}, 'TaskSay:exec');
     try {
       if (!credentials) {
         writeAlerts({
@@ -128,8 +243,6 @@ class TtsTask extends Task {
         }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
         throw new SpeechCredentialError('no provisioned speech credentials for TTS');
       }
-      // synthesize all of the text elements
-      let lastUpdated = false;
 
       /* produce an audio segment from the provided text */
       const generateAudio = async(text) => {
@@ -150,11 +263,12 @@ class TtsTask extends Task {
           const {filePath, servedFromCache, rtt} = await synthAudio(stats, {
             account_sid,
             text,
+            instructions: this.instructions,
             vendor,
             language,
             voice,
             engine,
-            model,
+            model: this.model,
             salt,
             credentials,
             options: this.options,
@@ -169,10 +283,6 @@ class TtsTask extends Task {
               this.otelSpan.end();
               this.otelSpan = null;
             }
-            if (!servedFromCache && !lastUpdated) {
-              lastUpdated = true;
-              updateSpeechCredentialLastUsed(credentials.speech_credential_sid).catch(() => {/* logged error */});
-            }
             if (!servedFromCache && rtt && !preCache  && !this._disableTracing) {
               this.notifyStatus({
                 event: 'synthesized-audio',

lib/utils/action-hook-delay.js

@@ -126,7 +126,12 @@ class ActionHookDelayProcessor extends Emitter {
     try {
       this._taskInProgress = makeTask(this.logger, t[0]);
       this._taskInProgress.disableTracing = true;
-      this._taskInProgress.exec(this.cs, {ep: this.ep});
+      this._taskInProgress.exec(this.cs, {ep: this.ep}).catch((err) => {
+        this.logger.info(`ActionHookDelayProcessor#_onNoResponseTimer: error playing file: ${err.message}`);
+        this._taskInProgress = null;
+        this.ep.removeAllListeners('playback-start');
+        this.ep.removeAllListeners('playback-stop');
+      });
     } catch (err) {
       this.logger.info(err, 'ActionHookDelayProcessor#_onNoResponseTimer: error starting action');
       this._taskInProgress = null;

lib/utils/amd-utils.js

@@ -45,6 +45,7 @@ if (VMD_HINTS_FILE) {
   });
 }
 
+
 class Amd extends Emitter {
   constructor(logger, cs, opts) {
     super();
@@ -68,6 +69,8 @@ class Amd extends Emitter {
     this.getIbmAccessToken = getIbmAccessToken;
     const {setChannelVarsForStt} = require('./transcription-utils')(logger);
     this.setChannelVarsForStt = setChannelVarsForStt;
+    this.digitCount = opts.digitCount || 0;
+    this.numberRegEx = RegExp(`[0-9]{${this.digitCount}}`);
 
     const {
       noSpeechTimeoutMs = 5000,
@@ -153,7 +156,7 @@ class Amd extends Emitter {
       const wordCount = t.alternatives[0].transcript.split(' ').length;
       const final = t.is_final;
 
-      const foundHint = hints.find((h) =>  t.alternatives[0].transcript.includes(h));
+      const foundHint = hints.find((h) =>  t.alternatives[0].transcript.toLowerCase().includes(h.toLowerCase()));
       if (foundHint) {
         /* we detected a common voice mail greeting */
         this.logger.debug(`Amd:evaluateTranscription: found hint ${foundHint}`);
@@ -163,6 +166,14 @@ class Amd extends Emitter {
           language: t.language_code
         });
       }
+      else if (this.digitCount != 0 && this.numberRegEx.test(t.alternatives[0].transcript)) {
+        /* a string of numbers is typically a machine */
+        this.emit(this.decision = AmdEvents.MachineDetected, {
+          reason: 'digit count',
+          greeting: t.alternatives[0].transcript,
+          language: t.language_code
+        });
+      }
       else if (final && wordCount < this.thresholdWordCount) {
         /* a short greeting is typically a human */
         this.emit(this.decision = AmdEvents.HumanDetected, {

lib/utils/aws-sns-lifecycle.js

@@ -4,7 +4,7 @@ const assert = require('assert');
 const {
   AWS_REGION,
   AWS_SNS_PORT: PORT,
-  AWS_SNS_TOPIC_ARM,
+  AWS_SNS_TOPIC_ARN,
   AWS_SNS_PORT_MAX,
 } = require('../config');
 const {LifeCycleEvents} = require('./constants');
@@ -55,12 +55,12 @@ class SnsNotifier extends Emitter {
   async _handlePost(req, res) {
     try {
       const parsedBody = JSON.parse(req.body);
-      this.logger.debug({headers: req.headers, body: parsedBody}, 'Received HTTP POST from AWS');
+      this.logger.info({headers: req.headers, body: parsedBody}, 'Received HTTP POST from AWS');
       if (!validatePayload(parsedBody)) {
         this.logger.info('incoming AWS SNS HTTP POST failed signature validation');
         return res.sendStatus(403);
       }
-      this.logger.debug('incoming HTTP POST passed validation');
+      this.logger.info('incoming HTTP POST passed validation');
       res.sendStatus(200);
 
       switch (parsedBody.Type) {
@@ -74,7 +74,18 @@ class SnsNotifier extends Emitter {
             subscriptionRequestId: this.subscriptionRequestId
           }, 'response from SNS SubscribeURL');
           const data = await this.describeInstance();
-          this.lifecycleState = data.AutoScalingGroups[0].Instances[0].LifecycleState;
+
+          const group = data.AutoScalingGroups.find((group) =>
+            group.Instances && group.Instances.some((instance) => instance.InstanceId === this.instanceId)
+          );
+          if (!group) {
+            this.logger.error('Current instance not found in any Auto Scaling group', data);
+          } else {
+            const instance = group.Instances.find((instance) => instance.InstanceId === this.instanceId);
+            this.lifecycleState = instance.LifecycleState;
+          }
+
+          //this.lifecycleState = data.AutoScalingGroups[0].Instances[0].LifecycleState;
           this.emit('SubscriptionConfirmation', {publicIp: this.publicIp});
           break;
 
@@ -94,7 +105,7 @@ class SnsNotifier extends Emitter {
               this.unsubscribe();
             }
             else {
-              this.logger.debug(`SnsNotifier - instance ${msg.EC2InstanceId} is scaling in (not us)`);
+              this.logger.info(`SnsNotifier - instance ${msg.EC2InstanceId} is scaling in (not us)`);
             }
           }
           break;
@@ -111,7 +122,7 @@ class SnsNotifier extends Emitter {
 
   async init() {
     try {
-      this.logger.debug('SnsNotifier: retrieving instance data');
+      this.logger.info('SnsNotifier: retrieving instance data');
       this.instanceId = await getString('http://169.254.169.254/latest/meta-data/instance-id');
       this.publicIp = await getString('http://169.254.169.254/latest/meta-data/public-ipv4');
       this.logger.info({
@@ -142,13 +153,13 @@ class SnsNotifier extends Emitter {
     try {
       const params = {
         Protocol: 'http',
-        TopicArn: AWS_SNS_TOPIC_ARM,
+        TopicArn: AWS_SNS_TOPIC_ARN,
         Endpoint: this.snsEndpoint
       };
       const response = await snsClient.send(new SubscribeCommand(params));
-      this.logger.info({response}, `response to SNS subscribe to ${AWS_SNS_TOPIC_ARM}`);
+      this.logger.info({response}, `response to SNS subscribe to ${AWS_SNS_TOPIC_ARN}`);
     } catch (err) {
-      this.logger.error({err}, `Error subscribing to SNS topic arn ${AWS_SNS_TOPIC_ARM}`);
+      this.logger.error({err}, `Error subscribing to SNS topic arn ${AWS_SNS_TOPIC_ARN}`);
     }
   }
 
@@ -159,9 +170,9 @@ class SnsNotifier extends Emitter {
         SubscriptionArn: this.subscriptionArn
       };
       const response = await snsClient.send(new UnsubscribeCommand(params));
-      this.logger.info({response}, `response to SNS unsubscribe to ${AWS_SNS_TOPIC_ARM}`);
+      this.logger.info({response}, `response to SNS unsubscribe to ${AWS_SNS_TOPIC_ARN}`);
     } catch (err) {
-      this.logger.error({err}, `Error unsubscribing to SNS topic arn ${AWS_SNS_TOPIC_ARM}`);
+      this.logger.error({err}, `Error unsubscribing to SNS topic arn ${AWS_SNS_TOPIC_ARN}`);
     }
   }
 

lib/utils/background-task-manager.js

@@ -46,6 +46,9 @@ class BackgroundTaskManager extends Emitter {
       case 'transcribe':
         task = await this._initTranscribe(opts);
         break;
+      case 'ttsStream':
+        task = await this._initTtsStream(opts);
+        break;
       default:
         break;
     }
@@ -100,6 +103,7 @@ class BackgroundTaskManager extends Emitter {
   async _initBargeIn(opts) {
     let task;
     try {
+      const copy = JSON.parse(JSON.stringify(opts));
       const t = normalizeJambones(this.logger, [opts]);
       task = makeTask(this.logger, t[0]);
       task
@@ -118,7 +122,7 @@ class BackgroundTaskManager extends Emitter {
           if (task.sticky && !this.cs.callGone && !this.cs._stopping) {
             this.logger.info('BackgroundTaskManager:_initBargeIn: restarting background bargeIn');
             this._bargeInHandled = false;
-            this.newTask('bargeIn', opts, true);
+            this.newTask('bargeIn', copy, true);
           }
           return;
         })
@@ -173,6 +177,25 @@ class BackgroundTaskManager extends Emitter {
     return task;
   }
 
+  // Initiate Tts Stream
+  async _initTtsStream(opts) {
+    let task;
+    try {
+      const t = normalizeJambones(this.logger, [opts]);
+      task = makeTask(this.logger, t[0]);
+      const resources = await this.cs._evaluatePreconditions(task);
+      const {span, ctx} = this.rootSpan.startChildSpan(`background-ttsStream:${task.summary}`);
+      task.span = span;
+      task.ctx = ctx;
+      task.exec(this.cs, resources)
+        .then(this._taskCompleted.bind(this, 'ttsStream', task))
+        .catch(this._taskError.bind(this, 'ttsStream', task));
+    } catch (err) {
+      this.logger.info(err, 'BackgroundTaskManager:_initTtsStream - Error creating ttsStream task');
+    }
+    return task;
+  }
+
   _taskCompleted(type, task) {
     this.logger.debug({type, task}, `BackgroundTaskManager:_taskCompleted: task completed, sticky: ${task.sticky}`);
     task.removeAllListeners();

lib/utils/base-requestor.js

@@ -1,6 +1,7 @@
 const assert = require('assert');
 const Emitter = require('events');
 const crypto = require('crypto');
+const parseUrl = require('parse-url');
 const timeSeries = require('@jambonz/time-series');
 const {NODE_ENV, JAMBONES_TIME_SERIES_HOST} = require('../config');
 let alerter ;
@@ -21,6 +22,10 @@ class BaseRequestor extends Emitter {
     const {stats} = require('../../').srf.locals;
     this.stats = stats;
 
+    const u = this._parsedUrl = parseUrl(this.url);
+    if (u.port) this._baseUrl = `${u.protocol}://${u.resource}:${u.port}`;
+    else this._baseUrl = `${u.protocol}://${u.resource}`;
+
     if (!alerter) {
       alerter = timeSeries(logger, {
         host: JAMBONES_TIME_SERIES_HOST,
@@ -30,6 +35,10 @@ class BaseRequestor extends Emitter {
     }
   }
 
+  get baseUrl() {
+    return this._baseUrl;
+  }
+
   get Alerter() {
     return alerter;
   }

lib/utils/constants.json

@@ -28,10 +28,11 @@
     "SipRedirect": "sip:redirect",
     "Say": "say",
     "SayLegacy": "say:legacy",
+    "Stream": "stream",
     "Tag": "tag",
     "Transcribe": "transcribe"
   },
-  "AllowedSipRecVerbs": ["answer", "config", "gather", "transcribe", "listen", "tag"],
+  "AllowedSipRecVerbs": ["answer", "config", "gather", "transcribe", "listen", "tag", "hangup", "sip:decline"],
   "AllowedConfirmSessionVerbs": ["config", "gather", "plays", "say", "tag"],
   "CallStatus": {
     "Trying": "trying",
@@ -129,12 +130,25 @@
   },
   "SpeechmaticsTranscriptionEvents": {
     "Transcription": "speechmatics_transcribe::transcription",
+    "Translation": "speechmatics_transcribe::translation",
     "Info": "speechmatics_transcribe::info",
     "RecognitionStarted": "speechmatics_transcribe::recognition_started",
     "ConnectFailure": "speechmatics_transcribe::connect_failed",
     "Connect": "speechmatics_transcribe::connect",
     "Error": "speechmatics_transcribe::error"
   },
+  "OpenAITranscriptionEvents": {
+    "Transcription": "openai_transcribe::transcription",
+    "Translation": "openai_transcribe::translation",
+    "SpeechStarted": "openai_transcribe::speech_started",
+    "SpeechStopped": "openai_transcribe::speech_stopped",
+    "PartialTranscript": "openai_transcribe::partial_transcript",
+    "Info": "openai_transcribe::info",
+    "RecognitionStarted": "openai_transcribe::recognition_started",
+    "ConnectFailure": "openai_transcribe::connect_failed",
+    "Connect": "openai_transcribe::connect",
+    "Error": "openai_transcribe::error"
+  },
   "JambonzTranscriptionEvents": {
     "Transcription": "jambonz_transcribe::transcription",
     "ConnectFailure": "jambonz_transcribe::connect_failed",
@@ -147,6 +161,12 @@
     "ConnectFailure": "assemblyai_transcribe::connect_failed",
     "Connect": "assemblyai_transcribe::connect"
   },
+  "VoxistTranscriptionEvents": {
+    "Transcription": "voxist_transcribe::transcription",
+    "Error": "voxist_transcribe::error",
+    "ConnectFailure": "voxist_transcribe::connect_failed",
+    "Connect": "voxist_transcribe::connect"
+  },
   "VadDetection": {
     "Detection": "vad_detect:detection"
   },
@@ -174,6 +194,34 @@
     "Disconnect": "openai_s2s::disconnect",
     "ServerEvent": "openai_s2s::server_event"
   },
+  "LlmEvents_Google": {
+    "Error": "error",
+    "Connect": "google_s2s::connect",
+    "ConnectFailure": "google_s2s::connect_failed",
+    "Disconnect": "google_s2s::disconnect",
+    "ServerEvent": "google_s2s::server_event"
+  },
+  "LlmEvents_Elevenlabs": {
+    "Error": "error",
+    "Connect": "elevenlabs_s2s::connect",
+    "ConnectFailure": "elevenlabs_s2s::connect_failed",
+    "Disconnect": "elevenlabs_s2s::disconnect",
+    "ServerEvent": "elevenlabs_s2s::server_event"
+  },
+  "LlmEvents_VoiceAgent": {
+    "Error": "error",
+    "Connect": "voice_agent_s2s::connect",
+    "ConnectFailure": "voice_agent_s2s::connect_failed",
+    "Disconnect": "voice_agent_s2s::disconnect",
+    "ServerEvent": "voice_agent_s2s::server_event"
+  },
+  "LlmEvents_Ultravox": {
+    "Error": "error",
+    "Connect": "ultravox_s2s::connect",
+    "ConnectFailure": "ultravox_s2s::connect_failed",
+    "Disconnect": "ultravox_s2s::disconnect",
+    "ServerEvent": "ultravox_s2s::server_event"
+  },
   "QueueResults": {
     "Bridged": "bridged",
     "Error": "error",
@@ -188,7 +236,8 @@
   },
   "KillReason": {
     "Hangup": "hangup",
-    "Replaced": "replaced"
+    "Replaced": "replaced",
+    "MediaTimeout": "media_timeout"
   },
   "HookMsgTypes": [
     "session:new",
@@ -202,6 +251,8 @@
     "verb:status",
     "llm:event",
     "llm:tool-call",
+    "tts:tokens-result",
+    "tts:streaming-event",
     "jambonz:error"
   ],
   "RecordState": {
@@ -220,7 +271,59 @@
     "ToneTimeout": "amd_tone_timeout",
     "Stopped": "amd_stopped"
   },
+  "MediaPath": {
+    "NoMedia": "no-media",
+    "PartialMedia": "partial-media",
+    "FullMedia": "full-media"
+  },
+  "DeepgramTtsStreamingEvents": {
+    "Empty": "deepgram_tts_streaming::empty",
+    "ConnectFailure": "deepgram_tts_streaming::connect_failed",
+    "Connect": "deepgram_tts_streaming::connect"
+  },
+  "CartesiaTtsStreamingEvents": {
+    "Empty": "cartesia_tts_streaming::empty",
+    "ConnectFailure": "cartesia_tts_streaming::connect_failed",
+    "Connect": "cartesia_tts_streaming::connect"
+  },
+  "ElevenlabsTtsStreamingEvents": {
+    "Empty": "elevenlabs_tts_streaming::empty",
+    "ConnectFailure": "elevenlabs_tts_streaming::connect_failed",
+    "Connect": "elevenlabs_tts_streaming::connect"
+  },
+  "RimelabsTtsStreamingEvents": {
+    "Empty": "rimelabs_tts_streaming::empty",
+    "ConnectFailure": "rimelabs_tts_streaming::connect_failed",
+    "Connect": "rimelabs_tts_streaming::connect"
+  },
+  "CustomTtsStreamingEvents": {
+    "Empty": "custom_tts_streaming::empty",
+    "ConnectFailure": "custom_tts_streaming::connect_failed",
+    "Connect": "custom_tts_streaming::connect"
+  },
+  "TtsStreamingEvents": {
+    "Empty": "tts_streaming::empty",
+    "Pause": "tts_streaming::pause",
+    "Resume": "tts_streaming::resume",
+    "ConnectFailure": "tts_streaming::connect_failed"
+  },
+  "TtsStreamingConnectionStatus": {
+    "NotConnected": "not_connected",
+    "Connected": "connected",
+    "Connecting": "connecting",
+    "Failed": "failed"
+  },
   "MAX_SIMRINGS": 10,
   "BONG_TONE": "tone_stream://v=-7;%(100,0,941.0,1477.0);v=-7;>=2;+=.1;%(1400,0,350,440)",
-  "FS_UUID_SET_NAME": "fsUUIDs"
+  "FS_UUID_SET_NAME": "fsUUIDs",
+  "SystemState" : {
+    "Online": "ONLINE",
+    "Offline": "OFFLINE",
+    "GracefulShutdownInProgress":"SHUTDOWN_IN_PROGRESS"
+  },
+  "FEATURE_SERVER" : "feature-server",
+  "WS_CLOSE_CODES": {
+    "NormalClosure": 1000,
+    "GoingAway": 1001
+  }
 }

lib/utils/db-utils.js

@@ -105,6 +105,13 @@ const speechMapper = (cred) => {
       obj.voice_engine = o.voice_engine;
       obj.options = o.options;
     }
+    else if ('cartesia' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = o.api_key;
+      obj.model_id = o.model_id;
+      obj.embedding = o.embedding;
+      obj.options = o.options;
+    }
     else if ('rimelabs' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
@@ -115,6 +122,10 @@ const speechMapper = (cred) => {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
     }
+    else if ('voxist' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = o.api_key;
+    }
     else if ('whisper' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
@@ -131,11 +142,17 @@ const speechMapper = (cred) => {
       obj.api_key = o.api_key;
       obj.speechmatics_stt_uri = o.speechmatics_stt_uri;
     }
+    else if ('openai' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = o.api_key;
+      obj.model_id = o.model_id;
+    }
     else if (obj.vendor.startsWith('custom:')) {
       const o = JSON.parse(decrypt(credential));
       obj.auth_token = o.auth_token;
       obj.custom_stt_url = o.custom_stt_url;
       obj.custom_tts_url = o.custom_tts_url;
+      obj.custom_tts_streaming_url = o.custom_tts_streaming_url;
     }
   } catch (err) {
     console.log(err);
@@ -215,11 +232,23 @@ module.exports = (logger, srf) => {
     }
   };
 
+  const lookupVoipCarrierBySid = async(sid) => {
+    const pp = pool.promise();
+    try {
+      const [r] = await pp.query('SELECT * FROM voip_carriers WHERE voip_carrier_sid = ?', [sid]);
+      return r;
+
+    } catch (err) {
+      logger.error({err}, `lookupVoipCarrierBySid: Error ${sid}`);
+    }
+  };
+
   return {
     lookupAccountDetails,
     updateSpeechCredentialLastUsed,
     lookupCarrier,
     lookupCarrierByPhoneNumber,
-    lookupGoogleCustomVoice
+    lookupGoogleCustomVoice,
+    lookupVoipCarrierBySid
   };
 };

lib/utils/error.js

@@ -1,9 +1,33 @@
-class SpeechCredentialError extends Error {
+class NonFatalTaskError extends Error {
   constructor(msg) {
     super(msg);
   }
 }
 
+class SpeechCredentialError extends NonFatalTaskError {
+  constructor(msg) {
+    super(msg);
+  }
+}
+
+class PlayFileNotFoundError extends NonFatalTaskError {
+  constructor(url) {
+    super('File not found');
+    this.url = url;
+  }
+}
+
+class HTTPResponseError extends Error {
+  constructor(statusCode) {
+    super('Unexpected HTTP Response');
+    delete this.stack;
+    this.statusCode = statusCode;
+  }
+}
+
 module.exports = {
-  SpeechCredentialError
+  SpeechCredentialError,
+  NonFatalTaskError,
+  PlayFileNotFoundError,
+  HTTPResponseError
 };

lib/utils/helpers.js

@@ -0,0 +1,5 @@
+
+const sleepFor = (ms) => new Promise((resolve) => setTimeout(() => resolve(), ms));
+module.exports = {
+  sleepFor
+};

lib/utils/http-requestor.js

@@ -16,6 +16,7 @@ const {
   NODE_ENV,
   HTTP_USER_AGENT_HEADER,
 } = require('../config');
+const {HTTPResponseError} = require('./error');
 
 const toBase64 = (str) => Buffer.from(str || '', 'utf8').toString('base64');
 
@@ -47,8 +48,6 @@ class HttpRequestor extends BaseRequestor {
     assert(['GET', 'POST'].includes(this.method));
 
     const u = this._parsedUrl = parseUrl(this.url);
-    if (u.port) this._baseUrl = `${u.protocol}://${u.resource}:${u.port}`;
-    else this._baseUrl = `${u.protocol}://${u.resource}`;
     this._protocol = u.protocol;
     this._resource = u.resource;
     this._port = u.port;
@@ -56,18 +55,18 @@ class HttpRequestor extends BaseRequestor {
     this._usePools = HTTP_POOL && parseInt(HTTP_POOL);
 
     if (this._usePools) {
-      if (pools.has(this._baseUrl)) {
-        this.client = pools.get(this._baseUrl);
+      if (pools.has(this.baseUrl)) {
+        this.client = pools.get(this.baseUrl);
       }
       else {
         const connections = HTTP_POOLSIZE ? parseInt(HTTP_POOLSIZE) : 10;
         const pipelining = HTTP_PIPELINING ? parseInt(HTTP_PIPELINING) : 1;
-        const pool = this.client = new Pool(this._baseUrl, {
+        const pool = this.client = new Pool(this.baseUrl, {
           connections,
           pipelining
         });
-        pools.set(this._baseUrl, pool);
-        this.logger.debug(`HttpRequestor:created pool for ${this._baseUrl}`);
+        pools.set(this.baseUrl, pool);
+        this.logger.debug(`HttpRequestor:created pool for ${this.baseUrl}`);
       }
     }
     else {
@@ -88,10 +87,6 @@ class HttpRequestor extends BaseRequestor {
     }
   }
 
-  get baseUrl() {
-    return this._baseUrl;
-  }
-
   close() {
     if (!this._usePools && !this.client?.closed) this.client.close();
   }
@@ -113,7 +108,7 @@ class HttpRequestor extends BaseRequestor {
 
     assert(HookMsgTypes.includes(type));
 
-    const payload = params ? snakeCaseKeys(params, ['customerData', 'sip']) : null;
+    const payload = params ? snakeCaseKeys(params, ['customerData', 'sip', 'env_vars', 'args']) : null;
     const url = hook.url || hook;
     const method = hook.method || 'POST';
     let buf = '';
@@ -190,8 +185,7 @@ class HttpRequestor extends BaseRequestor {
         followRedirects: false
       });
       if (![200, 202, 204].includes(statusCode)) {
-        const err = new Error();
-        err.statusCode = statusCode;
+        const err = new HTTPResponseError(statusCode);
         throw err;
       }
       if (headers['content-type']?.includes('application/json')) {
@@ -225,7 +219,7 @@ class HttpRequestor extends BaseRequestor {
     const rtt = this._roundTrip(startAt);
     if (buf) this.stats.histogram('app.hook.response_time', rtt, ['hook_type:app']);
 
-    if (buf && Array.isArray(buf)) {
+    if (buf && (Array.isArray(buf) || type == 'llm:tool-call')) {
       this.logger.info({response: buf}, `HttpRequestor:request ${method} ${url} succeeded in ${rtt}ms`);
       return buf;
     }

lib/utils/install-srf-locals.js

@@ -31,18 +31,26 @@ function getLocalIp() {
   return '127.0.0.1'; // Fallback to localhost if no suitable interface found
 }
 
-function initMS(logger, wrapper, ms) {
+function initMS(logger, wrapper, ms, {
+  onFreeswitchConnect,
+  onFreeswitchDisconnect
+}) {
   Object.assign(wrapper, {ms, active: true, connects: 1});
   logger.info(`connected to freeswitch at ${ms.address}`);
 
+  onFreeswitchConnect(wrapper);
+
   ms.conn
     .on('esl::end', () => {
       wrapper.active = false;
+      wrapper.connects = 0;
       logger.info(`lost connection to freeswitch at ${ms.address}`);
+      onFreeswitchDisconnect(wrapper);
+      ms.removeAllListeners();
     })
     .on('esl::ready', () => {
       if (wrapper.connects > 0) {
-        logger.info(`connected to freeswitch at ${ms.address}`);
+        logger.info(`esl::ready connected to freeswitch at ${ms.address}`);
       }
       wrapper.connects = 1;
       wrapper.active = true;
@@ -56,7 +64,10 @@ function initMS(logger, wrapper, ms) {
   });
 }
 
-function installSrfLocals(srf, logger) {
+function installSrfLocals(srf, logger, {
+  onFreeswitchConnect = () => {},
+  onFreeswitchDisconnect = () => {}
+}) {
   logger.debug('installing srf locals');
   assert(!srf.locals.dbHelpers);
   const {tracer} = srf.locals.otel;
@@ -91,7 +102,10 @@ function installSrfLocals(srf, logger) {
       mediaservers.push(val);
       try {
         const ms = await mrf.connect(fs);
-        initMS(logger, val, ms);
+        initMS(logger, val, ms, {
+          onFreeswitchConnect,
+          onFreeswitchDisconnect
+        });
       }
       catch (err) {
         logger.info({err}, `failed connecting to freeswitch at ${fs.address}, will retry shortly: ${err.message}`);
@@ -102,9 +116,15 @@ function installSrfLocals(srf, logger) {
       for (const val of mediaservers) {
         if (val.connects === 0) {
           try {
+            // make sure all listeners are removed before reconnecting
+            val.ms?.disconnect();
+            val.ms = null;
             logger.info({mediaserver: val.opts}, 'Retrying initial connection to media server');
             const ms = await mrf.connect(val.opts);
-            initMS(logger, val, ms);
+            initMS(logger, val, ms, {
+              onFreeswitchConnect,
+              onFreeswitchDisconnect
+            });
           } catch (err) {
             logger.info({err}, `failed connecting to freeswitch at ${val.opts.address}, will retry shortly`);
           }
@@ -199,7 +219,8 @@ function installSrfLocals(srf, logger) {
   } = require('@jambonz/speech-utils')({}, logger);
   const {
     writeAlerts,
-    AlertType
+    AlertType,
+    writeSystemAlerts
   } = require('@jambonz/time-series')(logger, {
     host: JAMBONES_TIME_SERIES_HOST,
     commitSize: 50,
@@ -269,7 +290,8 @@ function installSrfLocals(srf, logger) {
     getFreeswitch,
     stats: stats,
     writeAlerts,
-    AlertType
+    AlertType,
+    writeSystemAlerts
   };
 
   if (localIp) {

lib/utils/llm-mcp.js

@@ -0,0 +1,103 @@
+const { Client } = require('@modelcontextprotocol/sdk/client/index.js');
+
+class LlmMcpService {
+
+  constructor(logger, mcpServers) {
+    this.logger = logger;
+    this.mcpServers = mcpServers || [];
+    this.mcpClients = [];
+  }
+
+  // make sure we call init() before using any of the mcp clients
+  // this is to ensure that we have a valid connection to the MCP server
+  // and that we have collected the available tools.
+  async init() {
+    if (this.mcpClients.length > 0) {
+      return;
+    }
+    const { SSEClientTransport } = await import('@modelcontextprotocol/sdk/client/sse.js');
+    for (const server of this.mcpServers) {
+      const { url } = server;
+      if (url) {
+        try {
+          const transport = new SSEClientTransport(new URL(url), {});
+          const client = new Client({ name: 'Jambonz MCP Client', version: '1.0.0' });
+          await client.connect(transport);
+          // collect available tools
+          const { tools } = await client.listTools();
+          this.mcpClients.push({
+            url,
+            client,
+            tools
+          });
+        } catch (err) {
+          this.logger.error(`LlmMcpService: Failed to connect to MCP server at ${url}: ${err.message}`);
+        }
+      }
+    }
+  }
+
+  async getAvailableMcpTools() {
+    // returns a list of available tools from all MCP clients
+    const tools = [];
+    for (const mcpClient of this.mcpClients) {
+      const {tools: availableTools} = mcpClient;
+      if (availableTools) {
+        tools.push(...availableTools);
+      }
+    }
+    return tools;
+  }
+
+  async getMcpClientByToolName(name) {
+    for (const mcpClient of this.mcpClients) {
+      const { tools } = mcpClient;
+      if (tools && tools.some((tool) => tool.name === name)) {
+        return mcpClient.client;
+      }
+    }
+    return null;
+  }
+
+  async getMcpClientByToolId(id) {
+    for (const mcpClient of this.mcpClients) {
+      const { tools } = mcpClient;
+      if (tools && tools.some((tool) => tool.id === id)) {
+        return mcpClient.client;
+      }
+    }
+    return null;
+  }
+
+  async callMcpTool(name, input) {
+    const client = await this.getMcpClientByToolName(name);
+    if (client) {
+      try {
+        const result = await client.callTool({
+          name,
+          arguments: input,
+        });
+        this.logger.debug({result}, 'LlmMcpService - result');
+        return result;
+      } catch (err) {
+        this.logger.error({err}, 'LlmMcpService - error calling tool');
+        throw err;
+      }
+    }
+  }
+
+  async close() {
+    for (const mcpClient of this.mcpClients) {
+      const { client } = mcpClient;
+      if (client) {
+        await client.close();
+        this.logger.debug({url: mcpClient.url}, 'LlmMcpService - mcp client closed');
+      }
+    }
+    this.mcpClients = [];
+  }
+
+}
+
+module.exports = LlmMcpService;
+

lib/utils/network.js

@@ -0,0 +1,32 @@
+
+/**
+ * Parses a list of hostport entries and selects the first one that matches the specified protocol,
+ * excluding any entries with the localhost IP address ('127.0.0.1').
+ *
+ * Each hostport entry should be in the format: 'protocol/ip:port'
+ *
+ * @param {Object} logger - A logging object with a 'debug' method for logging debug messages.
+ * @param {string} hostport - A comma-separated string containing hostport entries.
+ * @param {string} protocol - The protocol to match (e.g., 'udp', 'tcp').
+ * @returns {Array} An array containing:
+ * 0: protocol
+ * 1: ip address
+ * 2: port
+ */
+const selectHostPort = (logger, hostport, protocol) => {
+  logger.debug(`selectHostPort: ${hostport}, ${protocol}`);
+  const sel = hostport
+    .split(',')
+    .map((hp) => {
+      const arr = /(.*)\/(.*):(.*)/.exec(hp);
+      return [arr[1], arr[2], arr[3]];
+    })
+    .filter((hp) => {
+      return hp[0] === protocol && hp[1] !== '127.0.0.1';
+    });
+  return sel[0];
+};
+
+module.exports = {
+  selectHostPort
+};

lib/utils/place-outdial.js

@@ -1,5 +1,5 @@
 const Emitter = require('events');
-const {CallStatus} = require('./constants');
+const {CallStatus, MediaPath} = require('./constants');
 const SipError = require('drachtio-srf').SipError;
 const {TaskPreconditions, CallDirection} = require('../utils/constants');
 const CallInfo = require('../session/call-info');
@@ -12,13 +12,17 @@ const deepcopy = require('deepcopy');
 const moment = require('moment');
 const stripCodecs = require('./strip-ancillary-codecs');
 const RootSpan = require('./call-tracer');
-const uuidv4 = require('uuid-random');
+const crypto = require('crypto');
 const HttpRequestor = require('./http-requestor');
 const WsRequestor = require('./ws-requestor');
-const {makeOpusFirst} = require('./sdp-utils');
+const {makeOpusFirst, removeVideoSdp} = require('./sdp-utils');
 const {
-  JAMBONES_USE_FREESWITCH_TIMER_FD
+  JAMBONES_USE_FREESWITCH_TIMER_FD,
+  JAMBONES_MEDIA_TIMEOUT_MS,
+  JAMBONES_MEDIA_HOLD_TIMEOUT_MS,
+  JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS
 } = require('../config');
+const { sleepFor } = require('./helpers');
 
 class SingleDialer extends Emitter {
   constructor({logger, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask,
@@ -43,7 +47,7 @@ class SingleDialer extends Emitter {
 
     this.callGone = false;
 
-    this.callSid = uuidv4();
+    this.callSid = crypto.randomUUID();
     this.dialTask = dialTask;
     this.onHoldMusic = onHoldMusic;
 
@@ -150,15 +154,21 @@ class SingleDialer extends Emitter {
         return;
       }
       let lastSdp;
-      const connectStream = async(remoteSdp) => {
+      const connectStream = async(remoteSdp, isVideoCall) => {
         if (remoteSdp === lastSdp) return;
+        if (process.env.JAMBONES_VIDEO_CALLS_ENABLED_IN_FS && !isVideoCall) {
+          remoteSdp = removeVideoSdp(remoteSdp);
+        }
         lastSdp = remoteSdp;
         return this.ep.modify(remoteSdp);
       };
-
+      let localSdp = this.ep.local.sdp;
+      if (process.env.JAMBONES_VIDEO_CALLS_ENABLED_IN_FS && !opts.isVideoCall) {
+        localSdp = removeVideoSdp(localSdp);
+      }
       Object.assign(opts, {
         proxy: `sip:${this.sbcAddress}`,
-        localSdp: opts.opusFirst ? makeOpusFirst(this.ep.local.sdp) : this.ep.local.sdp
+        localSdp: opts.opusFirst ? makeOpusFirst(localSdp) : localSdp
       });
       if (this.target.auth) opts.auth = this.target.auth;
       inviteSpan = this.startSpan('invite', {
@@ -220,13 +230,13 @@ class SingleDialer extends Emitter {
               status.callStatus = CallStatus.EarlyMedia;
               this.emit('earlyMedia');
             }
-            connectStream(prov.body);
+            connectStream(prov.body, opts.isVideoCall);
           }
           else status.callStatus = CallStatus.Ringing;
           this.emit('callStatusChange', status);
         }
       });
-      await connectStream(this.dlg.remote.sdp);
+      await connectStream(this.dlg.remote.sdp, opts.isVideoCall);
       this.dlg.callSid = this.callSid;
       this.inviteInProgress = null;
       this.emit('callStatusChange', {
@@ -269,7 +279,12 @@ class SingleDialer extends Emitter {
                 this.logger.info('dial is onhold, emit event');
                 this.emit('reinvite', req, res);
               } else {
-                const newSdp = await this.ep.modify(req.body);
+                let newSdp = await this.ep.modify(req.body);
+                // in case of reINVITE if video call is enabled in FS and the call is not a video call,
+                // remove video media from the SDP
+                if (process.env.JAMBONES_VIDEO_CALLS_ENABLED_IN_FS && !this.opts?.isVideoCall) {
+                  newSdp = removeVideoSdp(newSdp);
+                }
                 res.send(200, {body: newSdp});
                 this.logger.info({offer: req.body, answer: newSdp}, 'SingleDialer:exec: handling reINVITE');
               }
@@ -317,14 +332,19 @@ class SingleDialer extends Emitter {
   /**
    * kill the call in progress or the stable dialog, whichever we have
    */
-  async kill() {
+  async kill(Reason) {
     this.killed = true;
     if (this.inviteInProgress) await this.inviteInProgress.cancel();
     else if (this.dlg && this.dlg.connected) {
       const duration = moment().diff(this.dlg.connectTime, 'seconds');
       this.logger.debug('SingleDialer:kill hanging up called party');
       this.emit('callStatusChange', {callStatus: CallStatus.Completed, duration});
-      this.dlg.destroy();
+      const headers = {
+        ...(Reason && {'X-Reason': Reason})
+      };
+      this.dlg.destroy({
+        headers
+      });
     }
     if (this.ep) {
       this.logger.debug(`SingleDialer:kill - deleting endpoint ${this.ep.uuid}`);
@@ -335,11 +355,40 @@ class SingleDialer extends Emitter {
   _configMsEndpoint() {
     const opts = {
       ...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
-      ...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'})
+      ...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'}),
+      ...(JAMBONES_MEDIA_TIMEOUT_MS && {media_timeout: JAMBONES_MEDIA_TIMEOUT_MS}),
+      ...(JAMBONES_MEDIA_HOLD_TIMEOUT_MS && {media_hold_timeout: JAMBONES_MEDIA_HOLD_TIMEOUT_MS})
     };
     if (Object.keys(opts).length > 0) {
       this.ep.set(opts);
     }
+    if (this.dialTask?.inbandDtmfEnabled && !this.ep.inbandDtmfEnabled) {
+      // https://developer.signalwire.com/freeswitch/FreeSWITCH-Explained/Modules/mod-dptools/6587132/#0-about
+      try {
+        this.ep.execute('start_dtmf');
+        this.ep.inbandDtmfEnabled = true;
+      } catch (err) {
+        this.logger.info(err, 'place-outdial:_configMsEndpoint - error enable inband DTMF');
+      }
+    }
+
+    const origDestroy = this.ep.destroy.bind(this.ep);
+    this.ep.destroy = async() => {
+      try {
+        if (this.dialTask.transcribeTask && JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS) {
+          // transcribe task is being used, wait for some time before destroy
+          // if final transcription is received but endpoint is already closed,
+          // freeswitch module will not be able to send the transcription
+
+          this.logger.info('SingleDialer:_configMsEndpoint -' +
+            ' Dial with transcribe task, wait for some time before destroy');
+          await sleepFor(JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS);
+        }
+        await origDestroy();
+      } catch (err) {
+        this.logger.error(err, 'SingleDialer:_configMsEndpoint - error destroying endpoint');
+      }
+    };
   }
 
   /**
@@ -381,7 +430,8 @@ class SingleDialer extends Emitter {
         callInfo: this.callInfo,
         accountInfo: this.accountInfo,
         tasks,
-        rootSpan: this.rootSpan
+        rootSpan: this.rootSpan,
+        req: this.req
       });
       await cs.exec();
 
@@ -390,7 +440,10 @@ class SingleDialer extends Emitter {
     } catch (err) {
       this.logger.debug(err, 'SingleDialer:_executeApp: error');
       this.emit('decline');
-      if (this.dlg.connected) this.dlg.destroy();
+      if (this.dlg.connected) {
+        this.dlg.destroy();
+        this.ep.destroy();
+      }
     }
   }
 
@@ -455,21 +508,26 @@ class SingleDialer extends Emitter {
     return cs;
   }
 
-  async releaseMediaToSBC(remoteSdp, localSdp) {
+  async releaseMediaToSBC(remoteSdp, localSdp, releaseMediaEntirely) {
     assert(this.dlg && this.dlg.connected && this.ep && typeof remoteSdp === 'string');
     const sdp = stripCodecs(this.logger, remoteSdp, localSdp) || remoteSdp;
     await this.dlg.modify(sdp, {
       headers: {
-        'X-Reason': 'release-media'
+        'X-Reason': releaseMediaEntirely ? 'release-media-entirely' : 'release-media'
       }
     });
-    this.ep.destroy()
-      .then(() => this.ep = null)
-      .catch((err) => this.logger.error({err}, 'SingleDialer:releaseMediaToSBC: Error destroying endpoint'));
+    try {
+      await this.ep.destroy();
+    } catch (err) {
+      this.logger.error({err}, 'SingleDialer:releaseMediaToSBC: Error destroying endpoint');
+    }
+    this.ep = null;
   }
 
-  async reAnchorMedia() {
+  async reAnchorMedia(currentMediaRoute = MediaPath.PartialMedia) {
     assert(this.dlg && this.dlg.connected && !this.ep);
+
+    this.logger.debug('SingleDialer:reAnchorMedia: re-anchoring media after partial media');
     this.ep = await this.ms.createEndpoint({remoteSdp: this.dlg.remote.sdp});
     this._configMsEndpoint();
     await this.dlg.modify(this.ep.local.sdp, {
@@ -477,6 +535,11 @@ class SingleDialer extends Emitter {
         'X-Reason': 'anchor-media'
       }
     });
+
+    if (currentMediaRoute === MediaPath.NoMedia) {
+      this.logger.debug('SingleDialer:reAnchorMedia: repoint endpoint after no media');
+      await this.ep.modify(this.dlg.remote.sdp);
+    }
   }
 
   _notifyCallStatusChange({callStatus, sipStatus, sipReason, duration}) {
@@ -507,7 +570,8 @@ function placeOutdial({
 }) {
   const myOpts = deepcopy(opts);
   const sd = new SingleDialer({
-    logger, sbcAddress, target, myOpts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask, onHoldMusic
+    logger, sbcAddress, target, opts: myOpts, application, callInfo,
+    accountInfo, rootSpan, startSpan, dialTask, onHoldMusic
   });
   sd.exec(srf, ms, myOpts);
   return sd;

lib/utils/sbc-pinger.js

@@ -1,5 +1,5 @@
 const assert = require('assert');
-const uuidv4 = require('uuid-random');
+const crypto = require('crypto');
 const {LifeCycleEvents, FS_UUID_SET_NAME} = require('./constants');
 const Emitter = require('events');
 const debug = require('debug')('jambonz:feature-server');
@@ -8,7 +8,7 @@ const {
   JAMBONES_SBCS,
   K8S,
   K8S_SBC_SIP_SERVICE_NAME,
-  AWS_SNS_TOPIC_ARM,
+  AWS_SNS_TOPIC_ARN,
   OPTIONS_PING_INTERVAL,
   AWS_REGION,
   NODE_ENV,
@@ -35,7 +35,7 @@ module.exports = (logger) => {
   // listen for SNS lifecycle changes
   let lifecycleEmitter = new Emitter();
   let dryUpCalls = false;
-  if (AWS_SNS_TOPIC_ARM && AWS_REGION) {
+  if (AWS_SNS_TOPIC_ARN && AWS_REGION) {
 
     (async function() {
       try {
@@ -46,12 +46,24 @@ module.exports = (logger) => {
             const {srf} = require('../..');
             srf.locals.publicIp = publicIp;
           })
-          .on(LifeCycleEvents.ScaleIn, () => {
+          .on(LifeCycleEvents.ScaleIn, async() => {
             logger.info('AWS scale-in notification: begin drying up calls');
             dryUpCalls = true;
             lifecycleEmitter.operationalState = LifeCycleEvents.ScaleIn;
 
             const {srf} = require('../..');
+            const {writeSystemAlerts} = srf.locals;
+            if (writeSystemAlerts) {
+              const {SystemState, FEATURE_SERVER} = require('./constants');
+              await writeSystemAlerts({
+                system_component: FEATURE_SERVER,
+                state : SystemState.GracefulShutdownInProgress,
+                fields : {
+                  detail: `feature-server with process_id ${process.pid} shutdown in progress`,
+                  host: srf.locals?.ipv4
+                }
+              });
+            }
             pingProxies(srf);
 
             // if we have zero calls, we can complete the scale-in right
@@ -118,7 +130,7 @@ module.exports = (logger) => {
       logger.info('disabling OPTIONS pings since we are running as a kubernetes service');
       const {srf} = require('../..');
       const {addToSet} = srf.locals.dbHelpers;
-      const uuid = srf.locals.fsUUID = uuidv4();
+      const uuid = srf.locals.fsUUID = crypto.randomUUID();
 
       /* in case redis is restarted, re-insert our key every so often */
       setInterval(() => {

lib/utils/sdp-utils.js

@@ -35,6 +35,12 @@ const makeOpusFirst = (sdp) => {
   }
   return sdpTransform.write(parsedSdp);
 };
+const removeVideoSdp = (sdp) => {
+  const parsedSdp = sdpTransform.parse(sdp);
+  // Filter out video media sections, keeping only non-video media
+  parsedSdp.media = parsedSdp.media.filter((media) => media.type !== 'video');
+  return sdpTransform.write(parsedSdp);
+};
 
 const extractSdpMedia = (sdp) => {
   const parsedSdp1 = sdpTransform.parse(sdp);
@@ -54,5 +60,6 @@ module.exports = {
   mergeSdpMedia,
   extractSdpMedia,
   isOpusFirst,
-  makeOpusFirst
+  makeOpusFirst,
+  removeVideoSdp
 };

lib/utils/siprec-utils.js

@@ -1,5 +1,5 @@
 const xmlParser = require('xml2js').parseString;
-const uuidv4 = require('uuid-random');
+const crypto = require('crypto');
 const parseUri = require('drachtio-srf').parseUri;
 const transform = require('sdp-transform');
 const debug = require('debug')('jambonz:feature-server');
@@ -52,7 +52,7 @@ const parseSiprecPayload = (req, logger) => {
       const arr = /^([^]+)(m=[^]+?)(m=[^]+?)$/.exec(sdp);
       opts.sdp1 = `${arr[1]}${arr[2]}`;
       opts.sdp2 = `${arr[1]}${arr[3]}\r\n`;
-      opts.sessionId = uuidv4();
+      opts.sessionId = crypto.randomUUID();
       logger.info({ payload: req.payload }, 'SIPREC payload with no metadata (e.g. Cisco NBR)');
       resolve(opts);
     } else if (!sdp || !meta) {
@@ -64,7 +64,7 @@ const parseSiprecPayload = (req, logger) => {
       if (err) { throw err; }
 
       opts.recordingData = result ;
-      opts.sessionId = uuidv4() ;
+      opts.sessionId = crypto.randomUUID();
 
       const arr = /^([^]+)(m=[^]+?)(m=[^]+?)$/.exec(sdp) ;
       opts.sdp1 = `${arr[1]}${arr[2]}` ;

lib/utils/sticky-event-emitter.js

@@ -0,0 +1,70 @@
+const EventEmitter = require('events');
+
+/**
+ * A specialized EventEmitter that caches the most recent event emissions.
+ * When new listeners are added, they immediately receive the most recent
+ * event if it was previously emitted. This is useful for handling state
+ * changes where late subscribers need to know the current state.
+ *
+ * Features:
+ * - Caches the most recent emission for each event type
+ * - New listeners immediately receive the cached event if available
+ * - Supports both regular (on) and one-time (once) listeners
+ * - Maintains compatibility with Node's EventEmitter interface
+ */
+class StickyEventEmitter extends EventEmitter {
+  constructor() {
+    super();
+    this._eventCache = new Map();
+    this._onceListeners = new Map(); // For storing once listeners if needed
+  }
+  destroy() {
+    this._eventCache.clear();
+    this._onceListeners.clear();
+    this.removeAllListeners();
+  }
+  emit(event, ...args) {
+    // Store the event and its args
+    this._eventCache.set(event, args);
+
+    // If there are any 'once' listeners waiting, call them
+    if (this._onceListeners.has(event)) {
+      const listeners = this._onceListeners.get(event);
+      for (const listener of listeners) {
+        listener(...args);
+      }
+      if (this.onSuccess) {
+        this.onSuccess();
+      }
+      this._onceListeners.delete(event);
+    }
+
+    return super.emit(event, ...args);
+  }
+
+  on(event, listener) {
+    if (this._eventCache.has(event)) {
+      listener(...this._eventCache.get(event));
+    }
+    return super.on(event, listener);
+  }
+
+  once(event, listener) {
+    if (this._eventCache.has(event)) {
+      listener(...this._eventCache.get(event));
+      if (this.onSuccess) {
+        this.onSuccess();
+      }
+    } else {
+      // Store listener in case emit comes before
+      if (!this._onceListeners.has(event)) {
+        this._onceListeners.set(event, []);
+      }
+      this._onceListeners.get(event).push(listener);
+      super.once(event, listener); // Also attach to native once
+    }
+    return this;
+  }
+}
+
+module.exports = StickyEventEmitter;

lib/utils/transcription-utils.js

@@ -30,6 +30,7 @@ const stickyVars = {
     'DEEPGRAM_SPEECH_TIER',
     'DEEPGRAM_SPEECH_MODEL',
     'DEEPGRAM_SPEECH_ENABLE_SMART_FORMAT',
+    'DEEPGRAM_SPEECH_ENABLE_NO_DELAY',
     'DEEPGRAM_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION',
     'DEEPGRAM_SPEECH_PROFANITY_FILTER',
     'DEEPGRAM_SPEECH_REDACT',
@@ -43,7 +44,9 @@ const stickyVars = {
     'DEEPGRAM_SPEECH_UTTERANCE_END_MS',
     'DEEPGRAM_SPEECH_VAD_TURNOFF',
     'DEEPGRAM_SPEECH_TAG',
-    'DEEPGRAM_SPEECH_MODEL_VERSION'
+    'DEEPGRAM_SPEECH_MODEL_VERSION',
+    'DEEPGRAM_SPEECH_FILLER_WORDS',
+    'DEEPGRAM_SPEECH_KEYTERMS',
   ],
   aws: [
     'AWS_VOCABULARY_NAME',
@@ -104,12 +107,26 @@ const stickyVars = {
     'ASSEMBLYAI_API_KEY',
     'ASSEMBLYAI_WORD_BOOST'
   ],
+  voxist: [
+    'VOXIST_API_KEY',
+  ],
   speechmatics: [
     'SPEECHMATICS_API_KEY',
     'SPEECHMATICS_HOST',
     'SPEECHMATICS_PATH',
     'SPEECHMATICS_SPEECH_HINTS',
-  ]
+    'SPEECHMATICS_TRANSLATION_LANGUAGES',
+    'SPEECHMATICS_TRANSLATION_PARTIALS'
+  ],
+  openai: [
+    'OPENAI_API_KEY',
+    'OPENAI_MODEL',
+    'OPENAI_INPUT_AUDIO_NOISE_REDUCTION',
+    'OPENAI_TURN_DETECTION_TYPE',
+    'OPENAI_TURN_DETECTION_THRESHOLD',
+    'OPENAI_TURN_DETECTION_PREFIX_PADDING_MS',
+    'OPENAI_TURN_DETECTION_SILENCE_DURATION_MS',
+  ],
 };
 
 /**
@@ -183,7 +200,10 @@ const selectDefaultGoogleModel = (task, language, version) => {
     (useV2 ? 'long' : 'latest_long');
 };
 const consolidateTranscripts = (bufferedTranscripts, channel, language, vendor) => {
-  if (bufferedTranscripts.length === 1) return bufferedTranscripts[0];
+  if (bufferedTranscripts.length === 1) {
+    bufferedTranscripts[0].is_final = true;
+    return bufferedTranscripts[0];
+  }
   let totalConfidence = 0;
   const finalTranscript = bufferedTranscripts.reduce((acc, evt) => {
     totalConfidence += evt.alternatives[0].confidence;
@@ -203,7 +223,7 @@ const consolidateTranscripts = (bufferedTranscripts, channel, language, vendor)
     const lastChar = acc.alternatives[0].transcript.slice(-1);
     const firstChar = newTranscript.charAt(0);
 
-    if (lastChar.match(/\d/) && firstChar.match(/\d/)) {
+    if (vendor === 'speechmatics' || (lastChar.match(/\d/) && firstChar.match(/\d/))) {
       acc.alternatives[0].transcript += newTranscript;
     } else {
       acc.alternatives[0].transcript += ` ${newTranscript}`;
@@ -293,13 +313,18 @@ const normalizeDeepgram = (evt, channel, language, shortUtterance) => {
       confidence: alt.confidence,
       transcript: alt.transcript,
     }));
-
+  /**
+   * Some models (nova-2-general) return the detected language in the
+   * alternatives.languages array if the language is set as multi.
+   * If the language is detected, we use it as the language_code.
+   */
+  const detectedLanguage = evt.channel?.alternatives?.[0]?.languages?.[0];
   /**
    * note difference between is_final and speech_final in Deepgram:
    * https://developers.deepgram.com/docs/understand-endpointing-interim-results
    */
   return {
-    language_code: language,
+    language_code: detectedLanguage || language,
     channel_tag: channel,
     is_final: shortUtterance ? evt.is_final : evt.speech_final,
     alternatives: alternatives.length ? [alternatives[0]] : [],
@@ -511,6 +536,25 @@ const normalizeAssemblyAi = (evt, channel, language) => {
   };
 };
 
+const normalizeVoxist = (evt, channel, language) => {
+  const copy = JSON.parse(JSON.stringify(evt));
+  return {
+    language_code: language,
+    channel_tag: channel,
+    is_final: evt.type === 'final',
+    alternatives: [
+      {
+        confidence: 1.00,
+        transcript: evt.text,
+      }
+    ],
+    vendor: {
+      name: 'voxist',
+      evt: copy
+    }
+  };
+};
+
 const normalizeSpeechmatics = (evt, channel, language) => {
   const copy = JSON.parse(JSON.stringify(evt));
   const is_final = evt.message === 'AddTranscript';
@@ -536,6 +580,35 @@ const normalizeSpeechmatics = (evt, channel, language) => {
   return obj;
 };
 
+const calculateConfidence = (logprobsArray) => {
+  // Sum the individual log probabilities
+  const totalLogProb = logprobsArray.reduce((sum, tokenInfo) => sum + tokenInfo.logprob, 0);
+
+  // Convert the total log probability back to a regular probability
+  const confidence = Math.exp(totalLogProb);
+  return confidence;
+};
+
+const normalizeOpenAI = (evt, channel, language) => {
+  const copy = JSON.parse(JSON.stringify(evt));
+  const obj = {
+    language_code: language,
+    channel_tag: channel,
+    is_final: true,
+    alternatives: [
+      {
+        transcript: evt.transcript,
+        confidence: evt.logprobs ? calculateConfidence(evt.logprobs) : 1.0,
+      }
+    ],
+    vendor: {
+      name: 'openai',
+      evt: copy
+    }
+  };
+  return obj;
+};
+
 module.exports = (logger) => {
   const normalizeTranscription = (evt, vendor, channel, language, shortUtterance, punctuation) => {
 
@@ -561,10 +634,14 @@ module.exports = (logger) => {
         return normalizeCobalt(evt, channel, language);
       case 'assemblyai':
         return normalizeAssemblyAi(evt, channel, language, shortUtterance);
+      case 'voxist':
+        return normalizeVoxist(evt, channel, language);
       case 'verbio':
         return normalizeVerbio(evt, channel, language);
       case 'speechmatics':
         return normalizeSpeechmatics(evt, channel, language);
+      case 'openai':
+        return normalizeOpenAI(evt, channel, language);
       default:
         if (vendor.startsWith('custom:')) {
           return normalizeCustom(evt, channel, language, vendor);
@@ -699,6 +776,8 @@ module.exports = (logger) => {
         //azureSttEndpointId overrides sttCredentials.custom_stt_endpoint
         ...(rOpts.azureSttEndpointId &&
           {AZURE_SERVICE_ENDPOINT_ID: rOpts.azureSttEndpointId}),
+        ...(azureOptions.speechRecognitionMode &&
+          {AZURE_RECOGNITION_MODE: azureOptions.speechRecognitionMode}),
       };
     }
     else if ('nuance' === vendor) {
@@ -750,7 +829,7 @@ module.exports = (logger) => {
       };
     }
     else if ('deepgram' === vendor) {
-      let {model} = rOpts;
+      let model = rOpts.deepgramOptions?.model || rOpts.model;
       const {deepgramOptions = {}} = rOpts;
       const deepgramUri = deepgramOptions.deepgramSttUri || sttCredentials.deepgram_stt_uri;
       const useTls = deepgramOptions.deepgramSttUseTls || sttCredentials.deepgram_stt_use_tls;
@@ -772,6 +851,8 @@ module.exports = (logger) => {
           {DEEPGRAM_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION: 1},
         ...(deepgramOptions.smartFormatting) &&
           {DEEPGRAM_SPEECH_ENABLE_SMART_FORMAT: 1},
+        ...(deepgramOptions.noDelay) &&
+          {DEEPGRAM_SPEECH_ENABLE_NO_DELAY: 1},
         ...(deepgramOptions.profanityFilter) &&
           {DEEPGRAM_SPEECH_PROFANITY_FILTER: 1},
         ...(deepgramOptions.redact) &&
@@ -807,7 +888,11 @@ module.exports = (logger) => {
         ...(deepgramOptions.tag) &&
           {DEEPGRAM_SPEECH_TAG: deepgramOptions.tag},
         ...(deepgramOptions.version) &&
-          {DEEPGRAM_SPEECH_MODEL_VERSION: deepgramOptions.version}
+          {DEEPGRAM_SPEECH_MODEL_VERSION: deepgramOptions.version},
+        ...(deepgramOptions.fillerWords) &&
+          {DEEPGRAM_SPEECH_FILLER_WORDS: deepgramOptions.fillerWords},
+        ...((Array.isArray(deepgramOptions.keyterms) && deepgramOptions.keyterms.length > 0) &&
+          {DEEPGRAM_SPEECH_KEYTERMS: deepgramOptions.keyterms.join(',')})
       };
     }
     else if ('soniox' === vendor) {
@@ -916,6 +1001,43 @@ module.exports = (logger) => {
           {ASSEMBLYAI_WORD_BOOST: JSON.stringify(rOpts.hints)})
       };
     }
+    else if ('voxist' === vendor) {
+      opts = {
+        ...opts,
+        ...(sttCredentials.api_key) &&
+          {VOXIST_API_KEY: sttCredentials.api_key},
+      };
+    }
+    else if ('openai' === vendor) {
+      const {openaiOptions = {}} = rOpts;
+      const model = openaiOptions.model || rOpts.model || sttCredentials.model_id || 'whisper-1';
+      const apiKey = openaiOptions.apiKey || sttCredentials.api_key;
+
+      opts = {
+        OPENAI_MODEL: model,
+        OPENAI_API_KEY: apiKey,
+        ...opts,
+        ...(openaiOptions.prompt && {OPENAI_PROMPT: openaiOptions.prompt}),
+        ...(openaiOptions.input_audio_noise_reduction &&
+          {OPENAI_INPUT_AUDIO_NOISE_REDUCTION: openaiOptions.input_audio_noise_reduction}),
+      };
+
+      if (openaiOptions.turn_detection) {
+        opts = {
+          ...opts,
+          OPENAI_TURN_DETECTION_TYPE: openaiOptions.turn_detection.type,
+          ...(openaiOptions.turn_detection.threshold && {
+            OPENAI_TURN_DETECTION_THRESHOLD: openaiOptions.turn_detection.threshold
+          }),
+          ...(openaiOptions.turn_detection.prefix_padding_ms && {
+            OPENAI_TURN_DETECTION_PREFIX_PADDING_MS: openaiOptions.turn_detection.prefix_padding_ms
+          }),
+          ...(openaiOptions.turn_detection.silence_duration_ms && {
+            OPENAI_TURN_DETECTION_SILENCE_DURATION_MS: openaiOptions.turn_detection.silence_duration_ms
+          }),
+        };
+      }
+    }
     else if ('verbio' === vendor) {
       const {verbioOptions = {}} = rOpts;
       opts = {
@@ -937,11 +1059,48 @@ module.exports = (logger) => {
       };
     }
     else if ('speechmatics' === vendor) {
+      const {speechmaticsOptions = {}} = rOpts;
       opts = {
         ...opts,
         ...(sttCredentials.api_key) && {SPEECHMATICS_API_KEY: sttCredentials.api_key},
         ...(sttCredentials.speechmatics_stt_uri) && {SPEECHMATICS_HOST: sttCredentials.speechmatics_stt_uri},
         ...(rOpts.hints?.length > 0 && {SPEECHMATICS_SPEECH_HINTS: rOpts.hints.join(',')}),
+        ...(speechmaticsOptions.translation_config &&
+          {
+            SPEECHMATICS_TRANSLATION_LANGUAGES: speechmaticsOptions.translation_config.target_languages.join(','),
+            SPEECHMATICS_TRANSLATION_PARTIALS: speechmaticsOptions.translation_config.enable_partials ? 1 : 0
+          }
+        ),
+        ...(speechmaticsOptions.transcription_config?.domain &&
+          {SPEECHMATICS_DOMAIN: speechmaticsOptions.transcription_config.domain}),
+        ...{SPEECHMATICS_MAX_DELAY: speechmaticsOptions.transcription_config?.max_delay || 0.7},
+        ...{SPEECHMATICS_MAX_DELAY_MODE: speechmaticsOptions.transcription_config?.max_delay_mode || 'flexible'},
+        ...(speechmaticsOptions.transcription_config?.diarization &&
+            {SPEECHMATICS_DIARIZATION: speechmaticsOptions.transcription_config.diarization}),
+        ...(speechmaticsOptions.transcription_config?.speaker_diarization_config?.speaker_sensitivity &&
+          {SPEECHMATICS_DIARIZATION_SPEAKER_SENSITIVITY:
+            speechmaticsOptions.transcription_config.speaker_diarization_config.speaker_sensitivity}),
+        ...(speechmaticsOptions.transcription_config?.speaker_diarization_config?.max_speakers &&
+          {SPEECHMATICS_DIARIZATION_MAX_SPEAKERS:
+            speechmaticsOptions.transcription_config.speaker_diarization_config.max_speakers}),
+        ...(speechmaticsOptions.transcription_config?.output_locale &&
+          {SPEECHMATICS_OUTPUT_LOCALE: speechmaticsOptions.transcription_config.output_locale}),
+        ...(speechmaticsOptions.transcription_config?.punctuation_overrides?.permitted_marks &&
+          {SPEECHMATICS_PUNCTUATION_ALLOWED:
+            speechmaticsOptions.transcription_config.punctuation_overrides.permitted_marks.join(',')}),
+        ...(speechmaticsOptions.transcription_config?.punctuation_overrides?.sensitivity &&
+          {SPEECHMATICS_PUNCTUATION_SENSITIVITY:
+            speechmaticsOptions.transcription_config?.punctuation_overrides?.sensitivity}),
+        ...(speechmaticsOptions.transcription_config?.operating_point &&
+          {SPEECHMATICS_OPERATING_POINT: speechmaticsOptions.transcription_config.operating_point}),
+        ...(speechmaticsOptions.transcription_config?.enable_entities &&
+          {SPEECHMATICS_ENABLE_ENTTIES: speechmaticsOptions.transcription_config.enable_entities}),
+        ...(speechmaticsOptions.transcription_config?.audio_filtering_config?.volume_threshold &&
+          {SPEECHMATICS_VOLUME_THRESHOLD:
+            speechmaticsOptions.transcription_config.audio_filtering_config.volume_threshold}),
+        ...(speechmaticsOptions.transcription_config?.transcript_filtering_config?.remove_disfluencies &&
+          {SPEECHMATICS_REMOVE_DISFLUENCIES:
+            speechmaticsOptions.transcription_config.transcript_filtering_config.remove_disfluencies})
       };
     }
     else if (vendor.startsWith('custom:')) {

lib/utils/tts-streaming-buffer.js

@@ -0,0 +1,452 @@
+const Emitter = require('events');
+const assert = require('assert');
+const {
+  TtsStreamingEvents,
+  TtsStreamingConnectionStatus
+} = require('../utils/constants');
+
+const MAX_CHUNK_SIZE = 1800;
+const HIGH_WATER_BUFFER_SIZE = 1000;
+const LOW_WATER_BUFFER_SIZE = 200;
+const TIMEOUT_RETRY_MSECS = 1000; // 1 second
+
+
+const isWhitespace = (str) => /^\s*$/.test(str);
+
+/**
+ * Each queue item is an object:
+ *   - { type: 'text', value: '…' } for text tokens.
+ *   - { type: 'flush' } for a flush command.
+ */
+class TtsStreamingBuffer extends Emitter {
+  constructor(cs) {
+    super();
+    this.cs = cs;
+    this.logger = cs.logger;
+
+    // Use an array to hold our structured items.
+    this.queue = [];
+    // Track total number of characters in text items.
+    this.bufferedLength = 0;
+    this.eventHandlers = [];
+    this._isFull = false;
+    this._connectionStatus = TtsStreamingConnectionStatus.NotConnected;
+    this.timer = null;
+    // Record the last time the text buffer was updated.
+    this.lastUpdateTime = 0;
+  }
+
+  get isEmpty() {
+    return this.queue.length === 0;
+  }
+
+  get size() {
+    return this.bufferedLength;
+  }
+
+  get isFull() {
+    return this._isFull;
+  }
+
+  get ep() {
+    return this.cs?.ep;
+  }
+
+  async start() {
+    assert.ok(
+      this._connectionStatus === TtsStreamingConnectionStatus.NotConnected,
+      'TtsStreamingBuffer:start already started, or has failed'
+    );
+
+    this.vendor = this.cs.getTsStreamingVendor();
+    if (!this.vendor) {
+      this.logger.info('TtsStreamingBuffer:start No TTS streaming vendor configured');
+      throw new Error('No TTS streaming vendor configured');
+    }
+
+    this.logger.info(`TtsStreamingBuffer:start Connecting to TTS streaming with vendor ${this.vendor}`);
+
+    this._connectionStatus = TtsStreamingConnectionStatus.Connecting;
+    try {
+      if (this.eventHandlers.length === 0) this._initHandlers(this.ep);
+      await this._api(this.ep, [this.ep.uuid, 'connect']);
+    } catch (err) {
+      this.logger.info({ err }, 'TtsStreamingBuffer:start Error connecting to TTS streaming');
+      this._connectionStatus = TtsStreamingConnectionStatus.Failed;
+    }
+  }
+
+  stop() {
+    clearTimeout(this.timer);
+    this.removeCustomEventListeners();
+    if (this.ep) {
+      this._api(this.ep, [this.ep.uuid, 'close'])
+        .catch((err) =>
+          this.logger.info({ err }, 'TtsStreamingBuffer:stop Error closing TTS streaming')
+        );
+    }
+    this.timer = null;
+    this.queue = [];
+    this.bufferedLength = 0;
+    this._connectionStatus = TtsStreamingConnectionStatus.NotConnected;
+  }
+
+  /**
+   * Buffer new text tokens.
+   */
+  async bufferTokens(tokens) {
+    if (this._connectionStatus === TtsStreamingConnectionStatus.Failed) {
+      this.logger.info('TtsStreamingBuffer:bufferTokens TTS streaming connection failed, rejecting request');
+      return { status: 'failed', reason: `connection to ${this.vendor} failed` };
+    }
+
+    if (0 === this.bufferedLength && isWhitespace(tokens)) {
+      this.logger.debug({tokens}, 'TtsStreamingBuffer:bufferTokens discarded whitespace tokens');
+      return { status: 'ok' };
+    }
+
+    const displayedTokens = tokens.length <= 40 ? tokens : tokens.substring(0, 40);
+    const totalLength = tokens.length;
+
+    if (this.bufferedLength + totalLength > HIGH_WATER_BUFFER_SIZE) {
+      this.logger.info(
+        `TtsStreamingBuffer throttling: buffer is full, rejecting request to buffer ${totalLength} tokens`
+      );
+      if (!this._isFull) {
+        this._isFull = true;
+        this.emit(TtsStreamingEvents.Pause);
+      }
+      return { status: 'failed', reason: 'full' };
+    }
+
+    this.logger.debug(
+      `TtsStreamingBuffer:bufferTokens "${displayedTokens}" (length: ${totalLength})`
+    );
+    this.queue.push({ type: 'text', value: tokens });
+    this.bufferedLength += totalLength;
+    // Update the last update time each time new text is buffered.
+    this.lastUpdateTime = Date.now();
+
+    await this._feedQueue();
+    return { status: 'ok' };
+  }
+
+  /**
+   * Insert a flush command. If no text is queued, flush immediately.
+   * Otherwise, append a flush marker so that all text preceding it will be sent
+   * (regardless of sentence boundaries) before the flush is issued.
+   */
+  flush() {
+    if (this._connectionStatus === TtsStreamingConnectionStatus.Connecting) {
+      this.logger.debug('TtsStreamingBuffer:flush TTS stream is not quite ready - wait for connect');
+      if (this.queue.length === 0 || this.queue[this.queue.length - 1].type !== 'flush') {
+        this.queue.push({ type: 'flush' });
+      }
+      return;
+    }
+    else if (this._connectionStatus === TtsStreamingConnectionStatus.Connected) {
+      if (this.isEmpty) {
+        this._doFlush();
+      }
+      else {
+        if (this.queue[this.queue.length - 1].type !== 'flush') {
+          this.queue.push({ type: 'flush' });
+          this.logger.debug('TtsStreamingBuffer:flush added flush marker to queue');
+        }
+      }
+    }
+    else {
+      this.logger.debug(
+        `TtsStreamingBuffer:flush TTS stream is not connected, status: ${this._connectionStatus}`
+      );
+    }
+  }
+
+  clear() {
+    this.logger.debug('TtsStreamingBuffer:clear');
+    if (this._connectionStatus !== TtsStreamingConnectionStatus.Connected) return;
+    clearTimeout(this.timer);
+    this._api(this.ep, [this.ep.uuid, 'clear']).catch((err) =>
+      this.logger.info({ err }, 'TtsStreamingBuffer:clear Error clearing TTS streaming')
+    );
+    this.queue = [];
+    this.bufferedLength = 0;
+    this.timer = null;
+    this._isFull = false;
+  }
+
+  /**
+   * Process the queue in two phases.
+   *
+   * Phase 1: Look for flush markers. When a flush marker is found (even if not at the very front),
+   *   send all text tokens that came before it immediately (ignoring sentence boundaries)
+   *   and then send the flush command. Repeat until there are no flush markers left.
+   *
+   * Phase 2: With the remaining queue (now containing only text items), accumulate text
+   *   up to MAX_CHUNK_SIZE and use sentence-boundary logic to determine a chunk.
+   *   Then, remove the exact tokens (or portions thereof) that were consumed.
+   */
+  async _feedQueue(handlingTimeout = false) {
+    this.logger.debug({ queue: this.queue }, 'TtsStreamingBuffer:_feedQueue');
+    try {
+      if (!this.cs.isTtsStreamOpen || !this.ep) {
+        this.logger.debug('TtsStreamingBuffer:_feedQueue TTS stream is not open or no endpoint available');
+        return;
+      }
+      if (
+        this._connectionStatus === TtsStreamingConnectionStatus.NotConnected ||
+        this._connectionStatus === TtsStreamingConnectionStatus.Failed
+      ) {
+        this.logger.debug('TtsStreamingBuffer:_feedQueue TTS stream is not connected');
+        return;
+      }
+
+      // --- Phase 1: Process flush markers ---
+      // Process any flush marker that isn’t in the very first position.
+      let flushIndex = this.queue.findIndex((item, idx) => item.type === 'flush' && idx > 0);
+      while (flushIndex !== -1) {
+        let flushText = '';
+        // Accumulate all text tokens preceding the flush marker.
+        for (let i = 0; i < flushIndex; i++) {
+          if (this.queue[i].type === 'text') {
+            flushText += this.queue[i].value;
+          }
+        }
+        // Remove those text items.
+        for (let i = 0; i < flushIndex; i++) {
+          const item = this.queue.shift();
+          if (item.type === 'text') {
+            this.bufferedLength -= item.value.length;
+          }
+        }
+        // Remove the flush marker (now at the front).
+        if (this.queue.length > 0 && this.queue[0].type === 'flush') {
+          this.queue.shift();
+        }
+        // Immediately send all accumulated text (ignoring sentence boundaries).
+        if (flushText.length > 0) {
+          const modifiedFlushText = flushText.replace(/\n\n/g, '\n \n');
+          try {
+            await this._api(this.ep, [this.ep.uuid, 'send', modifiedFlushText]);
+          } catch (err) {
+            this.logger.info({ err, flushText }, 'TtsStreamingBuffer:_feedQueue Error sending TTS chunk');
+          }
+        }
+        // Send the flush command.
+        await this._doFlush();
+
+        flushIndex = this.queue.findIndex((item, idx) => item.type === 'flush' && idx > 0);
+      }
+
+      // If a flush marker is at the very front, process it.
+      while (this.queue.length > 0 && this.queue[0].type === 'flush') {
+        this.queue.shift();
+        await this._doFlush();
+      }
+
+      // --- Phase 2: Process remaining text tokens ---
+      if (this.queue.length === 0) {
+        this._removeTimer();
+        return;
+      }
+
+      // Accumulate contiguous text tokens (from the front) up to MAX_CHUNK_SIZE.
+      let combinedText = '';
+      for (const item of this.queue) {
+        if (item.type !== 'text') break;
+        combinedText += item.value;
+        if (combinedText.length >= MAX_CHUNK_SIZE) break;
+      }
+      if (combinedText.length === 0) {
+        this._removeTimer();
+        return;
+      }
+
+      const limit = Math.min(MAX_CHUNK_SIZE, combinedText.length);
+      let chunkEnd = findSentenceBoundary(combinedText, limit);
+      if (chunkEnd <= 0) {
+        if (handlingTimeout) {
+          chunkEnd = findWordBoundary(combinedText, limit);
+          if (chunkEnd <= 0) {
+            this._setTimerIfNeeded();
+            return;
+          }
+        } else {
+          this._setTimerIfNeeded();
+          return;
+        }
+      }
+      const chunk = combinedText.slice(0, chunkEnd);
+
+      // Now we iterate over the queue items
+      // and deduct their lengths until we've accounted for chunkEnd characters.
+      let remaining = chunkEnd;
+      let tokensProcessed = 0;
+      for (let i = 0; i < this.queue.length; i++) {
+        const token = this.queue[i];
+        if (token.type !== 'text') break;
+        if (remaining >= token.value.length) {
+          remaining -= token.value.length;
+          tokensProcessed = i + 1;
+        } else {
+          // Partially consumed token: update its value to remove the consumed part.
+          token.value = token.value.slice(remaining);
+          tokensProcessed = i;
+          remaining = 0;
+          break;
+        }
+      }
+      // Remove the fully consumed tokens from the front of the queue.
+      this.queue.splice(0, tokensProcessed);
+      this.bufferedLength -= chunkEnd;
+
+      const modifiedChunk = chunk.replace(/\n\n/g, '\n \n');
+      this.logger.debug(`TtsStreamingBuffer:_feedQueue sending chunk to tts: ${modifiedChunk}`);
+
+      try {
+        await this._api(this.ep, [this.ep.uuid, 'send', modifiedChunk]);
+      } catch (err) {
+        this.logger.info({ err, chunk }, 'TtsStreamingBuffer:_feedQueue Error sending TTS chunk');
+      }
+
+      if (this._isFull && this.bufferedLength <= LOW_WATER_BUFFER_SIZE) {
+        this.logger.info('TtsStreamingBuffer throttling: buffer is no longer full - resuming');
+        this._isFull = false;
+        this.emit(TtsStreamingEvents.Resume);
+      }
+
+      return this._feedQueue();
+    } catch (err) {
+      this.logger.info({ err }, 'TtsStreamingBuffer:_feedQueue Error sending TTS chunk');
+      this.queue = [];
+      this.bufferedLength = 0;
+    }
+  }
+
+  async _api(ep, args) {
+    const apiCmd = `uuid_${this.vendor.startsWith('custom:') ? 'custom' : this.vendor}_tts_streaming`;
+    const res = await ep.api(apiCmd, `^^|${args.join('|')}`);
+    if (!res.body?.startsWith('+OK')) {
+      this.logger.info({ args }, `Error calling ${apiCmd}: ${res.body}`);
+      throw new Error(`Error calling ${apiCmd}: ${res.body}`);
+    }
+  }
+
+  _doFlush() {
+    return this._api(this.ep, [this.ep.uuid, 'flush'])
+      .then(() => this.logger.debug('TtsStreamingBuffer:_doFlush sent flush command'))
+      .catch((err) =>
+        this.logger.info(
+          { err },
+          `TtsStreamingBuffer:_doFlush Error flushing TTS streaming: ${JSON.stringify(err)}`
+        )
+      );
+  }
+
+  async _onConnect(vendor) {
+    this.logger.info(`TtsStreamingBuffer:_onConnect streaming tts connection made to ${vendor} successful`);
+    this._connectionStatus = TtsStreamingConnectionStatus.Connected;
+    if (this.queue.length > 0) {
+      await this._feedQueue();
+    }
+  }
+
+  _onConnectFailure(vendor) {
+    this.logger.info(`TtsStreamingBuffer:_onConnectFailure streaming tts connection failed to ${vendor}`);
+    this._connectionStatus = TtsStreamingConnectionStatus.Failed;
+    this.queue = [];
+    this.bufferedLength = 0;
+    this.emit(TtsStreamingEvents.ConnectFailure, { vendor });
+  }
+
+  _setTimerIfNeeded() {
+    if (this.bufferedLength > 0 && !this.timer) {
+      this.logger.debug({queue: this.queue},
+        `TtsStreamingBuffer:_setTimerIfNeeded setting timer because ${this.bufferedLength} buffered`);
+      this.timer = setTimeout(this._onTimeout.bind(this), TIMEOUT_RETRY_MSECS);
+    }
+  }
+
+  _removeTimer() {
+    if (this.timer) {
+      this.logger.debug('TtsStreamingBuffer:_removeTimer clearing timer');
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+  }
+
+  _onTimeout() {
+    this.logger.debug('TtsStreamingBuffer:_onTimeout Timeout waiting for sentence boundary');
+    this.timer = null;
+    // Check if new text has been added since the timer was set.
+    const now = Date.now();
+    if (now - this.lastUpdateTime < TIMEOUT_RETRY_MSECS) {
+      this.logger.debug('TtsStreamingBuffer:_onTimeout New text received recently; postponing flush.');
+      this._setTimerIfNeeded();
+      return;
+    }
+    this._feedQueue(true);
+  }
+
+  _onTtsEmpty(vendor) {
+    this.emit(TtsStreamingEvents.Empty, { vendor });
+  }
+
+  addCustomEventListener(ep, event, handler) {
+    this.eventHandlers.push({ ep, event, handler });
+    ep.addCustomEventListener(event, handler);
+  }
+
+  removeCustomEventListeners() {
+    this.eventHandlers.forEach((h) => h.ep.removeCustomEventListener(h.event, h.handler));
+  }
+
+  _initHandlers(ep) {
+    [
+      'deepgram',
+      'cartesia',
+      'elevenlabs',
+      'rimelabs',
+      'custom'
+    ].forEach((vendor) => {
+      const eventClassName = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}TtsStreamingEvents`;
+      const eventClass = require('../utils/constants')[eventClassName];
+      if (!eventClass) throw new Error(`Event class for vendor ${vendor} not found`);
+
+      this.addCustomEventListener(ep, eventClass.Connect, this._onConnect.bind(this, vendor));
+      this.addCustomEventListener(ep, eventClass.ConnectFailure, this._onConnectFailure.bind(this, vendor));
+      this.addCustomEventListener(ep, eventClass.Empty, this._onTtsEmpty.bind(this, vendor));
+    });
+  }
+}
+
+const findSentenceBoundary = (text, limit) => {
+  // Look for punctuation or double newline that signals sentence end.
+  const sentenceEndRegex = /[.!?](?=\s|$)|\n\n/g;
+  let lastSentenceBoundary = -1;
+  let match;
+  while ((match = sentenceEndRegex.exec(text)) && match.index < limit) {
+    const precedingText = text.slice(0, match.index).trim();
+    if (precedingText.length > 0) {
+      if (
+        match[0] === '\n\n' ||
+        (match.index === 0 || !/\d$/.test(text[match.index - 1]))
+      ) {
+        lastSentenceBoundary = match.index + (match[0] === '\n\n' ? 2 : 1);
+      }
+    }
+  }
+  return lastSentenceBoundary;
+};
+
+const findWordBoundary = (text, limit) => {
+  const wordBoundaryRegex = /\s+/g;
+  let lastWordBoundary = -1;
+  let match;
+  while ((match = wordBoundaryRegex.exec(text)) && match.index < limit) {
+    lastWordBoundary = match.index;
+  }
+  return lastWordBoundary;
+};
+
+module.exports = TtsStreamingBuffer;

lib/utils/ws-requestor.js

@@ -1,7 +1,7 @@
 const assert = require('assert');
 const BaseRequestor = require('./base-requestor');
 const short = require('short-uuid');
-const {HookMsgTypes} = require('./constants.json');
+const {HookMsgTypes, WS_CLOSE_CODES} = require('./constants.json');
 const Websocket = require('ws');
 const snakeCaseKeys = require('./snakecase-keys');
 const {
@@ -12,6 +12,20 @@ const {
   JAMBONES_WS_MAX_PAYLOAD,
   HTTP_USER_AGENT_HEADER
 } = require('../config');
+const MTYPE_WANTS_ACK = [
+  'call:status',
+  'verb:status',
+  'jambonz:error',
+  'llm:event',
+  'llm:tool-call',
+  'tts:streaming-event',
+  'tts:tokens-result',
+];
+const MTYPE_NO_DATA = [
+  'llm:tool-output',
+  'tts:flush',
+  'tts:clear'
+];
 
 class WsRequestor extends BaseRequestor {
   constructor(logger, account_sid, hook, secret) {
@@ -44,7 +58,7 @@ class WsRequestor extends BaseRequestor {
   async request(type, hook, params, httpHeaders = {}) {
     assert(HookMsgTypes.includes(type));
     const url = hook.url || hook;
-    const wantsAck = !['call:status', 'verb:status', 'jambonz:error', 'llm:event', 'llm:tool-call'].includes(type);
+    const wantsAck = !MTYPE_WANTS_ACK.includes(type);
 
     if (this.maliciousClient) {
       this.logger.info({url: this.url}, 'WsRequestor:request - discarding msg to malicious client');
@@ -118,8 +132,8 @@ class WsRequestor extends BaseRequestor {
     assert(this.ws);
 
     /* prepare and send message */
-    let payload = params ? snakeCaseKeys(params, ['customerData', 'sip']) : null;
-    if (type === 'session:new') this._sessionData = payload;
+    let payload = params ? snakeCaseKeys(params, ['customerData', 'sip', 'env_vars', 'args']) : null;
+    if (type === 'session:new' || type === 'session:adulting') this._sessionData = payload;
     if (type === 'session:reconnect') payload = this._sessionData;
     assert.ok(url, 'WsRequestor:request url was not provided');
 
@@ -132,7 +146,9 @@ class WsRequestor extends BaseRequestor {
       type,
       msgid,
       call_sid: this.call_sid,
-      hook: ['verb:hook', 'session:redirect', 'llm:event', 'llm:tool-call'].includes(type) ? url : undefined,
+      hook: [
+        'verb:hook', 'dial:confirm', 'session:redirect', 'llm:event', 'llm:tool-call'
+      ].includes(type) ? url : undefined,
       data: {...payload},
       ...b3
     };
@@ -247,13 +263,13 @@ class WsRequestor extends BaseRequestor {
     }
   }
 
-  close() {
+  close(code = WS_CLOSE_CODES.NormalClosure) {
     this.closedGracefully = true;
-    this.logger.debug('WsRequestor:close closing socket');
+    this.logger.debug(`WsRequestor:close closing socket with code ${code}`);
     this._stopPingTimer();
     try {
       if (this.ws) {
-        this.ws.close(1000);
+        this.ws.close(code);
         this.ws.removeAllListeners();
         this.ws = null;
       }
@@ -408,7 +424,7 @@ class WsRequestor extends BaseRequestor {
 
         case 'command':
           assert.ok(command, 'command property not supplied');
-          assert.ok(data || command === 'llm:tool-output', 'data property not supplied');
+          assert.ok(data || MTYPE_NO_DATA.includes(command), 'data property not supplied');
           this._recvCommand(msgid, command, call_sid, queueCommand, tool_call_id, data);
           break;
 
@@ -417,6 +433,21 @@ class WsRequestor extends BaseRequestor {
       }
     } catch (err) {
       this.logger.info({err, content}, 'WsRequestor:_onMessage - invalid incoming message');
+      const params = {
+        msg: 'InvalidMessage',
+        details: err.message,
+        content: Buffer.from(content).toString('utf-8')
+      };
+      const {writeAlerts, AlertType} = this.Alerter;
+      writeAlerts({
+        account_sid: this.account_sid,
+        alert_type: AlertType.INVALID_APP_PAYLOAD,
+        target_sid: this.call_sid,
+        message: err.message,
+
+      }).catch((err) => this.logger.info({err}, 'Error generating alert for invalid message'));
+      this.request('jambonz:error', '/error', params)
+        .catch((err) => this.logger.debug({err}, 'WsRequestor:_onMessage - Error sending'));
     }
   }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jambonz-feature-server",
-  "version": "0.9.2",
+  "version": "0.9.4",
   "main": "app.js",
   "engines": {
     "node": ">= 18.x"
@@ -27,14 +27,15 @@
   "dependencies": {
     "@aws-sdk/client-auto-scaling": "^3.549.0",
     "@aws-sdk/client-sns": "^3.549.0",
-    "@jambonz/db-helpers": "^0.9.6",
+    "@jambonz/db-helpers": "^0.9.12",
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/mw-registrar": "^0.2.7",
-    "@jambonz/realtimedb-helpers": "^0.8.8",
-    "@jambonz/speech-utils": "^0.1.20",
+    "@jambonz/realtimedb-helpers": "^0.8.13",
+    "@jambonz/speech-utils": "^0.2.10",
     "@jambonz/stats-collector": "^0.1.10",
-    "@jambonz/time-series": "^0.2.9",
-    "@jambonz/verb-specifications": "^0.0.83",
+    "@jambonz/time-series": "^0.2.13",
+    "@jambonz/verb-specifications": "^0.0.104",
+    "@modelcontextprotocol/sdk": "^1.9.0",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/exporter-jaeger": "^1.23.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
@@ -47,8 +48,8 @@
     "bent": "^7.3.12",
     "debug": "^4.3.4",
     "deepcopy": "^2.1.0",
-    "drachtio-fsmrf": "^3.0.45",
-    "drachtio-srf": "^4.5.35",
+    "drachtio-fsmrf": "^4.0.3",
+    "drachtio-srf": "^5.0.5",
     "express": "^4.19.2",
     "express-validator": "^7.0.1",
     "moment": "^2.30.1",
@@ -56,12 +57,11 @@
     "pino": "^8.20.0",
     "polly-ssml-split": "^0.1.0",
     "proxyquire": "^2.1.3",
-    "sdp-transform": "^2.14.2",
+    "sdp-transform": "^2.15.0",
     "short-uuid": "^5.1.0",
     "sinon": "^17.0.1",
     "to-snake-case": "^1.0.0",
-    "undici": "^6.20.0",
-    "uuid-random": "^1.3.2",
+    "undici": "^7.5.0",
     "verify-aws-sns-signature": "^0.1.0",
     "ws": "^8.18.0",
     "xml2js": "^0.6.2"

test/create-call-test.js

@@ -222,3 +222,62 @@ test('test create-call app_json', async(t) => {
     t.error(err);
   }
 });
+
+test('test create-call timeLimit', async(t) => {
+  clearModule.all();
+  const {srf, disconnect} = require('../app');
+
+  try {
+    await connect(srf);
+
+
+    // GIVEN
+    let from = 'create-call-app-json';
+    let account_sid = 'bb845d4b-83a9-4cde-a6e9-50f3743bab3f';
+
+    // Give UAS app time to come up
+    const p = sippUac('uas.xml', '172.38.0.10', from);
+    await waitFor(1000);
+
+    const startTime = Date.now();
+
+    const app_json = `[
+      {
+        "verb": "pause",
+        "length": 7
+      }
+    ]`;
+
+    const post = bent('http://127.0.0.1:3000/', 'POST', 'json', 201);
+    post('v1/createCall', {
+      'account_sid':account_sid,
+      "call_hook": {
+        "url": "http://127.0.0.1:3100/",
+        "method": "POST",
+        "username": "username",
+        "password": "password"
+      },
+      app_json,
+      "from": from,
+      "to": {
+        "type": "phone",
+        "number": "15583084809"
+      },
+      "timeLimit": 1,
+      "speech_recognizer_vendor": "google",
+      "speech_recognizer_language": "en"
+    });
+
+    //THEN
+    await p;
+    const endTime = Date.now();
+
+    t.ok(endTime - startTime < 2000, 'create-call: timeLimit is respected');
+
+    disconnect();
+  } catch (err) {
+    console.log(`error received: ${err}`);
+    disconnect();
+    t.error(err);
+  }
+});

test/db/jambones-sql.sql

@@ -351,6 +351,8 @@ speech_credential_sid CHAR(36) NOT NULL,
 model VARCHAR(512) NOT NULL,
 reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
 name VARCHAR(64) NOT NULL,
+voice_cloning_key MEDIUMTEXT,
+use_voice_cloning_key BOOLEAN DEFAULT false,
 PRIMARY KEY (google_custom_voice_sid)
 );
 

test/dial-tests.js

@@ -3,9 +3,8 @@ const { sippUac } = require('./sipp')('test_fs');
 const bent = require('bent');
 const getJSON = bent('json')
 const clearModule = require('clear-module');
-const {provisionCallHook} = require('./utils')
-
-const sleepFor = (ms) => new Promise((r) => setTimeout(r, ms));
+const {provisionCallHook} = require('./utils');
+const { sleepFor } = require('../lib/utils/helpers');
 
 process.on('unhandledRejection', (reason, p) => {
   console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);

test/docker-compose-testbed.yaml

@@ -42,7 +42,7 @@ services:
         ipv4_address: 172.38.0.7
 
   drachtio:
-    image: drachtio/drachtio-server:0.8.25-rc8
+    image: drachtio/drachtio-server:0.8.26
     restart: always
     command: drachtio --contact "sip:*;transport=udp" --mtu 4096 --address 0.0.0.0 --port 9022
     ports:
@@ -57,7 +57,7 @@ services:
         condition: service_healthy
 
   freeswitch:
-    image: drachtio/drachtio-freeswitch-mrf:latest
+    image: drachtio/drachtio-freeswitch-mrf:0.9.2-4
     restart: always
     command: freeswitch --rtp-range-start 20000 --rtp-range-end 20100
     environment:

test/queue-test.js

@@ -3,6 +3,7 @@ const { sippUac } = require('./sipp')('test_fs');
 const clearModule = require('clear-module');
 const {provisionCallHook, provisionActionHook, provisionAnyHook} = require('./utils');
 const bent = require('bent');
+const { sleepFor } = require('../lib/utils/helpers');
 const getJSON = bent('json');
 
 process.on('unhandledRejection', (reason, p) => {
@@ -17,8 +18,6 @@ function connect(connectable) {
   });
 }
 
-const sleepFor = (ms) => new Promise((resolve) => setTimeout(() => resolve(), ms));
-
 test('\'enqueue-dequeue\' tests', async(t) => {
 
   clearModule.all();

test/sip-refer-tests.js

@@ -3,10 +3,9 @@ const { sippUac } = require('./sipp')('test_fs');
 const clearModule = require('clear-module');
 const {provisionCallHook, provisionCustomHook, provisionActionHook} = require('./utils')
 const bent = require('bent');
+const { sleepFor } = require('../lib/utils/helpers');
 const getJSON = bent('json')
 
-const sleepFor = async(ms) => new Promise(resolve => setTimeout(resolve, ms));
-
 process.on('unhandledRejection', (reason, p) => {
   console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
 });

tracer.js

@@ -25,29 +25,38 @@ module.exports = (serviceName) => {
       }),
     });
 
-    let exporter;
+    const exporters = [];
+
     if (OTEL_EXPORTER_JAEGER_AGENT_HOST  || OTEL_EXPORTER_JAEGER_ENDPOINT) {
-      exporter = new JaegerExporter();
-    }
-    else if (OTEL_EXPORTER_ZIPKIN_URL) {
-      exporter = new ZipkinExporter({url:OTEL_EXPORTER_ZIPKIN_URL});
-    }
-    else {
-      exporter = new OTLPTraceExporter({
-        url: OTEL_EXPORTER_COLLECTOR_URL
-      });
+      exporters.push(new JaegerExporter());
     }
 
-    provider.addSpanProcessor(new BatchSpanProcessor(exporter, {
-      // The maximum queue size. After the size is reached spans are dropped.
-      maxQueueSize: 100,
-      // The maximum batch size of every export. It must be smaller or equal to maxQueueSize.
-      maxExportBatchSize: 10,
-      // The interval between two consecutive exports
-      scheduledDelayMillis: 500,
-      // How long the export can run before it is cancelled
-      exportTimeoutMillis: 30000,
-    }));
+    if (OTEL_EXPORTER_ZIPKIN_URL) {
+      exporters.push(new ZipkinExporter({url:OTEL_EXPORTER_ZIPKIN_URL}));
+    }
+
+    if (OTEL_EXPORTER_ZIPKIN_URL) {
+      exporters.push(new ZipkinExporter({url:OTEL_EXPORTER_ZIPKIN_URL}));
+    }
+
+    if (OTEL_EXPORTER_COLLECTOR_URL) {
+      exporters.push(new OTLPTraceExporter({
+        url: OTEL_EXPORTER_COLLECTOR_URL
+      }));
+    }
+
+    exporters.forEach((element) => {
+      provider.addSpanProcessor(new BatchSpanProcessor(element, {
+        // The maximum queue size. After the size is reached spans are dropped.
+        maxQueueSize: 100,
+        // The maximum batch size of every export. It must be smaller or equal to maxQueueSize.
+        maxExportBatchSize: 10,
+        // The interval between two consecutive exports
+        scheduledDelayMillis: 500,
+        // How long the export can run before it is cancelled
+        exportTimeoutMillis: 30000,
+      }));
+    });
 
     // Initialize the OpenTelemetry APIs to use the NodeTracerProvider bindings
     provider.register();