create outbound dial from webhook ws (#581)

* wip, create outbound dial from webhook ws

* wip, create outbound dial from webhook ws

* clean

lib/config.js

@@ -120,6 +120,7 @@ const HTTP_TIMEOUT = 10000;
 const HTTP_PROXY_IP = process.env.JAMBONES_HTTP_PROXY_IP;
 const HTTP_PROXY_PORT = process.env.JAMBONES_HTTP_PROXY_PORT;
 const HTTP_PROXY_PROTOCOL = process.env.JAMBONES_HTTP_PROXY_PROTOCOL || 'http';
+const HTTP_USER_AGENT_HEADER = process.env.JAMBONES_HTTP_USER_AGENT_HEADER || 'jambonz';
 
 const OPTIONS_PING_INTERVAL = parseInt(process.env.OPTIONS_PING_INTERVAL, 10) || 30000;
 
@@ -129,6 +130,10 @@ const JAMBONZ_RECORD_WS_PASSWORD = process.env.JAMBONZ_RECORD_WS_PASSWORD || pro
 const JAMBONZ_DISABLE_DIAL_PAI_HEADER = process.env.JAMBONZ_DISABLE_DIAL_PAI_HEADER || false;
 const JAMBONES_DISABLE_DIRECT_P2P_CALL = process.env.JAMBONES_DISABLE_DIRECT_P2P_CALL || false;
 
+const JAMBONES_EAGERLY_PRE_CACHE_AUDIO = process.env.JAMBONES_EAGERLY_PRE_CACHE_AUDIO;
+
+const JAMBONES_USE_FREESWITCH_TIMER_FD = process.env.JAMBONES_USE_FREESWITCH_TIMER_FD;
+
 module.exports = {
   JAMBONES_MYSQL_HOST,
   JAMBONES_MYSQL_USER,
@@ -151,6 +156,7 @@ module.exports = {
   JAMBONES_API_BASE_URL,
   JAMBONES_TIME_SERIES_HOST,
   JAMBONES_INJECT_CONTENT,
+  JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
   JAMBONES_ESL_LISTEN_ADDRESS,
   JAMBONES_SBCS,
   JAMBONES_OTEL_ENABLED,
@@ -193,6 +199,7 @@ module.exports = {
   HTTP_PROXY_IP,
   HTTP_PROXY_PORT,
   HTTP_PROXY_PROTOCOL,
+  HTTP_USER_AGENT_HEADER,
   OPTIONS_PING_INTERVAL,
   RESPONSE_TIMEOUT_MS,
   JAMBONES_WS_HANDSHAKE_TIMEOUT_MS,
@@ -208,5 +215,6 @@ module.exports = {
   JAMBONZ_RECORD_WS_USERNAME,
   JAMBONZ_RECORD_WS_PASSWORD,
   JAMBONZ_DISABLE_DIAL_PAI_HEADER,
-  JAMBONES_DISABLE_DIRECT_P2P_CALL
+  JAMBONES_DISABLE_DIRECT_P2P_CALL,
+  JAMBONES_USE_FREESWITCH_TIMER_FD
 };

lib/middleware.js

@@ -97,7 +97,7 @@ module.exports = function(srf, logger) {
     if (req.has('X-MS-Teams-Tenant-FQDN')) req.locals.msTeamsTenant = req.get('X-MS-Teams-Tenant-FQDN');
     if (req.has('X-Cisco-Recording-Participant')) {
       const ciscoParticipants = req.get('X-Cisco-Recording-Participant');
-      const regex = /sip:[\d]+@[\d]+\.[\d]+\.[\d]+\.[\d]+/g;
+      const regex = /sip:[a-zA-Z0-9]+@[a-zA-Z0-9.-_]+/g;
       const sipURIs = ciscoParticipants.match(regex);
       logger.info(`X-Cisco-Recording-Participant : ${sipURIs} `);
       if (sipURIs && sipURIs.length > 0) {
@@ -382,7 +382,7 @@ module.exports = function(srf, logger) {
           },
           recognizer: {
             vendor: app.speech_recognizer_vendor,
-            ...(app.speech_synthesis_label && {label: app.speech_synthesis_label}),
+            ...(app.speech_recognizer_label && {label: app.speech_recognizer_label}),
             language: app.speech_recognizer_language,
             ...(app.fallback_speech_recognizer_vendor && {fallback_vendor: app.fallback_speech_recognizer_vendor}),
             ...(app.fallback_speech_recognizer_label && {fallback_label: app.fallback_speech_recognizer_label}),

lib/session/call-session.js

@@ -19,8 +19,11 @@ const HttpRequestor = require('../utils/http-requestor');
 const WsRequestor = require('../utils/ws-requestor');
 const {
   JAMBONES_INJECT_CONTENT,
+  JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
   AWS_REGION,
+  JAMBONES_USE_FREESWITCH_TIMER_FD
 } = require('../config');
+const bent = require('bent');
 const BackgroundTaskManager = require('../utils/background-task-manager');
 const BADPRECONDITIONS = 'preconditions not met';
 const CALLER_CANCELLED_ERR_MSG = 'Response not sent due to unknown transaction';
@@ -688,7 +691,7 @@ class CallSession extends Emitter {
         (type === 'stt' && credential.use_for_stt)
       )) {
         this.logger.info(
-          `Speech vendor: ${credential.vendor} ${credential.label ? `, label: ${credential.label}` : ''} selected`);
+          `${type}: ${credential.vendor} ${credential.label ? `, label: ${credential.label}` : ''}  `);
         if ('google' === vendor) {
           if (type === 'tts' && !credential.tts_tested_ok ||
             type === 'stt' && !credential.stt_tested_ok) {
@@ -749,7 +752,9 @@ class CallSession extends Emitter {
         else if ('deepgram' === vendor) {
           return {
             speech_credential_sid: credential.speech_credential_sid,
-            api_key: credential.api_key
+            api_key: credential.api_key,
+            deepgram_stt_uri: credential.deepgram_stt_uri,
+            deepgram_stt_use_tls: credential.deepgram_stt_use_tls
           };
         }
         else if ('soniox' === vendor) {
@@ -827,6 +832,10 @@ class CallSession extends Emitter {
       const task = this.tasks.shift();
       this.logger.info(`CallSession:exec starting task #${stackNum}:${taskNum}: ${task.name}`);
       this._notifyTaskStatus(task, {event: 'starting'});
+      // Register verbhook span wait for end
+      task.on('VerbHookSpanWaitForEnd', ({span}) => {
+        this.verbHookSpan = span;
+      });
       try {
         const resources = await this._evaluatePreconditions(task);
         let skip = false;
@@ -969,6 +978,64 @@ class CallSession extends Emitter {
     }
   }
 
+  /**
+   * perform live call control - create rest:dial
+   * @param {obj} opts create call options
+   */
+  async _lccCallDial(opts) {
+    try {
+      const restDialUrl = `${this.srf.locals.serviceUrl}/v1/createCall`;
+      await this.transformInputIfRequired(opts);
+      const resp = bent('POST', 'json', 201)(restDialUrl, opts);
+      this.logger.info(resp.body, 'successfully create outbound call');
+      return resp.body;
+    } catch (err) {
+      if (err.json) {
+        err.body = await err.json();
+      }
+      this.logger.error(err, 'failed to create outbound call from ' + this.callSid);
+      this._notifyTaskError(err.body);
+    }
+  }
+
+  async transformInputIfRequired(opts) {
+    const {
+      lookupAppBySid
+    }  = this.srf.locals.dbHelpers;
+    opts.account_sid = this.accountSid;
+
+    if (opts.application_sid) {
+      this.logger.debug(`Callsession:_validateCreateCall retrieving application ${opts.application_sid}`);
+      const application = await lookupAppBySid(opts.application_sid);
+      Object.assign(opts, {
+        call_hook: application.call_hook,
+        app_json: application.app_json,
+        call_status_hook: application.call_status_hook,
+        speech_synthesis_vendor: application.speech_synthesis_vendor,
+        speech_synthesis_language: application.speech_synthesis_language,
+        speech_synthesis_voice: application.speech_synthesis_voice,
+        speech_recognizer_vendor: application.speech_recognizer_vendor,
+        speech_recognizer_language: application.speech_recognizer_language
+      });
+      this.logger.debug({opts, application}, 'Callsession:_validateCreateCall augmented with application settings');
+    }
+
+    if (typeof opts.call_hook === 'string') {
+      const url = opts.call_hook;
+      opts.call_hook = {
+        url,
+        method: 'POST'
+      };
+    }
+    if (typeof opts.call_status_hook === 'string') {
+      const url = opts.call_status_hook;
+      opts.call_status_hook = {
+        url,
+        method: 'POST'
+      };
+    }
+  }
+
   /**
    * perform live call control -- set a new call_hook
    * @param {object} opts
@@ -1029,11 +1096,17 @@ class CallSession extends Emitter {
       const t = normalizeJambones(this.logger, tasks).map((tdata) => makeTask(this.logger, tdata));
       this.logger.info({tasks: listTaskNames(t)}, 'CallSession:_lccCallHook new task list');
       this.replaceApplication(t);
+      if (this.wakeupResolver) {
+        //this.logger.debug({resolution}, 'CallSession:_onCommand - got commands, waking up..');
+        this.wakeupResolver({reason: 'lcc: new tasks'});
+        this.wakeupResolver = null;
+      }
     }
     else {
       /* we started a new app on the child leg, but nothing given for parent so hang him up */
       this.currentTask.kill(this);
     }
+    this._endVerbHookSpan();
   }
 
   /**
@@ -1059,6 +1132,9 @@ class CallSession extends Emitter {
    * @param {string} opts.transcribe_status - 'pause' or 'resume'
   */
   async _lccTranscribeStatus(opts) {
+    if (this.backgroundTaskManager.isTaskRunning('transcribe')) {
+      this.backgroundTaskManager.getTask('transcribe').updateTranscribe(opts.transcribe_status);
+    }
     const task = this.currentTask;
     if (!task || ![TaskName.Dial, TaskName.Transcribe].includes(task.name)) {
       return this.logger.info(`CallSession:_lccTranscribeStatus - invalid transcribe_status in task ${task.name}`);
@@ -1305,6 +1381,7 @@ Duration=${duration} `
   kill(onBackgroundGatherBargein = false) {
     if (this.isConfirmCallSession) this.logger.debug('CallSession:kill (ConfirmSession)');
     else this.logger.info('CallSession:kill');
+    this._endVerbHookSpan();
     if (this.currentTask) {
       this.currentTask.kill(this);
       this.currentTask = null;
@@ -1330,6 +1407,35 @@ Duration=${duration} `
     this.taskIdx = 0;
   }
 
+  _preCacheAudio(newTasks) {
+    for (const task of newTasks) {
+      if (task.name === TaskName.Config && task.hasSynthesizer) {
+        /* if they change synthesizer settings don't try to precache */
+        break;
+      }
+      if (task.name === TaskName.Say) {
+        /* identify vendor language, voice, and label */
+        const vendor = task.synthesizer.vendor && task.synthesizer.vendor !== 'default' ?
+          task.synthesizer.vendor :
+          this.speechSynthesisVendor;
+        const language = task.synthesizer.language && task.synthesizer.language !== 'default' ?
+          task.synthesizer.language :
+          this.speechSynthesisLanguage ;
+        const voice =  task.synthesizer.voice && task.synthesizer.voice !== 'default' ?
+          task.synthesizer.voice :
+          this.speechSynthesisVoice;
+        const label = task.synthesizer.label && task.synthesizer.label !== 'default' ?
+          task.synthesizer.label :
+          this.speechSynthesisLabel;
+
+        this.logger.info({vendor, language, voice, label},
+          'CallSession:_preCacheAudio - precaching audio for future prompt');
+        task._synthesizeWithSpecificVendor(this, this.ep, {vendor, language, voice, label, preCache: true})
+          .catch((err) => this.logger.error(err, 'CallSession:_preCacheAudio - error precaching audio'));
+      }
+    }
+  }
+
   /**
    * Append tasks to the current execution stack UNLESS there is a gather in the stack.
    * in that case, insert the tasks before the gather AND if the tasks include
@@ -1376,10 +1482,11 @@ Duration=${duration} `
 
   _onCommand({msgid, command, call_sid, queueCommand, data}) {
     this.logger.info({msgid, command, queueCommand, data}, 'CallSession:_onCommand - received command');
-    const resolution = {reason: 'received command', queue: queueCommand, command};
+    let resolution;
     switch (command) {
       case 'redirect':
         if (Array.isArray(data)) {
+          this._endVerbHookSpan();
           const t = normalizeJambones(this.logger, data)
             .map((tdata) => makeTask(this.logger, tdata));
           if (!queueCommand) {
@@ -1387,13 +1494,16 @@ Duration=${duration} `
             this.replaceApplication(t);
           }
           else if (JAMBONES_INJECT_CONTENT) {
+            if (JAMBONES_EAGERLY_PRE_CACHE_AUDIO) this._preCacheAudio(t);
             this._injectTasks(t);
             this.logger.info({tasks: listTaskNames(this.tasks)}, 'CallSession:_onCommand - updated task list');
           }
           else {
+            if (JAMBONES_EAGERLY_PRE_CACHE_AUDIO) this._preCacheAudio(t);
             this.tasks.push(...t);
             this.logger.info({tasks: listTaskNames(this.tasks)}, 'CallSession:_onCommand - updated task list');
           }
+          resolution = {reason: 'received command, new tasks', queue: queueCommand, command};
           resolution.command = listTaskNames(t);
         }
         else this._lccCallHook(data);
@@ -1403,6 +1513,10 @@ Duration=${duration} `
         this._lccCallStatus(data);
         break;
 
+      case 'dial':
+        this._lccCallDial(data);
+        break;
+
       case 'mute:status':
         this._lccMuteStatus(call_sid, data);
         break;
@@ -1444,7 +1558,7 @@ Duration=${duration} `
       default:
         this.logger.info(`CallSession:_onCommand - invalid command ${command}`);
     }
-    if (this.wakeupResolver) {
+    if (this.wakeupResolver && resolution) {
       //this.logger.debug({resolution}, 'CallSession:_onCommand - got commands, waking up..');
       this.wakeupResolver(resolution);
       this.wakeupResolver = null;
@@ -1712,7 +1826,7 @@ Duration=${duration} `
           res.send(200, {body: this.ep.local.sdp});
         }
         else {
-          if (this.currentTask.name === TaskName.Dial && this.currentTask.isOnHold) {
+          if (this.currentTask.name === TaskName.Dial && this.currentTask.isOnHoldEnabled) {
             this.logger.info('onholdMusic reINVITE after media has been released');
             await this.currentTask.handleReinviteAfterMediaReleased(req, res);
           } else {
@@ -1988,8 +2102,12 @@ Duration=${duration} `
   }
 
   _configMsEndpoint() {
-    if (this.onHoldMusic) {
-      this.ep.set({hold_music: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`});
+    const opts = {
+      ...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
+      ...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'})
+    };
+    if (Object.keys(opts).length > 0) {
+      this.ep.set(opts);
     }
   }
 
@@ -2041,6 +2159,13 @@ Duration=${duration} `
   stopBackgroundTask(type) {
     this.backgroundTaskManager.stop(type);
   }
+
+  _endVerbHookSpan() {
+    if (this.verbHookSpan) {
+      this.verbHookSpan.end();
+      this.verbHookSpan = null;
+    }
+  }
 }
 
 module.exports = CallSession;

lib/tasks/dequeue.js

@@ -73,7 +73,8 @@ class TaskDequeue extends Task {
         try {
           let url;
           if (this.callSid) {
-            url = await retrieveByPatternSortedSet(this.queueName, `*${this.callSid}`);
+            const r = await retrieveByPatternSortedSet(this.queueName, `*${this.callSid}`);
+            url = r[0];
           } else {
             url = await retrieveFromSortedSet(this.queueName);
           }

lib/tasks/dial.js

@@ -100,6 +100,7 @@ class TaskDial extends Task {
     this.referHook = this.data.referHook;
     this.dtmfHook = this.data.dtmfHook;
     this.proxy = this.data.proxy;
+    this.tag = this.data.tag;
 
     if (this.dtmfHook) {
       const {parentDtmfCollector, childDtmfCollector} = parseDtmfOptions(logger, this.data.dtmfCapture || {});
@@ -138,13 +139,14 @@ class TaskDial extends Task {
 
   get name() { return TaskName.Dial; }
 
-  get isOnHold() {
-    return this.isIncomingLegHold || this.isOutgoingLegHold;
+  get isOnHoldEnabled() {
+    return !!this.data.onHoldHook;
   }
 
   get canReleaseMedia() {
     const keepAnchor = this.data.anchorMedia ||
     this.cs.isBackGroundListen ||
+    this.cs.onHoldMusic ||
     ANCHOR_MEDIA_ALWAYS ||
     this.listenTask ||
     this.transcribeTask ||
@@ -570,7 +572,8 @@ class TaskDial extends Task {
           accountInfo: cs.accountInfo,
           rootSpan: cs.rootSpan,
           startSpan: this.startSpan.bind(this),
-          dialTask: this
+          dialTask: this,
+          onHoldMusic: this.cs.onHoldMusic
         });
         this.dials.set(sd.callSid, sd);
 
@@ -679,22 +682,43 @@ class TaskDial extends Task {
   async _onReinvite(req, res) {
     try {
       let isHandled = false;
-      if (this.cs.onHoldMusic) {
-        if (isOnhold(req.body) && !this.epOther && !this.ep) {
-          await this.cs.handleReinviteAfterMediaReleased(req, res);
-          // Onhold but media is already released
-          // reconnect A Leg and Response B leg
-          await this.reAnchorMedia(this.cs, this.sd);
-          this.isOutgoingLegHold = true;
+      if (this.isOnHoldEnabled) {
+        if (isOnhold(req.body)) {
+          this.logger.debug('Dial: _onReinvite receive hold Request');
+          if (!this.epOther && !this.ep) {
+            this.logger.debug(`Dial: _onReinvite receive hold Request,
+ media already released, reconnect media server`);
+            // update caller leg for new SDP from callee.
+            await this.cs.handleReinviteAfterMediaReleased(req, res);
+            // Freeswitch media is released, reconnect
+            await this.reAnchorMedia(this.cs, this.sd);
+            this.isOutgoingLegHold = true;
+          } else {
+            this.logger.debug('Dial: _onReinvite receive hold Request, update SDP');
+            const newSdp = await this.ep.modify(req.body);
+            res.send(200, {body: newSdp});
+          }
           isHandled = true;
-          this._onHoldHook();
-        } else if (!isOnhold(req.body) && this.epOther && this.ep && this.isOutgoingLegHold && this.canReleaseMedia) {
-          // Offhold, time to release media
-          const newSdp = await this.ep.modify(req.body);
-          await res.send(200, {body: newSdp});
-          await this._releaseMedia(this.cs, this.sd);
+          // Media already connected, ask for onHoldHook
+          this._onHoldHook(req);
+        } else if (!isOnhold(req.body)) {
+          this.logger.debug('Dial: _onReinvite receive unhold Request');
+          if (this.epOther && this.ep && this.isOutgoingLegHold && this.canReleaseMedia) {
+            this.logger.debug('Dial: _onReinvite receive unhold Request, release media');
+            // Offhold, time to release media
+            const newSdp = await this.ep.modify(req.body);
+            await res.send(200, {body: newSdp});
+            await this._releaseMedia(this.cs, this.sd);
+            this.isOutgoingLegHold = false;
+          } else {
+            this.logger.debug('Dial: _onReinvite receive unhold Request, update media server');
+            const newSdp = await this.ep.modify(req.body);
+            res.send(200, {body: newSdp});
+          }
+          if (this._onHoldSession) {
+            this._onHoldSession.kill();
+          }
           isHandled = true;
-          this.isOutgoingLegHold = false;
         }
       }
       if (!isHandled) {
@@ -811,23 +835,34 @@ class TaskDial extends Task {
     this.epOther = cs.ep;
   }
 
+  // Handle RE-INVITE hold from caller leg.
   async handleReinviteAfterMediaReleased(req, res) {
     let isHandled = false;
-    if (isOnhold(req.body) && !this.epOther && !this.ep) {
-      const sdp = await this.dlg.modify(req.body);
-      res.send(200, {body: sdp});
-      // Onhold but media is already released
-      await this.reAnchorMedia(this.cs, this.sd);
-      isHandled = true;
-      this.isIncomingLegHold = true;
-      this._onHoldHook();
-    } else if (!isOnhold(req.body) && this.epOther && this.ep && this.isIncomingLegHold && this.canReleaseMedia) {
-      // Offhold, time to release media
-      const newSdp = await this.epOther.modify(req.body);
-      await res.send(200, {body: newSdp});
-      await this._releaseMedia(this.cs, this.sd);
-      isHandled = true;
-      this.isIncomingLegHold = false;
+    if (this.isOnHoldEnabled) {
+      if (isOnhold(req.body)) {
+        if (!this.epOther && !this.ep) {
+          // update callee leg for new SDP from caller.
+          const sdp = await this.dlg.modify(req.body);
+          res.send(200, {body: sdp});
+          // Onhold but media is already released, reconnect
+          await this.reAnchorMedia(this.cs, this.sd);
+          isHandled = true;
+          this.isIncomingLegHold = true;
+        }
+        this._onHoldHook(req);
+      } else if (!isOnhold(req.body)) {
+        if (this.epOther && this.ep && this.isIncomingLegHold && this.canReleaseMedia) {
+          // Offhold, time to release media
+          const newSdp = await this.epOther.modify(req.body);
+          await res.send(200, {body: newSdp});
+          await this._releaseMedia(this.cs, this.sd);
+          isHandled = true;
+        }
+        this.isIncomingLegHold = false;
+        if (this._onHoldSession) {
+          this._onHoldSession.kill();
+        }
+      }
     }
 
     if (!isHandled) {
@@ -846,7 +881,7 @@ class TaskDial extends Task {
       });
   }
 
-  async _onHoldHook(allowed = [TaskName.Play, TaskName.Say, TaskName.Pause]) {
+  async _onHoldHook(req, allowed = [TaskName.Play, TaskName.Say, TaskName.Pause]) {
     if (this.data.onHoldHook) {
       // send silence for keep Voice quality
       await this.epOther.play('silence_stream://500');
@@ -856,7 +891,13 @@ class TaskDial extends Task {
           const b3 = this.getTracingPropagation();
           const httpHeaders = b3 && {b3};
           const json = await this.cs.application.requestor.
-            request('verb:hook', this.data.onHoldHook, this.cs.callInfo.toJSON(), httpHeaders);
+            request('verb:hook', this.data.onHoldHook, {
+              ...this.cs.callInfo.toJSON(),
+              hold_detail: {
+                from: req.get('From'),
+                to: req.get('To')
+              }
+            }, httpHeaders);
           const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
           allowedTasks = tasks.filter((t) => allowed.includes(t.name));
           if (tasks.length !== allowedTasks.length) {
@@ -865,7 +906,7 @@ class TaskDial extends Task {
           }
           this.logger.debug(`DialTask:_onHoldHook: executing ${tasks.length} tasks`);
           if (tasks.length) {
-            this._playSession = new ConfirmCallSession({
+            this._onHoldSession = new ConfirmCallSession({
               logger: this.logger,
               application: this.cs.application,
               dlg: this.isIncomingLegHold ? this.dlg : this.cs.dlg,
@@ -875,12 +916,12 @@ class TaskDial extends Task {
               tasks,
               rootSpan: this.cs.rootSpan
             });
-            await this._playSession.exec();
-            this._playSession = null;
+            await this._onHoldSession.exec();
+            this._onHoldSession = null;
           }
         } catch (error) {
           this.logger.info(error, 'DialTask:_onHoldHook: failed retrieving waitHook');
-          this._playSession = null;
+          this._onHoldSession = null;
           break;
         }
       } while (allowedTasks && allowedTasks.length > 0 && !this.killed && this.isOnHold);

lib/tasks/gather.js

@@ -30,6 +30,10 @@ class TaskGather extends SttTask {
       'speechTimeout', 'timeout', 'say', 'play'
     ].forEach((k) => this[k] = this.data[k]);
 
+    // gather default input is digits
+    if (!this.input) {
+      this.input = ['digits'];
+    }
     /* when collecting dtmf, bargein on dtmf is true unless explicitly set to false */
     if (this.dtmfBargein !== false  && this.input.includes('digits')) this.dtmfBargein = true;
 
@@ -301,7 +305,7 @@ class TaskGather extends SttTask {
       if (this.data.recognizer?.deepgramOptions?.shortUtterance) this.shortUtterance = true;
     }
 
-    const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.data.recognizer);
+    const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.language, this.data.recognizer);
     switch (this.vendor) {
       case 'google':
         this.bugname = `${this.bugname_prefix}google_transcribe`;
@@ -351,9 +355,6 @@ class TaskGather extends SttTask {
         this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
         this.addCustomEventListener(ep, DeepgramTranscriptionEvents.ConnectFailure,
           this._onVendorConnectFailure.bind(this, cs, ep));
-
-        /* if app sets deepgramOptions.utteranceEndMs they essentially want continuous asr */
-        if (opts.DEEPGRAM_SPEECH_UTTERANCE_END_MS) this.isContinuousAsr = true;
         break;
 
       case 'soniox':
@@ -487,9 +488,8 @@ class TaskGather extends SttTask {
     this._clearTimer();
     this._timeoutTimer = setTimeout(() => {
       if (this.isContinuousAsr) this._startAsrTimer();
-      else if (this.interDigitTimeout <= 0 ||
-        this.digitBuffer.length < this.minDigits ||
-        this.needsStt && this.digitBuffer.length === 0) {
+      if (this.interDigitTimer) return; // let the inter-digit timer complete
+      else {
         this._resolve(this.digitBuffer.length >= this.minDigits ? 'dtmf-num-digits' : 'timeout');
       }
     }, this.timeout);
@@ -499,7 +499,9 @@ class TaskGather extends SttTask {
     if (this._timeoutTimer) {
       clearTimeout(this._timeoutTimer);
       this._timeoutTimer = null;
+      return true;
     }
+    return false;
   }
 
   _startAsrTimer() {
@@ -622,7 +624,9 @@ class TaskGather extends SttTask {
     if (evt.is_final) {
       if (evt.alternatives[0].transcript === '' && !this.callSession.callGone && !this.killed) {
         emptyTranscript = true;
-        if (finished === 'true' && ['microsoft', 'deepgram'].includes(this.vendor)) {
+        if (finished === 'true' &&
+          ['microsoft', 'deepgram'].includes(this.vendor) &&
+          this._bufferedTranscripts.length === 0) {
           this.logger.debug({evt}, 'TaskGather:_onTranscription - got empty transcript from old gather, disregarding');
           return;
         }
@@ -669,7 +673,7 @@ class TaskGather extends SttTask {
           this.logger.debug({evt, words, bufferedWords},
             'TaskGather:_onTranscription - final transcript but < min barge words');
           this._bufferedTranscripts.push(evt);
-          this._startTranscribing(ep);
+          if (!['soniox', 'aws', 'microsoft', 'deepgram'].includes(this.vendor)) this._startTranscribing(ep);
           return;
         }
         else {
@@ -682,14 +686,11 @@ class TaskGather extends SttTask {
           else if (this.vendor === 'deepgram') {
             /* compile transcripts into one */
             if (!emptyTranscript) this._bufferedTranscripts.push(evt);
-            if (this.data.recognizer?.deepgramOptions?.utteranceEndMs) {
-              this.logger.debug('TaskGather:_onTranscription - got speech_final waiting for UtteranceEnd event');
-              return;
-            }
-            this.logger.debug({evt}, 'TaskGather:_onTranscription - compiling deepgram transcripts');
+
+            /* deepgram can send an empty and final transcript; only if we have any buffered should we resolve */
+            if (this._bufferedTranscripts.length === 0) return;
             evt = this.consolidateTranscripts(this._bufferedTranscripts, 1, this.language);
             this._bufferedTranscripts = [];
-            this.logger.debug({evt}, 'TaskGather:_onTranscription - compiled deepgram transcripts');
           }
 
           /* here is where we return a final transcript */
@@ -698,8 +699,7 @@ class TaskGather extends SttTask {
       }
     }
     else {
-      this._clearTimer();
-      this._startTimer();
+      if (this._clearTimer()) this._startTimer();
       if (this.bargein && (words + bufferedWords) >= this.minBargeinWordCount) {
         if (!this.playComplete) {
           this.logger.debug({transcript: evt.alternatives[0].transcript}, 'killing audio due to speech');
@@ -835,6 +835,9 @@ class TaskGather extends SttTask {
     if (this.resolved) return;
 
     this.resolved = true;
+    // If bargin is false and ws application return ack to verb:hook
+    // the gather should not play any audio
+    this._killAudio(this.cs);
     // Clear dtmf event
     if (this.dtmfBargein) {
       this.ep.removeAllListeners('dtmf');

lib/tasks/listen.js

@@ -123,8 +123,6 @@ class TaskListen extends Task {
       ci,
       this.metadata);
     if (this.hook.auth) {
-      this.logger.debug({username: this.hook.auth.username, password: this.hook.auth.password},
-        'TaskListen:_startListening basic auth');
       await this.ep.set({
         'MOD_AUDIO_BASIC_AUTH_USERNAME': this.hook.auth.username,
         'MOD_AUDIO_BASIC_AUTH_PASSWORD': this.hook.auth.password

lib/tasks/say.js

@@ -23,6 +23,12 @@ const breakLengthyTextIfNeeded = (logger,  text) => {
   }
 };
 
+const parseTextFromSayString = (text) => {
+  const closingBraceIndex = text.indexOf('}');
+  if (closingBraceIndex === -1) return text;
+  return text.slice(closingBraceIndex + 1);
+};
+
 class TaskSay extends Task {
   constructor(logger, opts, parentTask) {
     super(logger, opts);
@@ -59,8 +65,8 @@ class TaskSay extends Task {
     }
   }
 
-  async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label}) {
-    const {srf} = cs;
+  async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label, preCache = false}) {
+    const {srf, accountSid:account_sid} = cs;
     const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, srf);
     const {writeAlerts, AlertType, stats} = srf.locals;
     const {synthAudio} = srf.locals.dbHelpers;
@@ -97,11 +103,17 @@ class TaskSay extends Task {
       voice = this.options.voice_id || voice;
     }
 
-    this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
+    this.ep.set({
+      tts_engine: vendor,
+      tts_voice: voice,
+      cache_speech_handles: 1,
+    }).catch((err) => this.logger.info({err}, 'Error setting tts_engine on endpoint'));
+
+    if (!preCache) this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
     try {
       if (!credentials) {
         writeAlerts({
-          account_sid: cs.accountSid,
+          account_sid,
           alert_type: AlertType.TTS_NOT_PROVISIONED,
           vendor
         }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
@@ -120,14 +132,17 @@ class TaskSay extends Task {
         if (text.startsWith('silence_stream://')) return text;
 
         /* otel: trace time for tts */
-        const {span} = this.startChildSpan('tts-generation', {
-          'tts.vendor': vendor,
-          'tts.language': language,
-          'tts.voice': voice
-        });
+        if (!preCache)  {
+          const {span} = this.startChildSpan('tts-generation', {
+            'tts.vendor': vendor,
+            'tts.language': language,
+            'tts.voice': voice
+          });
+          this.otelSpan = span;
+        }
         try {
           const {filePath, servedFromCache, rtt} = await synthAudio(stats, {
-            account_sid: cs.accountSid,
+            account_sid,
             text,
             vendor,
             language,
@@ -137,30 +152,40 @@ class TaskSay extends Task {
             salt,
             credentials,
             options: this.options,
-            disableTtsCache : this.disableTtsCache
+            disableTtsCache : this.disableTtsCache,
+            preCache
           });
-          this.logger.debug(`file ${filePath}, served from cache ${servedFromCache}`);
-          if (filePath) cs.trackTmpFile(filePath);
-          if (!servedFromCache && !lastUpdated) {
-            lastUpdated = true;
-            updateSpeechCredentialLastUsed(credentials.speech_credential_sid)
-              .catch(() => {/*already logged error */});
+          if (!filePath.startsWith('say:')) {
+            this.logger.debug(`file ${filePath}, served from cache ${servedFromCache}`);
+            if (filePath) cs.trackTmpFile(filePath);
+            if (this.otelSpan) {
+              this.otelSpan.setAttributes({'tts.cached': servedFromCache});
+              this.otelSpan.end();
+              this.otelSpan = null;
+            }
+            if (!servedFromCache && !lastUpdated) {
+              lastUpdated = true;
+              updateSpeechCredentialLastUsed(credentials.speech_credential_sid).catch(() => {/* logged error */});
+            }
+            if (!servedFromCache && rtt && !preCache) {
+              this.notifyStatus({
+                event: 'synthesized-audio',
+                vendor,
+                language,
+                characters: text.length,
+                elapsedTime: rtt
+              });
+            }
           }
-          span.setAttributes({'tts.cached': servedFromCache});
-          span.end();
-          if (!servedFromCache && rtt) {
-            this.notifyStatus({
-              event: 'synthesized-audio',
-              vendor,
-              language,
-              characters: text.length,
-              elapsedTime: rtt
-            });
+          else {
+            this.logger.debug('a streaming tts api will be used');
+            const modifiedPath = filePath.replace('say:{', `say:{session-uuid=${this.ep.uuid},`);
+            return modifiedPath;
           }
           return filePath;
         } catch (err) {
           this.logger.info({err}, 'Error synthesizing tts');
-          span.end();
+          if (this.otelSpan) this.otelSpan.end();
           writeAlerts({
             account_sid: cs.accountSid,
             alert_type: AlertType.TTS_FAILURE,
@@ -182,6 +207,11 @@ class TaskSay extends Task {
   }
 
   async exec(cs, {ep}) {
+    const {srf, accountSid:account_sid} = cs;
+    const {writeAlerts, AlertType} = srf.locals;
+    const {addFileToCache} = srf.locals.dbHelpers;
+    const engine = this.synthesizer.engine || 'standard';
+
     await super.exec(cs);
     this.ep = ep;
 
@@ -239,8 +269,48 @@ class TaskSay extends Task {
           await this.playToConfMember(this.ep, memberId, confName, confUuid, filepath[segment]);
         }
         else {
-          this.logger.debug(`Say:exec sending command to play file ${filepath[segment]}`);
+          if (filepath[segment].startsWith('say:{')) {
+            const arr = /^say:\{.*\}\s*(.*)$/.exec(filepath[segment]);
+            if (arr) this.logger.debug(`Say:exec sending streaming tts request: ${arr[1].substring(0, 64)}..`);
+          }
+          else this.logger.debug(`Say:exec sending  ${filepath[segment].substring(0, 64)}`);
+          this.ep.once('playback-start', (evt) => {
+            this.logger.debug({evt}, 'got playback-start');
+            if (this.otelSpan) {
+              this.logger.debug({evt}, 'got playback-start');
+              this._addStreamingTtsAttributes(this.otelSpan, evt);
+              this.otelSpan.end();
+              this.otelSpan = null;
+              if (evt.variable_tts_cache_filename) cs.trackTmpFile(evt.variable_tts_cache_filename);
+            }
+          });
+          this.ep.once('playback-stop', (evt) => {
+            this.logger.debug({evt}, 'got playback-stop');
+            if (evt.variable_tts_error) {
+              writeAlerts({
+                account_sid,
+                alert_type: AlertType.TTS_FAILURE,
+                vendor,
+                detail: evt.variable_tts_error
+              }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
+            }
+            if (evt.variable_tts_cache_filename) {
+              const text = parseTextFromSayString(this.text[segment]);
+              addFileToCache(evt.variable_tts_cache_filename, {
+                account_sid,
+                vendor,
+                language,
+                voice,
+                engine,
+                text
+              }).catch((err) => this.logger.info({err}, 'Error adding file to cache'));
+            }
+          });
           await ep.play(filepath[segment]);
+          if (filepath[segment].startsWith('say:{')) {
+            const arr = /^say:\{.*\}\s*(.*)$/.exec(filepath[segment]);
+            if (arr) this.logger.debug(`Say:exec complete playing streaming tts request: ${arr[1].substring(0, 64)}..`);
+          }
           this.logger.debug(`Say:exec completed play file ${filepath[segment]}`);
         }
         segment++;
@@ -261,8 +331,30 @@ class TaskSay extends Task {
         this.notifyStatus({event: 'kill-playback'});
         this.ep.api('uuid_break', this.ep.uuid);
       }
+      this.ep.removeAllListeners('playback-start');
+      this.ep.removeAllListeners('playback-stop');
     }
   }
+
+  _addStreamingTtsAttributes(span, evt) {
+    const attrs = {'tts.cached': false};
+    for (const [key, value] of Object.entries(evt)) {
+      if (key.startsWith('variable_tts_')) {
+        let newKey = key.substring('variable_tts_'.length)
+          .replace('elevenlabs_', 'elevenlabs.');
+        if (spanMapping[newKey]) newKey = spanMapping[newKey];
+        attrs[newKey] = value;
+      }
+    }
+    span.setAttributes(attrs);
+  }
 }
 
+const spanMapping = {
+  'elevenlabs.reported_latency_ms': 'elevenlabs.latency_ms',
+  'elevenlabs.request_id': 'elevenlabs.req_id',
+  'elevenlabs.history_item_id': 'elevenlabs.item_id',
+  'elevenlabs.optimize_streaming_latency': 'elevenlabs.optimization',
+};
+
 module.exports = TaskSay;

lib/tasks/stt-task.js

@@ -110,7 +110,12 @@ class SttTask extends Task {
       this.data.recognizer.model = cs.speechRecognizerLanguage;
     }
 
-    if (!this.sttCredentials) {
+    if (
+      // not gather task, such as transcribe
+      (!this.input ||
+      // gather task with speech
+        this.input.includes('speech')) &&
+      !this.sttCredentials) {
       try {
         this.sttCredentials = await this._initSpeechCredentials(this.cs, this.vendor, this.label);
       } catch (error) {

lib/tasks/task.js

@@ -160,14 +160,24 @@ class Task extends Emitter {
       const httpHeaders = b3 && {b3};
       span.setAttributes({'http.body': JSON.stringify(params)});
       try {
+        if (this.id) params.verb_id = this.id;
         const json = await this.cs.requestor.request(type, this.actionHook, params, httpHeaders);
         span.setAttributes({'http.statusCode': 200});
-        span.end();
+        const isWsConnection = this.cs.requestor instanceof WsRequestor;
+        if (!isWsConnection || (expectResponse && json && Array.isArray(json) && json.length)) {
+          span.end();
+        } else {
+          /** we use this span to measure application response latency,
+           * and with websocket connections we generally get the application's response
+           * in a subsequent message from the far end, so we terminate the span when the
+           * first new set of verbs arrive after sending a transcript
+           * */
+          this.emit('VerbHookSpanWaitForEnd', {span});
+        }
         if (expectResponse && json && Array.isArray(json)) {
           const makeTask = require('./make_task');
           const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
           if (tasks && tasks.length > 0) {
-            this.logger.info({tasks: tasks}, `${this.name} replacing application with ${tasks.length} tasks`);
             this.callSession.replaceApplication(tasks);
           }
         }

lib/tasks/transcribe.js

@@ -41,6 +41,7 @@ class TaskTranscribe extends SttTask {
     /* buffer speech for continuous asr */
     this._bufferedTranscripts = [];
     this.bugname_prefix = 'transcribe_';
+    this.paused = false;
   }
 
   get name() { return TaskName.Transcribe; }
@@ -89,13 +90,13 @@ class TaskTranscribe extends SttTask {
       stopTranscription = true;
       this.ep.stopTranscription({
         vendor: this.vendor,
-        bugname: this.ep.transcribe_bugname
+        bugname: this.bugname
       })
         .catch((err) => this.logger.info(err, 'Error TaskTranscribe:kill'));
     }
     if (this.separateRecognitionPerChannel && this.ep2 && this.ep2.connected) {
       stopTranscription = true;
-      this.ep2.stopTranscription({vendor: this.vendor, bugname: this.ep2.transcribe_bugname})
+      this.ep2.stopTranscription({vendor: this.vendor, bugname: this.bugname})
         .catch((err) => this.logger.info(err, 'Error TaskTranscribe:kill'));
     }
 
@@ -117,9 +118,11 @@ class TaskTranscribe extends SttTask {
       this.logger.info(`TaskTranscribe:updateTranscribe status ${status}`);
       switch (status) {
         case TranscribeStatus.Pause:
+          this.paused = true;
           await this._stopTranscription();
           break;
         case TranscribeStatus.Resume:
+          this.paused = false;
           await this._startTranscribing(this.cs, this.ep, 1);
           if (this.separateRecognitionPerChannel && this.ep2) {
             await this._startTranscribing(this.cs, this.ep2, 2);
@@ -132,17 +135,16 @@ class TaskTranscribe extends SttTask {
   async _setSpeechHandlers(cs, ep, channel) {
     if (this[`_speechHandlersSet_${channel}`]) return;
     this[`_speechHandlersSet_${channel}`] = true;
-    let bugname;
 
     /* some special deepgram logic */
     if (this.vendor === 'deepgram') {
       if (this.isContinuousAsr) this._doContinuousAsrWithDeepgram(this.asrTimeout);
     }
 
-    const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.data.recognizer);
+    const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.language, this.data.recognizer);
     switch (this.vendor) {
       case 'google':
-        bugname = `${this.bugname_prefix}google_transcribe`;
+        this.bugname = `${this.bugname_prefix}google_transcribe`;
         this.addCustomEventListener(ep, GoogleTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
         this.addCustomEventListener(ep, GoogleTranscriptionEvents.NoAudioDetected,
@@ -153,7 +155,7 @@ class TaskTranscribe extends SttTask {
 
       case 'aws':
       case 'polly':
-        bugname = `${this.bugname_prefix}aws_transcribe`;
+        this.bugname = `${this.bugname_prefix}aws_transcribe`;
         this.addCustomEventListener(ep, AwsTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
         this.addCustomEventListener(ep, AwsTranscriptionEvents.NoAudioDetected,
@@ -162,19 +164,19 @@ class TaskTranscribe extends SttTask {
           this._onMaxDurationExceeded.bind(this, cs, ep, channel));
         break;
       case 'microsoft':
-        bugname = `${this.bugname_prefix}azure_transcribe`;
+        this.bugname = `${this.bugname_prefix}azure_transcribe`;
         this.addCustomEventListener(ep, AzureTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
         this.addCustomEventListener(ep, AzureTranscriptionEvents.NoSpeechDetected,
           this._onNoAudio.bind(this, cs, ep, channel));
         break;
       case 'nuance':
-        bugname = `${this.bugname_prefix}nuance_transcribe`;
+        this.bugname = `${this.bugname_prefix}nuance_transcribe`;
         this.addCustomEventListener(ep, NuanceTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
         break;
       case 'deepgram':
-        bugname = `${this.bugname_prefix}deepgram_transcribe`;
+        this.bugname = `${this.bugname_prefix}deepgram_transcribe`;
         this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
         this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Connect,
@@ -187,12 +189,12 @@ class TaskTranscribe extends SttTask {
 
         break;
       case 'soniox':
-        bugname = `${this.bugname_prefix}soniox_transcribe`;
+        this.bugname = `${this.bugname_prefix}soniox_transcribe`;
         this.addCustomEventListener(ep, SonioxTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
         break;
       case 'cobalt':
-        bugname = `${this.bugname_prefix}cobalt_transcribe`;
+        this.bugname = `${this.bugname_prefix}cobalt_transcribe`;
         this.addCustomEventListener(ep, CobaltTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
 
@@ -222,7 +224,7 @@ class TaskTranscribe extends SttTask {
         break;
 
       case 'ibm':
-        bugname = `${this.bugname_prefix}ibm_transcribe`;
+        this.bugname = `${this.bugname_prefix}ibm_transcribe`;
         this.addCustomEventListener(ep, IbmTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
         this.addCustomEventListener(ep, IbmTranscriptionEvents.Connect,
@@ -232,13 +234,13 @@ class TaskTranscribe extends SttTask {
         break;
 
       case 'nvidia':
-        bugname = `${this.bugname_prefix}nvidia_transcribe`;
+        this.bugname = `${this.bugname_prefix}nvidia_transcribe`;
         this.addCustomEventListener(ep, NvidiaTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
         break;
 
       case 'assemblyai':
-        bugname = `${this.bugname_prefix}assemblyai_transcribe`;
+        this.bugname = `${this.bugname_prefix}assemblyai_transcribe`;
         this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
         this.addCustomEventListener(ep,
@@ -250,7 +252,7 @@ class TaskTranscribe extends SttTask {
 
       default:
         if (this.vendor.startsWith('custom:')) {
-          bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
+          this.bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
           this.addCustomEventListener(ep, JambonzTranscriptionEvents.Transcription,
             this._onTranscription.bind(this, cs, ep, channel));
           this.addCustomEventListener(ep, JambonzTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
@@ -264,8 +266,7 @@ class TaskTranscribe extends SttTask {
           throw new Error(`Invalid vendor ${this.vendor}`);
         }
     }
-    // save dedicated bugname for each endpoint
-    ep.transcribe_bugname = `${bugname}_${Date.now()}`;
+
     /* common handler for all stt engine errors */
     this.addCustomEventListener(ep, JambonzTranscriptionEvents.Error, this._onJambonzError.bind(this, cs, ep));
     await ep.set(opts)
@@ -283,13 +284,13 @@ class TaskTranscribe extends SttTask {
 
   async _transcribe(ep) {
     this.logger.debug(
-      `TaskTranscribe:_transcribe - starting transcription vendor ${this.vendor} bugname ${ep.transcribe_bugname}`);
+      `TaskTranscribe:_transcribe - starting transcription vendor ${this.vendor} bugname ${this.bugname}`);
     await ep.startTranscription({
       vendor: this.vendor,
       interim: this.interim ? true : false,
       locale: this.language,
       channels: /*this.separateRecognitionPerChannel ? 2 : */ 1,
-      bugname: ep.transcribe_bugname,
+      bugname: this.bugname,
       hostport: this.hostport
     });
   }
@@ -297,7 +298,11 @@ class TaskTranscribe extends SttTask {
   async _onTranscription(cs, ep, channel, evt, fsEvent) {
     // make sure this is not a transcript from answering machine detection
     const bugname = fsEvent.getHeader('media-bugname');
-    if (bugname && ep.transcribe_bugname !== bugname) return;
+    if (bugname && this.bugname !== bugname) return;
+    if (this.paused) {
+      this.logger.debug({evt}, 'TaskTranscribe:_onTranscription - paused, ignoring transcript');
+    }
+
 
     if (this.vendor === 'ibm' && evt?.state === 'listening') return;
 
@@ -401,7 +406,8 @@ class TaskTranscribe extends SttTask {
   }
 
   _onNoAudio(cs, ep, channel) {
-    this.logger.debug(`TaskTranscribe:_onNoAudio restarting transcription on channel ${channel}`);
+    this.logger.debug(`TaskTranscribe:_onNoAudio on channel ${channel}`);
+    if (this.paused) return;
     if (this.childSpan[channel - 1] && this.childSpan[channel - 1].span) {
       this.childSpan[channel - 1].span.setAttributes({
         channel,
@@ -417,7 +423,8 @@ class TaskTranscribe extends SttTask {
   }
 
   _onMaxDurationExceeded(cs, ep, channel) {
-    this.logger.debug(`TaskTranscribe:_onMaxDurationExceeded restarting transcription on channel ${channel}`);
+    this.logger.debug(`TaskTranscribe:_onMaxDurationExceeded on channel ${channel}`);
+    if (this.paused) return;
     if (this.childSpan[channel - 1] && this.childSpan[channel - 1].span) {
       this.childSpan[channel - 1].span.setAttributes({
         channel,
@@ -442,10 +449,11 @@ class TaskTranscribe extends SttTask {
 
   async _onJambonzError(cs, _ep, evt) {
     this.logger.info({evt}, 'TaskTranscribe:_onJambonzError');
+    if (this.paused) return;
     if (this.isHandledByPrimaryProvider && this.fallbackVendor) {
       _ep.stopTranscription({
         vendor: this.vendor,
-        bugname: _ep.transcribe_bugname
+        bugname: this.bugname
       })
         .catch((err) => this.logger.error({err}, `Error stopping transcription for primary vendor ${this.vendor}`));
       const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);

lib/utils/amd-utils.js

@@ -266,7 +266,7 @@ module.exports = (logger) => {
 
     /* set stt options */
     logger.info(`starting amd for vendor ${vendor} and language ${language}`);
-    const sttOpts = amd.setChannelVarsForStt({name: TaskName.Gather}, sttCredentials, {
+    const sttOpts = amd.setChannelVarsForStt({name: TaskName.Gather}, sttCredentials, language, {
       vendor,
       hints,
       enhancedModel: true,

lib/utils/background-task-manager.js

@@ -65,18 +65,17 @@ class BackgroundTaskManager extends Emitter {
       // Remove task from managed List
       this.tasks.delete(type);
     } else {
-      this.logger.info(`stopping background task, ${type} is not running, skipped`);
+      this.logger.debug(`stopping background task, ${type} is not running, skipped`);
     }
   }
 
   stopAll() {
-    this.logger.info('BackgroundTaskManager:stopAll');
+    this.logger.debug('BackgroundTaskManager:stopAll');
     for (const key of this.tasks.keys()) {
       this.stop(key);
     }
   }
 
-  // Initiate Task
   // Initiate Listen
   async _initListen(opts, bugname = 'jambonz-background-listen', ignoreCustomerData = false, type = 'listen') {
     let task;
@@ -176,7 +175,7 @@ class BackgroundTaskManager extends Emitter {
   }
 
   _taskCompleted(type, task) {
-    this.logger.info({type, task}, 'BackgroundTaskManager:_taskCompleted: task completed');
+    this.logger.debug({type, task}, 'BackgroundTaskManager:_taskCompleted: task completed');
     task.removeAllListeners();
     task.span.end();
     this.tasks.delete(type);
@@ -189,7 +188,7 @@ class BackgroundTaskManager extends Emitter {
   }
 
   _bargeInTaskCompleted(evt) {
-    this.logger.info({evt}, 'BackgroundTaskManager:_bargeInTaskCompleted on event from background bargeIn');
+    this.logger.debug({evt}, 'BackgroundTaskManager:_bargeInTaskCompleted on event from background bargeIn');
     this.emit('bargeIn-done', evt);
   }
 }

lib/utils/db-utils.js

@@ -75,6 +75,8 @@ const speechMapper = (cred) => {
     else if ('deepgram' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
+      obj.deepgram_stt_uri = o.deepgram_stt_uri;
+      obj.deepgram_stt_use_tls = o.deepgram_stt_use_tls;
     }
     else if ('soniox' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));

lib/utils/http-requestor.js

@@ -14,6 +14,7 @@ const {
   HTTP_PROXY_PORT,
   HTTP_PROXY_PROTOCOL,
   NODE_ENV,
+  HTTP_USER_AGENT_HEADER,
 } = require('../config');
 
 const toBase64 = (str) => Buffer.from(str || '', 'utf8').toString('base64');
@@ -116,6 +117,10 @@ class HttpRequestor extends BaseRequestor {
     const url = hook.url || hook;
     const method = hook.method || 'POST';
     let buf = '';
+    httpHeaders = {
+      ...httpHeaders,
+      ...(HTTP_USER_AGENT_HEADER && {'user-agent' : HTTP_USER_AGENT_HEADER})
+    };
 
     assert.ok(url, 'HttpRequestor:request url was not provided');
     assert.ok, (['GET', 'POST'].includes(method), `HttpRequestor:request method must be 'GET' or 'POST' not ${method}`);

lib/utils/install-srf-locals.js

@@ -176,6 +176,7 @@ function installSrfLocals(srf, logger) {
   const registrar = new Registrar(logger, client);
   const {
     synthAudio,
+    addFileToCache,
     getNuanceAccessToken,
     getIbmAccessToken,
   } = require('@jambonz/speech-utils')({}, logger);
@@ -215,6 +216,7 @@ function installSrfLocals(srf, logger) {
       listCalls,
       deleteCall,
       synthAudio,
+      addFileToCache,
       createHash,
       retrieveHash,
       deleteKey,

lib/utils/place-outdial.js

@@ -16,9 +16,13 @@ const uuidv4 = require('uuid-random');
 const HttpRequestor = require('./http-requestor');
 const WsRequestor = require('./ws-requestor');
 const {makeOpusFirst} = require('./sdp-utils');
+const {
+  JAMBONES_USE_FREESWITCH_TIMER_FD
+} = require('../config');
 
 class SingleDialer extends Emitter {
-  constructor({logger, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask}) {
+  constructor({logger, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask,
+    onHoldMusic}) {
     super();
     assert(target.type);
 
@@ -41,6 +45,7 @@ class SingleDialer extends Emitter {
 
     this.callSid = uuidv4();
     this.dialTask = dialTask;
+    this.onHoldMusic = onHoldMusic;
 
     this.on('callStatusChange', this._notifyCallStatusChange.bind(this));
   }
@@ -131,6 +136,7 @@ class SingleDialer extends Emitter {
       this.serviceUrl = srf.locals.serviceUrl;
 
       this.ep = await ms.createEndpoint();
+      this._configMsEndpoint();
       this.logger.debug(`SingleDialer:exec - created endpoint ${this.ep.uuid}`);
 
       /**
@@ -189,6 +195,10 @@ class SingleDialer extends Emitter {
             callSid: this.callSid,
             traceId: this.rootSpan.traceId
           });
+          if (this.dialTask && this.dialTask.tag !== null &&
+            typeof this.dialTask.tag === 'object' && !Array.isArray(this.dialTask.tag)) {
+            this.callInfo.customerData = this.dialTask.tag;
+          }
           this.logger = srf.locals.parentLogger.child({
             callSid: this.callSid,
             parentCallSid: this.parentCallInfo.callSid,
@@ -253,7 +263,7 @@ class SingleDialer extends Emitter {
         .on('modify', async(req, res) => {
           try {
             if (this.ep) {
-              if (this.dialTask && this.dialTask.isOnHold) {
+              if (this.dialTask && this.dialTask.isOnHoldEnabled) {
                 this.logger.info('dial is onhold, emit event');
                 this.emit('reinvite', req, res);
               } else {
@@ -320,6 +330,16 @@ class SingleDialer extends Emitter {
     }
   }
 
+  _configMsEndpoint() {
+    const opts = {
+      ...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
+      ...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'})
+    };
+    if (Object.keys(opts).length > 0) {
+      this.ep.set(opts);
+    }
+  }
+
   /**
    * Run an application on the call after answer, e.g. call screening.
    * Once the application completes in some fashion, emit an 'accepted' event
@@ -406,6 +426,8 @@ class SingleDialer extends Emitter {
         this.accountInfo.account.webhook_secret);
       else app.notifier = {request: () => {}, close: () => {}};
     }
+    // Replace old application with new application.
+    this.application = app;
     const cs = new AdultingCallSession({
       logger: newLogger,
       singleDialer: this,
@@ -436,6 +458,7 @@ class SingleDialer extends Emitter {
   async reAnchorMedia() {
     assert(this.dlg && this.dlg.connected && !this.ep);
     this.ep = await this.ms.createEndpoint({remoteSdp: this.dlg.remote.sdp});
+    this._configMsEndpoint();
     await this.dlg.modify(this.ep.local.sdp, {
       headers: {
         'X-Reason': 'anchor-media'
@@ -466,11 +489,12 @@ class SingleDialer extends Emitter {
 }
 
 function placeOutdial({
-  logger, srf, ms, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask
+  logger, srf, ms, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask,
+  onHoldMusic
 }) {
   const myOpts = deepcopy(opts);
   const sd = new SingleDialer({
-    logger, sbcAddress, target, myOpts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask
+    logger, sbcAddress, target, myOpts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask, onHoldMusic
   });
   sd.exec(srf, ms, myOpts);
   return sd;

lib/utils/transcription-utils.js

@@ -102,6 +102,54 @@ const stickyVars = {
   ]
 };
 
+/**
+ * @see https://developers.deepgram.com/docs/models-languages-overview
+ */
+const optimalDeepramModels = {
+  zh: ['base', 'base'],
+  'zh-CN':['base', 'base'],
+  'zh-TW': ['base', 'base'],
+  da: ['enhanced', 'enhanced'],
+  en: ['nova-2-phonecall', 'nova-2'],
+  'en-US': ['nova-2-phonecall', 'nova-2'],
+  'en-AU': ['nova-2', 'nova-2'],
+  'en-GB': ['nova-2', 'nova-2'],
+  'en-IN': ['nova-2', 'nova-2'],
+  'en-NZ': ['nova-2', 'nova-2'],
+  nl: ['nova-2', 'nova-2'],
+  fr: ['nova-2', 'nova-2'],
+  'fr-CA': ['nova-2', 'nova-2'],
+  de: ['nova-2', 'nova-2'],
+  hi: ['nova-2', 'nova-2'],
+  'hi-Latn': ['nova-2', 'nova-2'],
+  id: ['base', 'base'],
+  it: ['nova-2', 'nova-2'],
+  ja: ['enhanced', 'enhanced'],
+  ko: ['nova-2', 'nova-2'],
+  no: ['nova-2', 'nova-2'],
+  pl: ['nova-2', 'nova-2'],
+  pt: ['nova-2', 'nova-2'],
+  'pt-BR': ['nova-2', 'nova-2'],
+  'pt-PT': ['nova-2', 'nova-2'],
+  ru: ['nova-2', 'nova-2'],
+  es: ['nova-2', 'nova-2'],
+  'es-419': ['nova-2', 'nova-2'],
+  'es-LATAM': ['enhanced', 'enhanced'],
+  sv: ['nova-2', 'nova-2'],
+  ta: ['enhanced', 'enhanced'],
+  taq: ['enhanced', 'enhanced'],
+  tr: ['nova-2', 'nova-2'],
+  uk: ['nova-2', 'nova-2']
+};
+
+const selectDefaultDeepgramModel = (task, language) => {
+  if (language in optimalDeepramModels) {
+    const [gather, transcribe] = optimalDeepramModels[language];
+    return task.name === TaskName.Gather ? gather : transcribe;
+  }
+  return 'base';
+};
+
 const consolidateTranscripts = (bufferedTranscripts, channel, language) => {
   if (bufferedTranscripts.length === 1) return bufferedTranscripts[0];
   let totalConfidence = 0;
@@ -424,7 +472,7 @@ module.exports = (logger) => {
     }
   };
 
-  const setChannelVarsForStt = (task, sttCredentials, rOpts = {}) => {
+  const setChannelVarsForStt = (task, sttCredentials, language, rOpts = {}) => {
     let opts = {};
     const {enable, voiceMs = 0, mode = -1} = rOpts.vad || {};
     const vad = {enable, voiceMs, mode};
@@ -516,6 +564,9 @@ module.exports = (logger) => {
         }),
         ...(sttCredentials.use_custom_stt && sttCredentials.custom_stt_endpoint &&
           {AZURE_SERVICE_ENDPOINT_ID: sttCredentials.custom_stt_endpoint}),
+        //azureSttEndpointId overrides sttCredentials.custom_stt_endpoint
+        ...(rOpts.azureSttEndpointId &&
+          {AZURE_SERVICE_ENDPOINT_ID: rOpts.azureSttEndpointId}),
       };
     }
     else if ('nuance' === vendor) {
@@ -567,15 +618,24 @@ module.exports = (logger) => {
       };
     }
     else if ('deepgram' === vendor) {
+      let {model} = rOpts;
       const {deepgramOptions = {}} = rOpts;
+      const deepgramUri = deepgramOptions.deepgramSttUri || sttCredentials.deepgram_stt_uri;
+      const useTls = deepgramOptions.deepgramSttUseTls || sttCredentials.deepgram_stt_use_tls;
+
+      /* default to a sensible model if not supplied */
+      if (!model) {
+        model = selectDefaultDeepgramModel(task, language);
+      }
       opts = {
         ...opts,
+        DEEPGRAM_SPEECH_MODEL: model,
+        ...(deepgramUri && {DEEPGRAM_URI: deepgramUri}),
+        ...(deepgramUri && useTls && {DEEPGRAM_USE_TLS: 1}),
         ...(sttCredentials.api_key) &&
           {DEEPGRAM_API_KEY: sttCredentials.api_key},
         ...(deepgramOptions.tier) &&
           {DEEPGRAM_SPEECH_TIER: deepgramOptions.tier},
-        ...(deepgramOptions.model) &&
-          {DEEPGRAM_SPEECH_MODEL: deepgramOptions.model},
         ...(deepgramOptions.punctuate) &&
           {DEEPGRAM_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION: 1},
         ...(deepgramOptions.smartFormatting) &&

lib/utils/ws-requestor.js

@@ -9,7 +9,8 @@ const {
   JAMBONES_WS_PING_INTERVAL_MS,
   MAX_RECONNECTS,
   JAMBONES_WS_HANDSHAKE_TIMEOUT_MS,
-  JAMBONES_WS_MAX_PAYLOAD
+  JAMBONES_WS_MAX_PAYLOAD,
+  HTTP_USER_AGENT_HEADER
 } = require('../config');
 
 class WsRequestor extends BaseRequestor {
@@ -228,6 +229,9 @@ class WsRequestor extends BaseRequestor {
         maxRedirects: 2,
         handshakeTimeout,
         maxPayload: JAMBONES_WS_MAX_PAYLOAD ? parseInt(JAMBONES_WS_MAX_PAYLOAD) : 24 * 1024,
+        headers: {
+          ...(HTTP_USER_AGENT_HEADER && {'user-agent' : HTTP_USER_AGENT_HEADER})
+        }
       };
       if (this.username && this.password) opts = {...opts, auth: `${this.username}:${this.password}`};
 
@@ -322,7 +326,9 @@ class WsRequestor extends BaseRequestor {
           'WsRequestor:_onSocketClosed time to reconnect');
         if (!this.ws && !this.connectInProgress) {
           this.connectInProgress = true;
-          this._connect().catch((err) => this.connectInProgress = false);
+          return this._connect()
+            .catch((err) => this.logger.error('WsRequestor:_onSocketClosed There is error while reconnect', err))
+            .finally(() => this.connectInProgress = false);
         }
       }, this.backoffMs);
       this.backoffMs = this.backoffMs < 2000 ? this.backoffMs * 2 : (this.backoffMs + 2000);

package.json

@@ -1,9 +1,9 @@
 {
   "name": "jambonz-feature-server",
-  "version": "0.8.5",
+  "version": "0.8.6",
   "main": "app.js",
   "engines": {
-    "node": ">= 10.16.0"
+    "node": ">= 18.x"
   },
   "keywords": [
     "sip",
@@ -27,14 +27,14 @@
   "dependencies": {
     "@aws-sdk/client-auto-scaling": "^3.360.0",
     "@aws-sdk/client-sns": "^3.360.0",
-    "@jambonz/db-helpers": "^0.9.1",
+    "@jambonz/db-helpers": "^0.9.3",
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/mw-registrar": "^0.2.4",
     "@jambonz/realtimedb-helpers": "^0.8.7",
-    "@jambonz/speech-utils": "^0.0.33",
+    "@jambonz/speech-utils": "^0.0.41",
     "@jambonz/stats-collector": "^0.1.9",
     "@jambonz/time-series": "^0.2.8",
-    "@jambonz/verb-specifications": "^0.0.50",
+    "@jambonz/verb-specifications": "^0.0.53",
     "@opentelemetry/api": "^1.4.0",
     "@opentelemetry/exporter-jaeger": "^1.9.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.35.0",
@@ -47,7 +47,7 @@
     "bent": "^7.3.12",
     "debug": "^4.3.4",
     "deepcopy": "^2.1.0",
-    "drachtio-fsmrf": "^3.0.33",
+    "drachtio-fsmrf": "^3.0.37",
     "drachtio-srf": "^4.5.31",
     "express": "^4.18.2",
     "express-validator": "^7.0.1",

test/create-call-test.js

@@ -99,6 +99,8 @@ test('test create-call call-hook basic authentication', async(t) => {
     let obj = await getJSON(`http:127.0.0.1:3100/lastRequest/${from}`)
     t.ok(obj.headers.Authorization = 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=',
       'create-call: call-hook contains basic authentication header');
+    t.ok(obj.headers['user-agent'] = 'jambonz',
+    'create-call: call-hook contains user-agent header');
     disconnect();
   } catch (err) {
     console.log(`error received: ${err}`);

test/transcribe-tests.js

@@ -347,8 +347,7 @@ test('\'transcribe\' test - deepgram config options altLanguages', async(t) => {
             "en-US"
           ],
           "deepgramOptions": {
-            "model": "2-ea",
-            "tier": "nova",
+            "model": "nova-2",
             "numerals": true,
             "ner": true,
             "vadTurnoff": 10,
@@ -408,8 +407,7 @@ test('\'transcribe\' test - deepgram config options altLanguages', async(t) => {
             "en-US"
           ],
           "deepgramOptions": {
-            "model": "2-ea",
-            "tier": "nova",
+            "model": "nova-2",
             "numerals": true,
             "ner": true,
             "vadTurnoff": 10,