support verbio speech (#757)

* support verbio speech

* wip

* wip

* wip

* update speech utils

* update verb specification

data/example-voicemail-greetings.json

@@ -9,7 +9,112 @@
     "can't take your call",
     "will get back to you",
     "I'll get back to you",
-    "we are unable"
+    "we are unable",
+    "Unable to take your call now",
+    "I'll reply soon",
+    "I'll call back",
+    "I'll reach out to you as soon as possible",
+    "Leave a message",
+    "Away from phone",
+    "Not available now",
+    "I'll return call",
+    "On another call",
+    "Currently on another call",
+    "I will return call later",
+    "Busy please leave message",
+    "Message will be returned promptly",
+    "Currently unavailable to answer",
+    "Planning to return your call soon",
+    "Apologies for missing your call",
+    "Not by the phone at the moment",
+    "Expecting to return your call",
+    "Currently not accessible",
+    "Intend to call back",
+    "Appreciate your patience!",
+    "Engaged in another conversation",
+    "I Will respond promptly",
+    "Kindly leave a message",
+    "Currently occupied leave a message",
+    "Unfortunately unable to answer right now",
+    "Occupied at the moment",
+    "Not present leave a message",
+    "Regrettably unavailable kindly leave a message",
+    "Will ensure a prompt response to your message",
+    "Currently engaged",
+    "Will return your call at the earliest opportunity",
+    "Your message will receive my prompt attention",
+    "I'll respond as soon as I can",
+    "Your message is important please leave it after the beep",
+    "Away from the phone at the moment",
+    "Unable to answer right now",
+    "Engaged in another task",
+    "Not by the phone presently",
+    "I'll respond at my earliest convenience",
+    "Away from the phone momentarily",
+    "I'll return your call shortly",
+    "Currently not able to answer",
+    "Your message is important please leave it after the tone",
+    "I'm unable to take your call right now",
+    "Please leave your message for me",
+    "I'll get back to you soon",
+    "Your call has been missed",
+    "Please leave a detailed message for me to respond to",
+    "Leave a message I'll make sure to respond",
+    "Feel free to leave a message",
+    "Your call is important to me",
+    "I'll get back to you shortly",
+    "Your message will be attended to promptly",
+    "Not available at the moment",
+    "I'll be sure to get back to you",
+    "I'll call you back soon",
+    "I'll ensure a prompt response",
+    "Sorry for the inconvenience",
+    "I'll return your call",
+    "I'll make sure to get back to you",
+    "I'll call you back shortly",
+    "I'll return your call as soon as possible",
+    "Apologies for the inconvenience leave your message",
+    "Your call is appreciated",
+    "I'm unavailable to answer",
+    "I'm currently away",
+    "I'll return your call as soon as I can",
+    "I'm away from the phone",
+    "I'm currently unavailable to take your call",
+    "Sorry for missing your call",
+    "I'll ensure it receives my immediate attention",
+    "I'm away from the phone momentarily",
+    "I'll reach out to you shortly",
+    "Apologies for the inconvenience",
+    "Currently occupied",
+    "Unable to answer your call at the moment",
+    "I'll make sure to follow up with you",
+    "Sorry for not being available",
+    "I'll reach out to you as soon as I can",
+    "I'm currently engaged",
+    "I'm currently busy",
+    "I'm currently unavailable",
+    "I'll respond to you at my earliest convenience",
+    "Your message is appreciated",
+    "I'll get back to you promptly",
+    "I'll get back to you without delay",
+    "Currently away from the phone",
+    "I'll return your call at my earliest opportunity",
+    "Sorry for the missed call",
+    "I'll make sure to address your concerns",
+    "Please provide your details for a callback",
+    "I'll make every effort to respond promptly",
+    "I'll ensure it's attended to promptly",
+    "Away from the phone temporarily",
+    "I'll get back to you as soon as I return",
+    "Currently not in a position to answer your call",
+    "Your call cannot be answered at the moment",
+    "I'll ensure to respond as soon as I'm able",
+    "Your call is important please leave a message",
+    "Unable to answer right now please leave your message",
+    "Currently not accessible intending to return your call",
+    "I'll respond promptly to your message",
+    "leave a memo",
+    "please leave a memo"
   ],
   "es-ES": [
     "le pasamos la llamada",

lib/dynamic-apps.js

@@ -30,6 +30,20 @@ const appsMap = {
         }
       ]
     }]
+  },
+  conference: {
+    // Dummy hook to follow later feature server logic.
+    call_hook: {
+      url: 'https://jambonz.org',
+      method: 'GET'
+    },
+    account_sid: '',
+    app_json: [{
+      verb: 'conference',
+      name: '',
+      beep: false,
+      startConferenceOnEnter: true
+    }]
   }
 };
 
@@ -38,6 +52,7 @@ const createJambonzApp = (type, {account_sid, name, caller_id}) => {
   app.account_sid = account_sid;
   switch (type) {
     case 'queue':
+    case 'conference':
       app.app_json[0].name = name;
       break;
     case 'user':

lib/middleware.js

@@ -75,13 +75,19 @@ module.exports = function(srf, logger) {
       req.locals.application_sid = application_sid;
     }
     // check for call to queue
-    if (uri.user?.startsWith('queue-') && req.locals.originatingUser && clientDb?.allow_direct_queue_calling) {
+    else if (uri.user?.startsWith('queue-') && req.locals.originatingUser && clientDb?.allow_direct_queue_calling) {
       const queue_name = uri.user.match(/queue-(.*)/)[1];
       logger.debug(`got Queue from Request URI header: ${queue_name}`);
       req.locals.queue_name = queue_name;
     }
+    // check for call to conference
+    else if (uri.user?.startsWith('conference-') && req.locals.originatingUser && clientDb?.allow_direct_app_calling) {
+      const conference_id = uri.user.match(/conference-(.*)/)[1];
+      logger.debug(`got Conference from Request URI header: ${conference_id}`);
+      req.locals.conference_id = conference_id;
+    }
     // check for call to registered user
-    if (!JAMBONES_DISABLE_DIRECT_P2P_CALL && req.locals.originatingUser && clientDb?.allow_direct_user_calling) {
+    else if (!JAMBONES_DISABLE_DIRECT_P2P_CALL && req.locals.originatingUser && clientDb?.allow_direct_user_calling) {
       const arr = /^(.*)@(.*)/.exec(req.locals.originatingUser);
       if (arr) {
         const sipRealm = arr[2];
@@ -237,6 +243,9 @@ module.exports = function(srf, logger) {
         logger.debug(`calling to registered user ${req.locals.called_user}, generating dial app`);
         app = createJambonzApp('user',
           {account_sid, name: req.locals.called_user, caller_id: req.locals.callingNumber});
+      } else if (req.locals.conference_id) {
+        logger.debug(`calling to conference ${req.locals.conference_id}, generating conference app`);
+        app = createJambonzApp('conference', {account_sid, name: req.locals.conference_id});
       } else if (req.locals.application_sid) {
         app = await lookupAppBySid(req.locals.application_sid);
       } else if (req.locals.originatingUser) {

lib/session/call-session.js

@@ -338,6 +338,17 @@ class CallSession extends Emitter {
     this.application.fallback_speech_recognizer_language = language;
   }
 
+  /**
+   * Vad
+   */
+  get vad() {
+    return this._vad;
+  }
+
+  set vad(v) {
+    this._vad = v;
+  }
+
   /**
    * indicates whether the call currently in progress
    */
@@ -907,6 +918,12 @@ class CallSession extends Emitter {
             api_key: credential.api_key,
             model_id: credential.model_id
           };
+        }  else if ('verbio' === vendor) {
+          return {
+            client_id: credential.client_id,
+            client_secret: credential.client_secret,
+            engine_version: credential.engine_version
+          };
         } else if (vendor.startsWith('custom:')) {
           return {
             speech_credential_sid: credential.speech_credential_sid,
@@ -990,6 +1007,11 @@ class CallSession extends Emitter {
       ) {
         try {
           await this._awaitCommandsOrHangup();
+          if (this._isPlayingFillerNoise) {
+            this._isPlayingFillerNoise = false;
+            this.ep.api('uuid_break', this.ep.uuid)
+              .catch((err) => this.logger.info(err, 'Error killing filler noise'));
+          }
           if (this.callGone) break;
         } catch (err) {
           this.logger.info(err, 'CallSession:exec - error waiting for new commands');
@@ -1276,7 +1298,7 @@ class CallSession extends Emitter {
   async _lccConferenceParticipantAction(opts) {
     const task = this.currentTask;
     if (!task || TaskName.Conference !== task.name || !this.isInConference) {
-      return this.logger.info('CallSession:_lccConferenceParticipantState - invalid cmd, call is not in conference');
+      return this.logger.info('CallSession:_lccConferenceParticipantAction - invalid cmd, call is not in conference');
     }
     task.doConferenceParticipantAction(this, opts);
   }
@@ -1528,7 +1550,7 @@ Duration=${duration} `
       return this._lccTag(opts);
     }
     else if (opts.conferenceParticipantAction) {
-      return this._lccConferenceParticipantState(opts);
+      return this._lccConferenceParticipantAction(opts.conferenceParticipantAction);
     }
     else if (opts.dub) {
       return this._lccDub(opts);
@@ -1911,7 +1933,7 @@ Duration=${duration} `
       });
       //ep.cs = this;
       this.ep = ep;
-      this.logger.debug(`allocated endpoint ${ep.uuid}`);
+      this.logger.info(`allocated endpoint ${ep.uuid}`);
 
       this._configMsEndpoint();
 
@@ -2427,6 +2449,20 @@ Duration=${duration} `
     return new Promise((resolve, reject) => {
       this.logger.info('_awaitCommandsOrHangup - waiting...');
       this.wakeupResolver = resolve;
+
+      /* start filler noise if configured while we wait for new commands */
+      if (this.fillerNoise?.url && this.ep?.connected && !this.ep2) {
+        this.logger.debug('CallSession:_awaitCommandsOrHangup - playing filler noise');
+        this._isPlayingFillerNoise = true;
+        this.ep.play(this.fillerNoise.url);
+        this.ep.once('playback-start', (evt) => {
+          if (evt.file === this.fillerNoise.url && !this._isPlayingFillerNoise) {
+            this.logger.info('CallSession:_awaitCommandsOrHangup - filler noise started');
+            this.ep.api('uuid_break', this.ep.uuid)
+              .catch((err) => this.logger.info(err, 'Error killing filler noise'));
+          }
+        });
+      }
     });
   }
 

lib/session/inbound-call-session.js

@@ -79,6 +79,7 @@ class InboundCallSession extends CallSession {
       this.logger.info('InboundCallSession:_hangup - race condition, dlg cleared by app hangup');
       return;
     }
+    this.logger.info(`InboundCallSession: ${terminatedBy} hung up`);
     assert(this.dlg.connectTime);
     const duration = moment().diff(this.dlg.connectTime, 'seconds');
     this.rootSpan.setAttributes({'call.termination': `hangup by ${terminatedBy}`});
@@ -87,7 +88,6 @@ class InboundCallSession extends CallSession {
       callStatus: CallStatus.Completed,
       duration
     });
-    this.logger.info(`InboundCallSession: ${terminatedBy} hung up`);
     this._callReleased();
     this.req.removeAllListeners('cancel');
   }

lib/tasks/conference.js

@@ -518,7 +518,7 @@ class Conference extends Task {
         this.doConferenceMute(cs, {conf_mute_status: 'unmute'});
         break;
       default:
-        this.logger.info(`Conference:doConferenceParticipantState - unhandled action ${action}`);
+        this.logger.info(`Conference:doConferenceParticipantAction - unhandled action ${action}`);
         break;
     }
   }

lib/tasks/config.js

@@ -15,7 +15,8 @@ class TaskConfig extends Task {
       'transcribe',
       'fillerNoise',
       'actionHookDelayAction',
-      'boostAudioSignal'
+      'boostAudioSignal',
+      'vad'
     ].forEach((k) => this[k] = this.data[k] || {});
 
     if ('notifyEvents' in this.data) {
@@ -70,6 +71,7 @@ class TaskConfig extends Task {
   get hasListen() { return Object.keys(this.listen).length; }
   get hasTranscribe() { return Object.keys(this.transcribe).length; }
   get hasDub() { return Object.keys(this.dub).length; }
+  get hasVad() { return Object.keys(this.vad).length; }
   get hasFillerNoise() { return Object.keys(this.fillerNoise).length; }
 
   get summary() {
@@ -287,6 +289,16 @@ class TaskConfig extends Task {
         cs.enableFillerNoise(opts);
       }
     }
+
+    if (this.hasVad) {
+      cs.vad = {
+        enable: this.vad.enable || false,
+        voiceMs: this.vad.voiceMs || 250,
+        silenceMs: this.vad.silenceMs || 150,
+        strategy: this.vad.strategy || 'one-shot',
+        mode: this.vad.mod || 2
+      };
+    }
   }
 
   async kill(cs) {

lib/tasks/gather.js

@@ -10,7 +10,9 @@ const {
   IbmTranscriptionEvents,
   NvidiaTranscriptionEvents,
   JambonzTranscriptionEvents,
-  AssemblyAiTranscriptionEvents
+  AssemblyAiTranscriptionEvents,
+  VadDetection,
+  VerbioTranscriptionEvents
 } = require('../utils/constants.json');
 const {
   JAMBONES_GATHER_EARLY_HINTS_MATCH,
@@ -27,7 +29,7 @@ class TaskGather extends SttTask {
     [
       'finishOnKey', 'input', 'numDigits', 'minDigits', 'maxDigits',
       'interDigitTimeout', 'partialResultHook', 'bargein', 'dtmfBargein',
-      'speechTimeout', 'timeout', 'say', 'play', 'actionHookDelayAction', 'fillerNoise'
+      'speechTimeout', 'timeout', 'say', 'play', 'actionHookDelayAction', 'fillerNoise', 'vad'
     ].forEach((k) => this[k] = this.data[k]);
 
     // gather default input is digits
@@ -41,7 +43,8 @@ class TaskGather extends SttTask {
     this.timeout = this.timeout === 0 ? 0 : (this.timeout || 15) * 1000;
     this.interim = !!this.partialResultHook || this.bargein || (this.timeout > 0);
     this.listenDuringPrompt = this.data.listenDuringPrompt === false ? false : true;
-    this.minBargeinWordCount = this.data.minBargeinWordCount || 1;
+    this.minBargeinWordCount = this.data.minBargeinWordCount !== undefined ? this.data.minBargeinWordCount : 1;
+    this._vadEnabled = this.minBargeinWordCount === 0;
     if (this.data.recognizer) {
       /* continuous ASR (i.e. compile transcripts until a special timeout or dtmf key) */
       this.asrTimeout = typeof this.data.recognizer.asrTimeout === 'number' ?
@@ -128,6 +131,11 @@ class TaskGather extends SttTask {
       ...(this.fillerNoise || {})
     };
 
+    this.vad = {
+      ...(cs.vad || {}),
+      ...(this.vad || {})
+    };
+
     if (cs.hasGlobalSttHints && !this.maskGlobalSttHints) {
       const {hints, hintsBoost} = cs.globalSttHints;
       const setOfHints = new Set((this.data.recognizer.hints || [])
@@ -178,6 +186,8 @@ class TaskGather extends SttTask {
       retries: this._hookDelayRetries
     };
 
+    this._startVad();
+
     const startListening = async(cs, ep) => {
       this._startTimer();
       if (this.isContinuousAsr && 0 === this.timeout) this._startAsrTimer();
@@ -201,6 +211,7 @@ class TaskGather extends SttTask {
         const {span, ctx} = this.startChildSpan(`nested:${this.sayTask.summary}`);
         const process = () => {
           this.logger.debug('Gather: nested say task completed');
+          this._stopVad();
           if (!this.killed) {
             startListening(cs, ep);
             if (this.input.includes('speech') && this.vendor === 'nuance' && this.listenDuringPrompt) {
@@ -227,6 +238,7 @@ class TaskGather extends SttTask {
         const {span, ctx} = this.startChildSpan(`nested:${this.playTask.summary}`);
         const process = () => {
           this.logger.debug('Gather: nested play task completed');
+          this._stopVad();
           if (!this.killed) {
             startListening(cs, ep);
             if (this.input.includes('speech') && this.vendor === 'nuance' && this.listenDuringPrompt) {
@@ -259,9 +271,14 @@ class TaskGather extends SttTask {
 
       if (this.input.includes('speech') && this.listenDuringPrompt) {
         await this._setSpeechHandlers(cs, ep);
-        this._startTranscribing(ep);
-        updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid)
-          .catch(() => {/*already logged error */});
+        if (!this.resolved && !this.killed) {
+          this._startTranscribing(ep);
+          updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid)
+            .catch(() => {/*already logged error */});
+        }
+        else {
+          this.logger.info('Gather:exec - task was killed or resolved quickly, not starting transcription');
+        }
       }
 
       if (this.input.includes('digits') || this.dtmfBargein || this.asrDtmfTerminationDigit) {
@@ -269,6 +286,7 @@ class TaskGather extends SttTask {
       }
 
       await this.awaitTaskDone();
+      this._killAudio(cs);
     } catch (err) {
       this.logger.error(err, 'TaskGather:exec error');
     }
@@ -285,6 +303,7 @@ class TaskGather extends SttTask {
     this._clearAsrTimer();
     this.playTask?.span.end();
     this.sayTask?.span.end();
+    this._stopVad();
     this._resolve('killed');
   }
 
@@ -362,23 +381,19 @@ class TaskGather extends SttTask {
           ep, GoogleTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
         this.addCustomEventListener(
           ep, GoogleTranscriptionEvents.EndOfUtterance, this._onEndOfUtterance.bind(this, cs, ep));
-        this.addCustomEventListener(
-          ep, GoogleTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
         break;
 
       case 'aws':
       case 'polly':
         this.bugname = `${this.bugname_prefix}aws_transcribe`;
         this.addCustomEventListener(ep, AwsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
-        this.addCustomEventListener(ep, AwsTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
         break;
       case 'microsoft':
         this.bugname = `${this.bugname_prefix}azure_transcribe`;
         this.addCustomEventListener(
           ep, AzureTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
-        this.addCustomEventListener(ep, AzureTranscriptionEvents.NoSpeechDetected,
-          this._onNoSpeechDetected.bind(this, cs, ep));
-        this.addCustomEventListener(ep, AzureTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
+        //this.addCustomEventListener(ep, AzureTranscriptionEvents.NoSpeechDetected,
+        //this._onNoSpeechDetected.bind(this, cs, ep));
         break;
       case 'nuance':
         this.bugname = `${this.bugname_prefix}nuance_transcribe`;
@@ -388,8 +403,6 @@ class TaskGather extends SttTask {
           this._onStartOfSpeech.bind(this, cs, ep));
         this.addCustomEventListener(ep, NuanceTranscriptionEvents.TranscriptionComplete,
           this._onTranscriptionComplete.bind(this, cs, ep));
-        this.addCustomEventListener(ep, NuanceTranscriptionEvents.VadDetected,
-          this._onVadDetected.bind(this, cs, ep));
 
         /* stall timers until prompt finishes playing */
         if ((this.sayTask || this.playTask) && this.listenDuringPrompt) {
@@ -412,6 +425,12 @@ class TaskGather extends SttTask {
           ep, SonioxTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
         break;
 
+      case 'verbio':
+        this.bugname = `${this.bugname_prefix}verbio_transcribe`;
+        this.addCustomEventListener(
+          ep, VerbioTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        break;
+
       case 'cobalt':
         this.bugname = `${this.bugname_prefix}cobalt_transcribe`;
         this.addCustomEventListener(
@@ -459,8 +478,6 @@ class TaskGather extends SttTask {
           this._onStartOfSpeech.bind(this, cs, ep));
         this.addCustomEventListener(ep, NvidiaTranscriptionEvents.TranscriptionComplete,
           this._onTranscriptionComplete.bind(this, cs, ep));
-        this.addCustomEventListener(ep, NvidiaTranscriptionEvents.VadDetected,
-          this._onVadDetected.bind(this, cs, ep));
 
         /* I think nvidia has this (??) - stall timers until prompt finishes playing */
         if ((this.sayTask || this.playTask) && this.listenDuringPrompt) {
@@ -698,11 +715,44 @@ class TaskGather extends SttTask {
     this._finalAsrTimer = null;
   }
 
+
+  _startVad() {
+    if (!this._vadStarted && this._vadEnabled) {
+      this.logger.debug('_startVad');
+      this.addCustomEventListener(this.ep, VadDetection.Detection, this._onVadDetected.bind(this, this.cs, this.ep));
+      this.ep?.startVadDetection(this.vad);
+      this._vadStarted = true;
+    }
+  }
+
+  _stopVad() {
+    if (this._vadStarted) {
+      this.logger.debug('_stopVad');
+      this.ep?.stopVadDetection(this.vad);
+      this.ep?.removeCustomEventListener(VadDetection.Detection, this._onVadDetected);
+      this._vadStarted = false;
+    }
+  }
+
+  _startFillerNoise() {
+    this.logger.debug('Gather:_startFillerNoise - playing filler noise');
+    this.ep?.play(this.fillerNoise.url);
+    this._fillerNoiseOn = true;
+    this.ep.once('playback-start', (evt) => {
+      if (evt.file === this.fillerNoise.url && !this._fillerNoiseOn) {
+        this.logger.info({evt}, 'Gather:_startFillerNoise - race condition - kill filler noise here');
+        this.ep.api('uuid_break', this.ep.uuid)
+          .catch((err) => this.logger.info(err, 'Error killing filler noise'));
+        return;
+      } else this.logger.debug({evt}, 'Gather:_startFillerNoise - playback started');
+    });
+  }
+
   _startFillerNoiseTimer() {
     this._clearFillerNoiseTimer();
     this._fillerNoiseTimer = setTimeout(() => {
       this.logger.debug('Gather:_startFillerNoiseTimer - playing filler noise');
-      this.ep?.play(this.fillerNoise.url);
+      this._startFillerNoise();
     }, this.fillerNoise.startDelaySecs * 1000);
   }
 
@@ -723,6 +773,7 @@ class TaskGather extends SttTask {
       if (this.ep?.connected && (!this.playComplete || this.hasFillerNoise)) {
         this.logger.debug('Gather:_killAudio: killing playback of any audio');
         this.playComplete = true;
+        this._fillerNoiseOn = false;  // in a race, if we just started audio it may sneak through here
         this.ep.api('uuid_break', this.ep.uuid)
           .catch((err) => this.logger.info(err, 'Error killing audio'));
       }
@@ -1018,6 +1069,10 @@ class TaskGather extends SttTask {
       this._killAudio(cs);
       this.emit('vad');
     }
+    if (this.vad?.strategy === 'one-shot') {
+      this.ep?.removeCustomEventListener(VadDetection.Detection, this._onVadDetected);
+      this._vadStarted = false;
+    }
   }
 
   _onNoSpeechDetected(cs, ep, evt, fsEvent) {
@@ -1085,7 +1140,7 @@ class TaskGather extends SttTask {
       }
       else {
         this.logger.debug(`TaskGather:_resolve - playing filler noise: ${this.fillerNoiseUrl}`);
-        this.ep.play(this.fillerNoiseUrl);
+        this._startFillerNoise();
       }
     }
 

lib/tasks/say.js

@@ -100,7 +100,7 @@ class TaskSay extends TtsTask {
     }
 
     ep.set({
-      tts_engine: vendor,
+      tts_engine: vendor.startsWith('custom:') ? 'custom' : vendor,
       tts_voice: voice,
       cache_speech_handles: !cs.currentTtsVendor || cs.currentTtsVendor === vendor ? 1 : 0,
     }).catch((err) => this.logger.info({err}, 'Error setting tts_engine on endpoint'));
@@ -329,10 +329,10 @@ class TaskSay extends TtsTask {
           } catch (err) {
             try {
               await startFallback(err);
+              continue;
             } catch (err) {
               this.logger.info({err}, 'Error waiting for playback-stop event');
             }
-            continue;
           } finally {
             this._playPromise = null;
             this._playResolve = null;
@@ -341,8 +341,10 @@ class TaskSay extends TtsTask {
           if (filepath[segment].startsWith('say:{')) {
             const arr = /^say:\{.*\}\s*(.*)$/.exec(filepath[segment]);
             if (arr) this.logger.debug(`Say:exec complete playing streaming tts request: ${arr[1].substring(0, 64)}..`);
+          } else {
+            // This log will print spech credentials in say command for tts stream mode
+            this.logger.debug(`Say:exec completed play file ${filepath[segment]}`);
           }
-          this.logger.debug(`Say:exec completed play file ${filepath[segment]}`);
         }
         segment++;
       }
@@ -364,6 +366,8 @@ class TaskSay extends TtsTask {
       }
       this.ep.removeAllListeners('playback-start');
       this.ep.removeAllListeners('playback-stop');
+      // if we are waiting for playback-stop event, resolve the promise
+      if (this._playResolve) this._playResolve();
     }
   }
 
@@ -376,6 +380,7 @@ class TaskSay extends TtsTask {
           .replace('deepgram_', 'deepgram.')
           .replace('playht_', 'playht.')
           .replace('rimelabs_', 'rimelabs.')
+          .replace('verbio_', 'verbio.')
           .replace('elevenlabs_', 'elevenlabs.');
         if (spanMapping[newKey]) newKey = spanMapping[newKey];
         attrs[newKey] = value;
@@ -424,6 +429,10 @@ const spanMapping = {
   'rimelabs.name_lookup_time_ms': 'name_lookup_ms',
   'rimelabs.connect_time_ms': 'connect_ms',
   'rimelabs.final_response_time_ms': 'final_response_ms',
+  // verbio
+  'verbio.name_lookup_time_ms': 'name_lookup_ms',
+  'verbio.connect_time_ms': 'connect_ms',
+  'verbio.final_response_time_ms': 'final_response_ms',
 };
 
 module.exports = TaskSay;

lib/tasks/stt-task.js

@@ -166,7 +166,7 @@ class SttTask extends Task {
   }
 
   async _initSpeechCredentials(cs, vendor, label) {
-    const {getNuanceAccessToken, getIbmAccessToken, getAwsAuthToken} = cs.srf.locals.dbHelpers;
+    const {getNuanceAccessToken, getIbmAccessToken, getAwsAuthToken, getVerbioAccessToken} = cs.srf.locals.dbHelpers;
     let credentials = cs.getSpeechCredentials(vendor, 'stt', label);
 
     if (!credentials) {
@@ -200,6 +200,11 @@ class SttTask extends Task {
       const {accessKeyId, secretAccessKey, sessionToken, servedFromCache} = await getAwsAuthToken(roleArn);
       this.logger.debug({roleArn}, `got aws access token ${servedFromCache ? 'from cache' : ''}`);
       credentials = {...credentials, accessKeyId, secretAccessKey, sessionToken};
+    } else if (vendor === 'verbio' && credentials.client_id && credentials.client_secret) {
+      const {access_token, servedFromCache} = await getVerbioAccessToken(credentials);
+      this.logger.debug({client_id: credentials.client_id},
+        `got verbio access token ${servedFromCache ? 'from cache' : ''}`);
+      credentials.access_token = access_token;
     }
     return credentials;
   }

lib/tasks/transcribe.js

@@ -197,8 +197,8 @@ class TaskTranscribe extends SttTask {
         this.bugname = `${this.bugname_prefix}azure_transcribe`;
         this.addCustomEventListener(ep, AzureTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
-        this.addCustomEventListener(ep, AzureTranscriptionEvents.NoSpeechDetected,
-          this._onNoAudio.bind(this, cs, ep, channel));
+        //this.addCustomEventListener(ep, AzureTranscriptionEvents.NoSpeechDetected,
+        //  this._onNoAudio.bind(this, cs, ep, channel));
         break;
       case 'nuance':
         this.bugname = `${this.bugname_prefix}nuance_transcribe`;

lib/utils/constants.json

@@ -97,6 +97,10 @@
     "Transcription": "soniox_transcribe::transcription",
     "Error": "soniox_transcribe::error"
   },
+  "VerbioTranscriptionEvents": {
+    "Transcription": "verbio_transcribe::transcription",
+    "Error": "verbio_transcribe::error"
+  },
   "CobaltTranscriptionEvents": {
     "Transcription": "cobalt_speech::transcription",
     "CompileContext": "cobalt_speech::compile_context_response",
@@ -134,6 +138,9 @@
     "ConnectFailure": "assemblyai_transcribe::connect_failed",
     "Connect": "assemblyai_transcribe::connect"
   },
+  "VadDetection": {
+    "Detection": "vad_detect:detection"
+  },
   "ListenEvents": {
     "Connect": "mod_audio_fork::connect",
     "ConnectFailure": "mod_audio_fork::connect_failed",

lib/utils/db-utils.js

@@ -113,6 +113,11 @@ const speechMapper = (cred) => {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
       obj.model_id = o.model_id;
+    }  else if ('verbio' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.client_id = o.client_id;
+      obj.client_secret = o.client_secret;
+      obj.engine_version = o.engine_version;
     } else if (obj.vendor.startsWith('custom:')) {
       const o = JSON.parse(decrypt(credential));
       obj.auth_token = o.auth_token;

lib/utils/install-srf-locals.js

@@ -171,7 +171,7 @@ function installSrfLocals(srf, logger) {
     retrieveFromSortedSet,
     retrieveByPatternSortedSet,
     sortedSetLength,
-    sortedSetPositionByPattern
+    sortedSetPositionByPattern,
   } = require('@jambonz/realtimedb-helpers')({}, logger, tracer);
   const registrar = new Registrar(logger, client);
   const {
@@ -179,7 +179,8 @@ function installSrfLocals(srf, logger) {
     addFileToCache,
     getNuanceAccessToken,
     getIbmAccessToken,
-    getAwsAuthToken
+    getAwsAuthToken,
+    getVerbioAccessToken
   } = require('@jambonz/speech-utils')({}, logger);
   const {
     writeAlerts,
@@ -239,7 +240,8 @@ function installSrfLocals(srf, logger) {
       retrieveFromSortedSet,
       retrieveByPatternSortedSet,
       sortedSetLength,
-      sortedSetPositionByPattern
+      sortedSetPositionByPattern,
+      getVerbioAccessToken
     },
     parentLogger: logger,
     getSBC,

lib/utils/transcription-utils.js

@@ -474,18 +474,8 @@ module.exports = (logger) => {
 
   const setChannelVarsForStt = (task, sttCredentials, language, rOpts = {}) => {
     let opts = {};
-    const {enable, voiceMs = 0, mode = -1} = rOpts.vad || {};
-    const vad = {enable, voiceMs, mode};
     const vendor = rOpts.vendor;
 
-    /* voice activity detection works across vendors */
-    opts = {
-      ...opts,
-      ...(vad.enable && {START_RECOGNIZING_ON_VAD: 1}),
-      ...(vad.enable && vad.voiceMs && {RECOGNIZER_VAD_VOICE_MS: vad.voiceMs}),
-      ...(vad.enable && typeof vad.mode === 'number' && {RECOGNIZER_VAD_MODE: vad.mode}),
-    };
-
     if ('google' === vendor) {
       const useV2 = rOpts.googleOptions?.serviceVersion === 'v2';
       const model = task.name === TaskName.Gather ?
@@ -806,8 +796,26 @@ module.exports = (logger) => {
         ...(rOpts.hints?.length > 0 &&
           {ASSEMBLYAI_WORD_BOOST: JSON.stringify(rOpts.hints)})
       };
-    }
-    else if (vendor.startsWith('custom:')) {
+    } else if ('verbio' === vendor) {
+      const {verbioOptions = {}} = rOpts;
+      opts = {
+        ...opts,
+        ...(sttCredentials.access_token && { VERBIO_ACCESS_TOKEN: sttCredentials.access_token}),
+        ...(sttCredentials.engine_version && {VERBIO_ENGINE_VERSION: sttCredentials.engine_version}),
+        ...(language && {VERBIO_LANGUAGE: language}),
+        ...(verbioOptions.enable_formatting && {VERBIO_ENABLE_FORMATTING: verbioOptions.enable_formatting}),
+        ...(verbioOptions.enable_diarization && {VERBIO_ENABLE_DIARIZATION: verbioOptions.enable_diarization}),
+        ...(verbioOptions.topic && {VERBIO_TOPIC: verbioOptions.topic}),
+        ...(verbioOptions.inline_grammar && {VERBIO_INLINE_GRAMMAR: verbioOptions.inline_grammar}),
+        ...(verbioOptions.grammar_uri && {VERBIO_GRAMMAR_URI: verbioOptions.grammar_uri}),
+        ...(verbioOptions.label && {VERBIO_LABEL: verbioOptions.label}),
+        ...(verbioOptions.recognition_timeout && {VERBIO_RECOGNITION_TIMEOUT: verbioOptions.recognition_timeout}),
+        ...(verbioOptions.speech_complete_timeout &&
+          {VERBIO_SPEECH_COMPLETE_TIMEOUT: verbioOptions.speech_complete_timeout}),
+        ...(verbioOptions.speech_incomplete_timeout &&
+          {VERBIO_SPEECH_INCOMPLETE_TIMEOUT: verbioOptions.speech_incomplete_timeout}),
+      };
+    } else if (vendor.startsWith('custom:')) {
       let {options = {}} = rOpts;
       const {auth_token, custom_stt_url} = sttCredentials;
       options = {

package-lock.json

@@ -15,10 +15,10 @@
         "@jambonz/http-health-check": "^0.0.1",
         "@jambonz/mw-registrar": "^0.2.7",
         "@jambonz/realtimedb-helpers": "^0.8.8",
-        "@jambonz/speech-utils": "^0.1.0",
-        "@jambonz/stats-collector": "^0.1.9",
+        "@jambonz/speech-utils": "^0.1.3",
+        "@jambonz/stats-collector": "^0.1.10",
         "@jambonz/time-series": "^0.2.8",
-        "@jambonz/verb-specifications": "^0.0.69",
+        "@jambonz/verb-specifications": "^0.0.72",
         "@opentelemetry/api": "^1.8.0",
         "@opentelemetry/exporter-jaeger": "^1.23.0",
         "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
@@ -31,8 +31,8 @@
         "bent": "^7.3.12",
         "debug": "^4.3.4",
         "deepcopy": "^2.1.0",
-        "drachtio-fsmrf": "^3.0.41",
-        "drachtio-srf": "^4.5.31",
+        "drachtio-fsmrf": "^3.0.43",
+        "drachtio-srf": "^4.5.35",
         "express": "^4.19.2",
         "express-validator": "^7.0.1",
         "ip": "^2.0.1",
@@ -2322,9 +2322,9 @@
       }
     },
     "node_modules/@jambonz/speech-utils": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.1.0.tgz",
-      "integrity": "sha512-45K6Vrl2PMEbbcnvm65afCDujDxck/bEUq7+P6KRw/cei3mrKtwjGh3HXi1cKhC1gA5UF1+5YrUoPO9LdoZnog==",
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.1.3.tgz",
+      "integrity": "sha512-Ess3yc8XyJAoUvXFz9maLq4NgxkTkTgiN2uW3rgOFiRr7b6l9A3oNQrN/ZJCJc9Ge+I3CypFmcwCiLNehuqF/g==",
       "dependencies": {
         "@aws-sdk/client-polly": "^3.496.0",
         "@aws-sdk/client-sts": "^3.496.0",
@@ -2342,9 +2342,9 @@
       }
     },
     "node_modules/@jambonz/stats-collector": {
-      "version": "0.1.9",
-      "resolved": "https://registry.npmjs.org/@jambonz/stats-collector/-/stats-collector-0.1.9.tgz",
-      "integrity": "sha512-JNRBaHQ47pWsXydj4gUp7zc64/0pM89a6E9pA8uQ15l1KxPGYYTrNRdone5aJqLTFOoPl3tYeF1kXj+3nU1nEA==",
+      "version": "0.1.10",
+      "resolved": "https://registry.npmjs.org/@jambonz/stats-collector/-/stats-collector-0.1.10.tgz",
+      "integrity": "sha512-TTqE56/Mf0JDcR+JTB2aRheAYgiEqRWCeh1cxHKAXAuD3pvc9FKPt0TxZw797BZfkqMhK1inSPMe8HRUzm1wCQ==",
       "dependencies": {
         "debug": "^4.3.2",
         "hot-shots": "^8.5.0"
@@ -2360,9 +2360,9 @@
       }
     },
     "node_modules/@jambonz/verb-specifications": {
-      "version": "0.0.69",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.69.tgz",
-      "integrity": "sha512-DWnz7XRkCzpzyCVJH7NtScv+wSlUC414/EO8j/gPZs3RT4WBW1OBXwXpfjURHcSrDG7lycz+tfA+2WoUdW/W+g==",
+      "version": "0.0.72",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.72.tgz",
+      "integrity": "sha512-sjA+/LQP2p1zE02UByy9OaAaSxbfQNxQ6D0pwYoMG42U8n+8Det+GFM/9+oFVnbNjUH9bvgT8vrR57U0lU4Cpw==",
       "dependencies": {
         "debug": "^4.3.4",
         "pino": "^8.8.0"
@@ -4606,15 +4606,15 @@
       }
     },
     "node_modules/drachtio-fsmrf": {
-      "version": "3.0.41",
-      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.41.tgz",
-      "integrity": "sha512-LemyjXtOnd5tOcQSmMgGtKUSdFAM3pLkEwGtV1pRnRhLTS3oERqQwuqRv0LurfZQ38WpuMOrUBhCSb3+uW9t/w==",
+      "version": "3.0.43",
+      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.43.tgz",
+      "integrity": "sha512-ZDgO4WCzZssmOpf+ng20P8r5pu7demWFiECqwRivbWadXqAm3LNabZwWOred2MdCRDvzgHWUwBZoJPzCqKBL5g==",
       "dependencies": {
         "camel-case": "^4.1.2",
         "debug": "^2.6.9",
         "delegates": "^0.1.0",
         "drachtio-modesl": "^1.2.9",
-        "drachtio-srf": "^4.5.31",
+        "drachtio-srf": "^4.5.38",
         "only": "^0.0.2",
         "sdp-transform": "^2.14.1",
         "snake-case": "^3.0.4",
@@ -4663,9 +4663,9 @@
       }
     },
     "node_modules/drachtio-srf": {
-      "version": "4.5.31",
-      "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-4.5.31.tgz",
-      "integrity": "sha512-/M4J8h2aqHtMXWr8/UHngKQsY9sQQxjdd23jDTSpNVpCwgZ2/xZFhbg/B/UCjrarSRzbyDCvuluOAtaPRSw7Hw==",
+      "version": "4.5.38",
+      "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-4.5.38.tgz",
+      "integrity": "sha512-dEafgn1OPfkc30Gq3JwJBF4Q7O96fsuqMnE2OO1NqRG0DkwworMIPsusWDo0ePbi+NH74rMjLPmVyC/owA0HXg==",
       "dependencies": {
         "debug": "^3.2.7",
         "delegates": "^0.1.0",
@@ -11954,9 +11954,9 @@
       }
     },
     "@jambonz/speech-utils": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.1.0.tgz",
-      "integrity": "sha512-45K6Vrl2PMEbbcnvm65afCDujDxck/bEUq7+P6KRw/cei3mrKtwjGh3HXi1cKhC1gA5UF1+5YrUoPO9LdoZnog==",
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.1.3.tgz",
+      "integrity": "sha512-Ess3yc8XyJAoUvXFz9maLq4NgxkTkTgiN2uW3rgOFiRr7b6l9A3oNQrN/ZJCJc9Ge+I3CypFmcwCiLNehuqF/g==",
       "requires": {
         "@aws-sdk/client-polly": "^3.496.0",
         "@aws-sdk/client-sts": "^3.496.0",
@@ -11974,9 +11974,9 @@
       }
     },
     "@jambonz/stats-collector": {
-      "version": "0.1.9",
-      "resolved": "https://registry.npmjs.org/@jambonz/stats-collector/-/stats-collector-0.1.9.tgz",
-      "integrity": "sha512-JNRBaHQ47pWsXydj4gUp7zc64/0pM89a6E9pA8uQ15l1KxPGYYTrNRdone5aJqLTFOoPl3tYeF1kXj+3nU1nEA==",
+      "version": "0.1.10",
+      "resolved": "https://registry.npmjs.org/@jambonz/stats-collector/-/stats-collector-0.1.10.tgz",
+      "integrity": "sha512-TTqE56/Mf0JDcR+JTB2aRheAYgiEqRWCeh1cxHKAXAuD3pvc9FKPt0TxZw797BZfkqMhK1inSPMe8HRUzm1wCQ==",
       "requires": {
         "debug": "^4.3.2",
         "hot-shots": "^8.5.0"
@@ -11992,9 +11992,9 @@
       }
     },
     "@jambonz/verb-specifications": {
-      "version": "0.0.69",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.69.tgz",
-      "integrity": "sha512-DWnz7XRkCzpzyCVJH7NtScv+wSlUC414/EO8j/gPZs3RT4WBW1OBXwXpfjURHcSrDG7lycz+tfA+2WoUdW/W+g==",
+      "version": "0.0.72",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.72.tgz",
+      "integrity": "sha512-sjA+/LQP2p1zE02UByy9OaAaSxbfQNxQ6D0pwYoMG42U8n+8Det+GFM/9+oFVnbNjUH9bvgT8vrR57U0lU4Cpw==",
       "requires": {
         "debug": "^4.3.4",
         "pino": "^8.8.0"
@@ -13702,15 +13702,15 @@
       }
     },
     "drachtio-fsmrf": {
-      "version": "3.0.41",
-      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.41.tgz",
-      "integrity": "sha512-LemyjXtOnd5tOcQSmMgGtKUSdFAM3pLkEwGtV1pRnRhLTS3oERqQwuqRv0LurfZQ38WpuMOrUBhCSb3+uW9t/w==",
+      "version": "3.0.43",
+      "resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.43.tgz",
+      "integrity": "sha512-ZDgO4WCzZssmOpf+ng20P8r5pu7demWFiECqwRivbWadXqAm3LNabZwWOred2MdCRDvzgHWUwBZoJPzCqKBL5g==",
       "requires": {
         "camel-case": "^4.1.2",
         "debug": "^2.6.9",
         "delegates": "^0.1.0",
         "drachtio-modesl": "^1.2.9",
-        "drachtio-srf": "^4.5.31",
+        "drachtio-srf": "^4.5.38",
         "only": "^0.0.2",
         "sdp-transform": "^2.14.1",
         "snake-case": "^3.0.4",
@@ -13754,9 +13754,9 @@
       }
     },
     "drachtio-srf": {
-      "version": "4.5.31",
-      "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-4.5.31.tgz",
-      "integrity": "sha512-/M4J8h2aqHtMXWr8/UHngKQsY9sQQxjdd23jDTSpNVpCwgZ2/xZFhbg/B/UCjrarSRzbyDCvuluOAtaPRSw7Hw==",
+      "version": "4.5.38",
+      "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-4.5.38.tgz",
+      "integrity": "sha512-dEafgn1OPfkc30Gq3JwJBF4Q7O96fsuqMnE2OO1NqRG0DkwworMIPsusWDo0ePbi+NH74rMjLPmVyC/owA0HXg==",
       "requires": {
         "debug": "^3.2.7",
         "delegates": "^0.1.0",

package.json

@@ -31,10 +31,10 @@
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/mw-registrar": "^0.2.7",
     "@jambonz/realtimedb-helpers": "^0.8.8",
-    "@jambonz/speech-utils": "^0.1.0",
-    "@jambonz/stats-collector": "^0.1.9",
+    "@jambonz/speech-utils": "^0.1.3",
+    "@jambonz/stats-collector": "^0.1.10",
     "@jambonz/time-series": "^0.2.8",
-    "@jambonz/verb-specifications": "^0.0.69",
+    "@jambonz/verb-specifications": "^0.0.72",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/exporter-jaeger": "^1.23.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
@@ -47,8 +47,8 @@
     "bent": "^7.3.12",
     "debug": "^4.3.4",
     "deepcopy": "^2.1.0",
-    "drachtio-fsmrf": "^3.0.41",
-    "drachtio-srf": "^4.5.31",
+    "drachtio-fsmrf": "^3.0.43",
+    "drachtio-srf": "^4.5.35",
     "express": "^4.19.2",
     "express-validator": "^7.0.1",
     "ip": "^2.0.1",