change population of test data

* fix: Re-invite sip rec does not update media

* fix: Re-invite sip rec does not update media

README.md

@@ -18,6 +18,7 @@ Configuration is provided via environment variables:
 |DRACHTIO_PORT| listening port of drachtio server for control connections (typically 9022)|yes|
 |DRACHTIO_SECRET| shared secret|yes|
 |ENABLE_METRICS| if 1, metrics will be generated|no|
+|ENCRYPTION_SECRET| secret for credential encryption(JWT_SECRET is deprecated) |yes|
 |GOOGLE_APPLICATION_CREDENTIALS| path to gcp service key file|yes|
 |HTTP_PORT| tcp port to listen on for API requests from jambonz-api-server|yes|
 |JAMBONES_GATHER_EARLY_HINTS_MATCH| if true and hints are provided, gather will opportunistically review interim transcripts if possible to reduce ASR latency |no|

app.js

@@ -8,6 +8,7 @@ assert.ok(process.env.DRACHTIO_SECRET, 'missing DRACHTIO_SECRET env var');
 assert.ok(process.env.JAMBONES_FREESWITCH, 'missing JAMBONES_FREESWITCH env var');
 assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
 assert.ok(process.env.JAMBONES_NETWORK_CIDR || process.env.K8S, 'missing JAMBONES_SUBNET env var');
+assert.ok(process.env.ENCRYPTION_SECRET || process.env.JWT_SECRET, 'missing ENCRYPTION_SECRET env var');
 
 const Srf = require('drachtio-srf');
 const srf = new Srf();

lib/http-routes/api/create-call.js

@@ -104,7 +104,7 @@ router.post('/', async(req, res) => {
       proxy: `sip:${sbcAddress}`,
       localSdp: ep.local.sdp
     });
-    if (target.auth) opts.auth = this.target.auth;
+    if (target.auth) opts.auth = target.auth;
 
 
     /**

lib/middleware.js

@@ -27,7 +27,11 @@ module.exports = function(srf, logger) {
 
   function initLocals(req, res, next) {
     const callId = req.get('Call-ID');
-    logger.info({callId}, 'new incoming call');
+    logger.info({
+      callId,
+      callingNumber: req.callingNumber,
+      calledNumber: req.calledNumber
+    }, 'new incoming call');
     if (!req.has('X-Account-Sid')) {
       logger.info('getAccountDetails - rejecting call due to missing X-Account-Sid header');
       return res.send(500);

lib/session/call-session.js

@@ -511,12 +511,24 @@ class CallSession extends Emitter {
 
   async enableBotMode(gather, autoEnable) {
     try {
-      if (this.backgroundGatherTask) {
-        this.logger.info('CallSession:enableBotMode - bot mode currently enabled, ignoring request to start again');
-        return;
-      }
       const t = normalizeJambones(this.logger, [gather]);
-      this.backgroundGatherTask = makeTask(this.logger, t[0]);
+      const task = makeTask(this.logger, t[0]);
+
+      if (this.isBotModeEnabled) {
+        const currInput = this.backgroundGatherTask.input;
+        const newInput = task.input;
+        if (JSON.stringify(currInput) === JSON.stringify(newInput)) {
+          this.logger.info('CallSession:enableBotMode - bot mode currently enabled, ignoring request to start again');
+          return;
+        }
+        else {
+          this.logger.info({currInput, newInput},
+            'CallSession:enableBotMode - restarting background gather to apply new input type');
+          this.backgroundGatherTask.sticky = false;
+          this.disableBotMode();
+        }
+      }
+      this.backgroundGatherTask = task;
       this._bargeInEnabled = true;
       this.backgroundGatherTask
         .once('dtmf', this._clearTasks.bind(this, this.backgroundGatherTask))
@@ -528,13 +540,15 @@ class CallSession extends Emitter {
       const {span, ctx} = this.rootSpan.startChildSpan(`background-gather:${this.backgroundGatherTask.summary}`);
       this.backgroundGatherTask.span = span;
       this.backgroundGatherTask.ctx = ctx;
+      this.backgroundGatherTask.sticky = autoEnable;
       this.backgroundGatherTask.exec(this, resources)
         .then(() => {
           this.logger.info('CallSession:enableBotMode: gather completed');
           this.backgroundGatherTask && this.backgroundGatherTask.removeAllListeners();
           this.backgroundGatherTask && this.backgroundGatherTask.span.end();
+          const sticky = this.backgroundGatherTask?.sticky;
           this.backgroundGatherTask = null;
-          if (autoEnable && !this.callGone && !this._stopping && this._bargeInEnabled) {
+          if (sticky && !this.callGone && !this._stopping && this._bargeInEnabled) {
             this.logger.info('CallSession:enableBotMode: restarting background gather');
             setImmediate(() => this.enableBotMode(gather, true));
           }
@@ -636,7 +650,9 @@ class CallSession extends Emitter {
           return {
             speech_credential_sid: credential.speech_credential_sid,
             client_id: credential.client_id,
-            secret: credential.secret
+            secret: credential.secret,
+            nuance_tts_uri: credential.nuance_tts_uri,
+            nuance_stt_uri: credential.nuance_stt_uri
           };
         }
         else if ('deepgram' === vendor) {
@@ -660,6 +676,14 @@ class CallSession extends Emitter {
             stt_region: credential.stt_region
           };
         }
+        else if (vendor.startsWith('custom:')) {
+          return {
+            speech_credential_sid: credential.speech_credential_sid,
+            auth_token: credential.auth_token,
+            custom_stt_url: credential.custom_stt_url,
+            custom_tts_url: credential.custom_tts_url
+          };
+        }
       }
       else {
         writeAlerts({
@@ -690,7 +714,7 @@ class CallSession extends Emitter {
         let skip = false;
         this.currentTask = task;
         if (TaskName.Gather === task.name && this.isBotModeEnabled) {
-          if (this.backgroundGatherTask.updateTaskInProgress(task)) {
+          if (this.backgroundGatherTask.updateTaskInProgress(task) !== false) {
             this.logger.info(`CallSession:exec skipping #${stackNum}:${taskNum}: ${task.name}`);
             skip = true;
           }
@@ -754,7 +778,6 @@ class CallSession extends Emitter {
 
   trackTmpFile(path) {
     // TODO: don't add if its already in the list (should we make it a set?)
-    this.logger.debug(`adding tmp file to track ${path}`);
     this.tmpFiles.add(path);
   }
 
@@ -1129,14 +1152,14 @@ class CallSession extends Emitter {
   _injectTasks(newTasks) {
     const gatherPos = this.tasks.map((t) => t.name).indexOf(TaskName.Gather);
     const currentlyExecutingGather = this.currentTask?.name === TaskName.Gather;
-
+    /*
     this.logger.debug({
       currentTaskList: listTaskNames(this.tasks),
       newContent: listTaskNames(newTasks),
       currentlyExecutingGather,
       gatherPos
     }, 'CallSession:_injectTasks - starting');
-
+    */
     const killGather = () => {
       this.logger.debug('CallSession:_injectTasks - killing current gather because we have new content');
       this.currentTask.kill(this);
@@ -1145,10 +1168,11 @@ class CallSession extends Emitter {
     if (-1 === gatherPos) {
       /* no gather in the stack  simply append tasks */
       this.tasks.push(...newTasks);
+      /*
       this.logger.debug({
         updatedTaskList: listTaskNames(this.tasks)
       }, 'CallSession:_injectTasks - completed (simple append)');
-
+      */
       /* we do need to kill the current gather if we are executing one */
       if (currentlyExecutingGather) killGather();
       return;
@@ -1176,12 +1200,10 @@ class CallSession extends Emitter {
             this.replaceApplication(t);
           }
           else if (process.env.JAMBONES_INJECT_CONTENT) {
-            this.logger.debug({tasks: listTaskNames(t)}, 'CallSession:_onCommand - queueing tasks (injecting content)');
             this._injectTasks(t);
             this.logger.info({tasks: listTaskNames(this.tasks)}, 'CallSession:_onCommand - updated task list');
           }
           else {
-            this.logger.debug({tasks: listTaskNames(t)}, 'CallSession:_onCommand - queueing tasks');
             this.tasks.push(...t);
             this.logger.info({tasks: listTaskNames(this.tasks)}, 'CallSession:_onCommand - updated task list');
           }
@@ -1225,7 +1247,7 @@ class CallSession extends Emitter {
         this.logger.info(`CallSession:_onCommand - invalid command ${command}`);
     }
     if (this.wakeupResolver) {
-      this.logger.debug({resolution}, 'CallSession:_onCommand - got commands, waking up..');
+      //this.logger.debug({resolution}, 'CallSession:_onCommand - got commands, waking up..');
       this.wakeupResolver(resolution);
       this.wakeupResolver = null;
     }

lib/session/siprec-call-session.js

@@ -1,6 +1,7 @@
 const InboundCallSession = require('./inbound-call-session');
 const {createSipRecPayload} = require('../utils/siprec-utils');
 const {CallStatus} = require('../utils/constants');
+const {parseSiprecPayload} = require('../utils/siprec-utils');
 /**
  * @classdesc Subclass of InboundCallSession.  This represents a CallSession that is
  * established for an inbound SIPREC call.
@@ -16,6 +17,32 @@ class SipRecCallSession extends InboundCallSession {
     this.metadata = metadata;
   }
 
+  async _onReinvite(req, res) {
+    try {
+      this.logger.info(req.payload, 'SipRec Re-INVITE payload');
+      const {sdp1: reSdp1, sdp2: reSdp2, metadata: reMetadata} = await parseSiprecPayload(req, this.logger);
+      this.sdp1 = reSdp1;
+      this.sdp2 = reSdp2;
+      this.metadata = reMetadata;
+
+      if (this.ep && this.ep2) {
+        let remoteSdp = this.sdp1.replace(/sendonly/, 'sendrecv');
+        const newSdp1 = await this.ep.modify(remoteSdp);
+        remoteSdp = this.sdp2.replace(/sendonly/, 'sendrecv');
+        const newSdp2 = await this.ep2.modify(remoteSdp);
+        const combinedSdp = await createSipRecPayload(newSdp1, newSdp2, this.logger);
+        res.send(200, {body: combinedSdp});
+        this.logger.info({offer: req.body, answer: combinedSdp}, 'SipRec handling reINVITE');
+      }
+      else {
+        this.logger.info('got SipRec reINVITE but no endpoint and media has not been released');
+        res.send(488);
+      }
+    } catch (err) {
+      this.logger.error(err, 'Error handling reinvite');
+    }
+  }
+
   async answerSipRecCall() {
     try {
       this.ms = this.getMS();

lib/tasks/dial.js

@@ -400,15 +400,19 @@ class TaskDial extends Task {
     let fqdn;
 
     if (!sbcAddress) throw new Error('no SBC found for outbound call');
+    this.headers = {
+      'X-Account-Sid': cs.accountSid,
+      ...(req && req.has('X-CID') && {'X-CID': req.get('X-CID')}),
+      ...(req && req.has('P-Asserted-Identity') && {'P-Asserted-Identity': req.get('P-Asserted-Identity')}),
+      // Put headers at the end to make sure opt.headers override all default behavior.
+      ...this.headers
+    };
+
     const opts = {
-      headers: req && req.has('X-CID') ? Object.assign(this.headers, {'X-CID': req.get('X-CID')}) : this.headers,
+      headers: this.headers,
       proxy: `sip:${sbcAddress}`,
       callingNumber: this.callerId || req.callingNumber
     };
-    opts.headers = {
-      ...opts.headers,
-      'X-Account-Sid': cs.accountSid
-    };
 
     const t = this.target.find((t) => t.type === 'teams');
     if (t) {

lib/tasks/gather.js

@@ -9,7 +9,8 @@ const {
   DeepgramTranscriptionEvents,
   SonioxTranscriptionEvents,
   IbmTranscriptionEvents,
-  NvidiaTranscriptionEvents
+  NvidiaTranscriptionEvents,
+  JambonzTranscriptionEvents
 } = require('../utils/constants');
 
 const makeTask = require('./make_task');
@@ -53,7 +54,7 @@ class TaskGather extends Task {
 
     /* timeout of zero means no timeout */
     this.timeout = this.timeout === 0 ? 0 : (this.timeout || 15) * 1000;
-    this.interim = !!this.partialResultHook || this.bargein;
+    this.interim = !!this.partialResultHook || this.bargein || (this.timeout > 0);
     this.listenDuringPrompt = this.data.listenDuringPrompt === false ? false : true;
     this.minBargeinWordCount = this.data.minBargeinWordCount || 1;
     if (this.data.recognizer) {
@@ -69,6 +70,11 @@ class TaskGather extends Task {
       if (this.asrTimeout > 0) this.asrDtmfTerminationDigit = recognizer.asrDtmfTerminationDigit;
       this.isContinuousAsr = this.asrTimeout > 0;
 
+      if (Array.isArray(this.data.recognizer.hints) &&
+        0 == this.data.recognizer.hints.length && process.env.JAMBONES_GATHER_CLEAR_GLOBAL_HINTS_ON_EMPTY_HINTS) {
+        logger.debug('Gather: an empty hints array was supplied, so we will mask global hints');
+        this.maskGlobalSttHints = true;
+      }
       this.data.recognizer.hints = this.data.recognizer.hints || [];
       this.data.recognizer.altLanguages = this.data.recognizer.altLanguages || [];
     }
@@ -92,12 +98,17 @@ class TaskGather extends Task {
     this._sonioxTranscripts = [];
 
     this.parentTask = parentTask;
+    this.partialTranscriptsCount = 0;
   }
 
   get name() { return TaskName.Gather; }
 
   get needsStt() { return this.input.includes('speech'); }
 
+  get wantsSingleUtterance() {
+    return this.data.recognizer?.singleUtterance === true;
+  }
+
   get earlyMedia() {
     return (this.sayTask && this.sayTask.earlyMedia) ||
       (this.playTask && this.playTask.earlyMedia);
@@ -119,14 +130,17 @@ class TaskGather extends Task {
   }
 
   async exec(cs, {ep}) {
-    this.logger.debug('Gather:exec');
+    this.logger.debug({options: this.data}, 'Gather:exec');
     await super.exec(cs);
     const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
     const {getNuanceAccessToken, getIbmAccessToken} = cs.srf.locals.dbHelpers;
 
-    if (cs.hasGlobalSttHints) {
+    if (cs.hasGlobalSttHints && !this.maskGlobalSttHints) {
       const {hints, hintsBoost} = cs.globalSttHints;
-      this.data.recognizer.hints = this.data.recognizer.hints.concat(hints);
+      const setOfHints = new Set(this.data.recognizer.hints
+        .concat(hints)
+        .filter((h) => typeof h === 'string' && h.length > 0));
+      this.data.recognizer.hints = [...setOfHints];
       if (!this.data.recognizer.hintsBoost && hintsBoost) this.data.recognizer.hintsBoost = hintsBoost;
       this.logger.debug({hints: this.data.recognizer.hints, hintsBoost: this.data.recognizer.hintsBoost},
         'Gather:exec - applying global sttHints');
@@ -187,7 +201,6 @@ class TaskGather extends Task {
       throw new Error(`No speech-to-text service credentials for ${this.vendor} have been configured`);
     }
 
-    this.logger.info({sttCredentials: this.sttCredentials}, 'Gather:exec - sttCredentials');
     if (this.vendor === 'nuance' && this.sttCredentials.client_id) {
       /* get nuance access token */
       const {client_id, secret} = this.sttCredentials;
@@ -206,7 +219,6 @@ class TaskGather extends Task {
       this._startTimer();
       if (this.isContinuousAsr && 0 === this.timeout) this._startAsrTimer();
       if (this.input.includes('speech') && !this.listenDuringPrompt) {
-        this.logger.debug('Gather:exec - calling _initSpeech');
         this._initSpeech(cs, ep)
           .then(() => {
             if (this.killed) {
@@ -308,6 +320,7 @@ class TaskGather extends Task {
     const {timeout} = opts;
     this.timeout = timeout;
     this._startTimer();
+    return true;
   }
 
   _onDtmf(cs, ep, evt) {
@@ -347,7 +360,6 @@ class TaskGather extends Task {
 
   async _initSpeech(cs, ep) {
     const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.data.recognizer);
-    this.logger.debug(opts, 'TaskGather:_initSpeech - channel vars');
     switch (this.vendor) {
       case 'google':
         this.bugname = 'google_transcribe';
@@ -379,8 +391,6 @@ class TaskGather extends Task {
           this._onTranscriptionComplete.bind(this, cs, ep));
         ep.addCustomEventListener(NuanceTranscriptionEvents.VadDetected,
           this._onVadDetected.bind(this, cs, ep));
-        ep.addCustomEventListener(NuanceTranscriptionEvents.Error,
-          this._onNuanceError.bind(this, cs, ep));
 
         /* stall timers until prompt finishes playing */
         if ((this.sayTask || this.playTask) && this.listenDuringPrompt) {
@@ -399,8 +409,6 @@ class TaskGather extends Task {
       case 'soniox':
         this.bugname = 'soniox_transcribe';
         ep.addCustomEventListener(SonioxTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
-        ep.addCustomEventListener(SonioxTranscriptionEvents.Error,
-          this._onSonioxError.bind(this, cs, ep));
         break;
 
       case 'ibm':
@@ -409,8 +417,6 @@ class TaskGather extends Task {
         ep.addCustomEventListener(IbmTranscriptionEvents.Connect, this._onIbmConnect.bind(this, cs, ep));
         ep.addCustomEventListener(IbmTranscriptionEvents.ConnectFailure,
           this._onIbmConnectFailure.bind(this, cs, ep));
-        ep.addCustomEventListener(IbmTranscriptionEvents.Error,
-          this._onIbmError.bind(this, cs, ep));
         break;
 
       case 'nvidia':
@@ -423,8 +429,6 @@ class TaskGather extends Task {
           this._onTranscriptionComplete.bind(this, cs, ep));
         ep.addCustomEventListener(NvidiaTranscriptionEvents.VadDetected,
           this._onVadDetected.bind(this, cs, ep));
-        ep.addCustomEventListener(NvidiaTranscriptionEvents.Error,
-          this._onNvidiaError.bind(this, cs, ep));
 
         /* I think nvidia has this (??) - stall timers until prompt finishes playing */
         if ((this.sayTask || this.playTask) && this.listenDuringPrompt) {
@@ -433,11 +437,23 @@ class TaskGather extends Task {
         break;
 
       default:
-        this.notifyError({ msg: 'ASR error', details:`Invalid vendor ${this.vendor}`});
-        this.notifyTaskDone();
-        throw new Error(`Invalid vendor ${this.vendor}`);
+        if (this.vendor.startsWith('custom:')) {
+          this.bugname = `${this.vendor}_transcribe`;
+          ep.addCustomEventListener(JambonzTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+          ep.addCustomEventListener(JambonzTranscriptionEvents.Connect, this._onJambonzConnect.bind(this, cs, ep));
+          ep.addCustomEventListener(JambonzTranscriptionEvents.ConnectFailure,
+            this._onJambonzConnectFailure.bind(this, cs, ep));
+          break;
+        }
+        else {
+          this.notifyError({ msg: 'ASR error', details:`Invalid vendor ${this.vendor}`});
+          this.notifyTaskDone();
+          throw new Error(`Invalid vendor ${this.vendor}`);
+        }
     }
 
+    /* common handler for all stt engine errors */
+    ep.addCustomEventListener(JambonzTranscriptionEvents.Error, this._onJambonzError.bind(this, cs, ep));
     await ep.set(opts)
       .catch((err) => this.logger.info(err, 'Error setting channel variables'));
   }
@@ -545,8 +561,13 @@ class TaskGather extends Task {
     }
 
     evt = this.normalizeTranscription(evt, this.vendor, 1, this.language);
+    if (evt.alternatives.length === 0) {
+      this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, continue listening');
+      return;
+    }
 
-    if (this.earlyHintsMatch && evt.is_final === false) {
+    /* fast path: our first partial transcript exactly matches an early hint */
+    if (this.earlyHintsMatch && evt.is_final === false && this.partialTranscriptsCount++ === 0) {
       const transcript = evt.alternatives[0].transcript?.toLowerCase();
       const hints = this.data.recognizer?.hints || [];
       if (hints.find((h) => h.toLowerCase() === transcript)) {
@@ -620,6 +641,8 @@ class TaskGather extends Task {
         others do not.
       */
       //const isStableEnough = typeof evt.stability === 'undefined' || evt.stability > GATHER_STABILITY_THRESHOLD;
+      this._clearTimer();
+      this._startTimer();
       if (this.bargein && (words + bufferedWords) >= this.minBargeinWordCount) {
         if (!this.playComplete) {
           this.logger.debug({transcript: evt.alternatives[0].transcript}, 'killing audio due to speech');
@@ -648,7 +671,16 @@ class TaskGather extends Task {
       this._killAudio(cs);
     }
 
-    if (!this.resolved && !this.killed && !this._bufferedTranscripts.length) {
+    /**
+     * By default, Gather asks google for multiple utterances.
+     * The reason is that we can sometimes get an 'end_of_utterance' event without
+     * getting a transcription.  This can happen if someone coughs or mumbles.
+     * For that reason don't ask for a single utterance and we'll terminate the transcribe operation
+     * once we get a final transcript.
+     * However, if the usr has specified a singleUtterance, then we need to restart here
+     * since we dont have a final transcript yet.
+     */
+    if (!this.resolved && !this.killed && !this._bufferedTranscripts.length && this.wantsSingleUtterance) {
       this._startTranscribing(ep);
     }
   }
@@ -662,26 +694,30 @@ class TaskGather extends Task {
   _onTranscriptionComplete(cs, ep) {
     this.logger.debug('TaskGather:_onTranscriptionComplete');
   }
-  _onNuanceError(cs, ep, evt) {
-    const {code, error, details} = evt;
-    if (code === 404 && error === 'No speech') {
-      this.logger.debug({code, error, details}, 'TaskGather:_onNuanceError');
-      return this._resolve('timeout');
-    }
-    this.logger.info({code, error, details}, 'TaskGather:_onNuanceError');
-    if (code === 413 && error === 'Too much speech') {
-      return this._resolve('timeout');
-    }
-  }
-  _onSonioxError(cs, ep, evt) {
-    this.logger.info({evt}, 'TaskGather:_onSonioxError');
-  }
-  _onNvidiaError(cs, ep, evt) {
-    this.logger.info({evt}, 'TaskGather:_onNvidiaError');
-  }
   _onDeepgramConnect(_cs, _ep) {
     this.logger.debug('TaskGather:_onDeepgramConnect');
   }
+  _onJambonzConnect(_cs, _ep) {
+    this.logger.debug('TaskGather:_onJambonzConnect');
+  }
+  _onJambonzError(cs, _ep, evt) {
+    this.logger.info({evt}, 'TaskGather:_onJambonzError');
+    const {writeAlerts, AlertType} = cs.srf.locals;
+
+    if (this.vendor === 'nuance') {
+      const {code, error} = evt;
+      if (code === 404 && error === 'No speech') return this._resolve('timeout');
+      if (code === 413 && error === 'Too much speech') return this._resolve('timeout');
+    }
+    this.logger.info({evt}, 'TaskGather:_onJambonzError');
+    writeAlerts({
+      account_sid: cs.accountSid,
+      alert_type: AlertType.STT_FAILURE,
+      message: `Custom speech vendor ${this.vendor} error: ${evt.error}`,
+      vendor: this.vendor,
+    }).catch((err) => this.logger.info({err}, 'Error generating alert for jambonz custom connection failure'));
+    this.notifyError({msg: 'ASR error', details:`Custom speech vendor ${this.vendor} error: ${evt.error}`});
+  }
 
   _onDeepGramConnectFailure(cs, _ep, evt) {
     const {reason} = evt;
@@ -696,6 +732,19 @@ class TaskGather extends Task {
     this.notifyError({msg: 'ASR error', details:`Failed connecting to speech vendor deepgram: ${reason}`});
     this.notifyTaskDone();
   }
+  _onJambonzConnectFailure(cs, _ep, evt) {
+    const {reason} = evt;
+    const {writeAlerts, AlertType} = cs.srf.locals;
+    this.logger.info({evt}, 'TaskGather:_onJambonzConnectFailure');
+    writeAlerts({
+      account_sid: cs.accountSid,
+      alert_type: AlertType.STT_FAILURE,
+      message: `Failed connecting to ${this.vendor} speech recognizer: ${reason}`,
+      vendor: this.vendor,
+    }).catch((err) => this.logger.info({err}, 'Error generating alert for jambonz custom connection failure'));
+    this.notifyError({msg: 'ASR error', details:`Failed connecting to speech vendor ${this.vendor}: ${reason}`});
+    this.notifyTaskDone();
+  }
 
   _onIbmConnect(_cs, _ep) {
     this.logger.debug('TaskGather:_onIbmConnect');
@@ -745,6 +794,10 @@ class TaskGather extends Task {
     if (this.resolved) return;
 
     this.resolved = true;
+    // Clear dtmf event
+    if (this.dtmfBargein) {
+      this.ep.removeAllListeners('dtmf');
+    }
     clearTimeout(this.interDigitTimer);
     this._clearTimer();
 

lib/tasks/listen.js

@@ -2,6 +2,7 @@ const Task = require('./task');
 const {TaskName, TaskPreconditions, ListenEvents, ListenStatus} = require('../utils/constants');
 const makeTask = require('./make_task');
 const moment = require('moment');
+const MAX_PLAY_AUDIO_QUEUE_SIZE = 10;
 
 class TaskListen extends Task {
   constructor(logger, opts, parentTask) {
@@ -20,6 +21,8 @@ class TaskListen extends Task {
     this.nested = parentTask instanceof Task;
 
     this.results = {};
+    this.playAudioQueue = [];
+    this.isPlayingAudioFromQueue = false;
 
     if (this.transcribe) this.transcribeTask = makeTask(logger, {'transcribe': opts.transcribe}, this);
   }
@@ -58,6 +61,7 @@ class TaskListen extends Task {
     super.kill(cs);
     this.logger.debug(`TaskListen:kill endpoint connected? ${this.ep && this.ep.connected}`);
     this._clearTimer();
+    this.playAudioQueue = [];
     if (this.ep && this.ep.connected) {
       this.logger.debug('TaskListen:kill closing websocket');
       try {
@@ -184,16 +188,36 @@ class TaskListen extends Task {
     this.notifyTaskDone();
   }
 
-  async _onPlayAudio(ep, evt) {
-    this.logger.info(`received play_audio event: ${JSON.stringify(evt)}`);
+  async _playAudio(ep, evt, logger) {
     try {
       const results = await ep.play(evt.file);
-      this.logger.debug(`Finished playing file, result: ${JSON.stringify(results)}`);
+      logger.debug(`Finished playing file, result: ${JSON.stringify(results)}`);
       ep.forkAudioSendText({type: 'playDone', data: Object.assign({id: evt.id}, results)});
+    } catch (err) {
+      logger.error({err}, 'Error playing file');
     }
-    catch (err) {
-      this.logger.error({err}, 'Error playing file');
+  }
+
+  async _onPlayAudio(ep, evt) {
+    this.logger.info(`received play_audio event: ${JSON.stringify(evt)}`);
+    if (!evt.queuePlay) {
+      this.playAudioQueue = [];
+      this._playAudio(ep, evt, this.logger);
+      this.isPlayingAudioFromQueue = false;
+      return;
     }
+
+    if (this.playAudioQueue.length <= MAX_PLAY_AUDIO_QUEUE_SIZE) {
+      this.playAudioQueue.push(evt);
+    }
+
+    if (this.isPlayingAudioFromQueue) return;
+
+    this.isPlayingAudioFromQueue = true;
+    while (this.playAudioQueue.length > 0) {
+      await this._playAudio(ep, this.playAudioQueue.shift(), this.logger);
+    }
+    this.isPlayingAudioFromQueue = false;
   }
 
   _onKillAudio(ep) {

lib/tasks/say.js

@@ -143,7 +143,7 @@ class TaskSay extends Task {
           span.end();
           writeAlerts({
             account_sid: cs.accountSid,
-            alert_type: AlertType.TTS_NOT_PROVISIONED,
+            alert_type: AlertType.TTS_FAILURE,
             vendor,
             detail: err.message
           }).catch((err) => this.logger.info({err}, 'Error generating alert for tts failure'));
@@ -154,7 +154,6 @@ class TaskSay extends Task {
 
       const arr = this.text.map((t) => generateAudio(t));
       const filepath = (await Promise.all(arr)).filter((fp) => fp && fp.length);
-      this.logger.debug({filepath}, 'synthesized files for tts');
       this.notifyStatus({event: 'start-playback'});
 
       while (!this.killed && (this.loop === 'forever' || this.loop--) && this.ep?.connected) {

lib/tasks/transcribe.js

@@ -9,7 +9,8 @@ const {
   DeepgramTranscriptionEvents,
   SonioxTranscriptionEvents,
   IbmTranscriptionEvents,
-  NvidiaTranscriptionEvents
+  NvidiaTranscriptionEvents,
+  JambonzTranscriptionEvents
 } = require('../utils/constants');
 const { normalizeJambones } = require('@jambonz/verb-specifications');
 
@@ -23,11 +24,13 @@ class TaskTranscribe extends Task {
       setChannelVarsForStt,
       normalizeTranscription,
       removeSpeechListeners,
-      setSpeechCredentialsAtRuntime
+      setSpeechCredentialsAtRuntime,
+      compileSonioxTranscripts
     } = require('../utils/transcription-utils')(logger);
     this.setChannelVarsForStt = setChannelVarsForStt;
     this.normalizeTranscription = normalizeTranscription;
     this.removeSpeechListeners = removeSpeechListeners;
+    this.compileSonioxTranscripts = compileSonioxTranscripts;
 
     this.transcriptionHook = this.data.transcriptionHook;
     this.earlyMedia = this.data.earlyMedia === true || (parentTask && parentTask.earlyMedia);
@@ -41,6 +44,9 @@ class TaskTranscribe extends Task {
     /* let credentials be supplied in the recognizer object at runtime */
     this.sttCredentials = setSpeechCredentialsAtRuntime(recognizer);
 
+    /* buffer for soniox transcripts */
+    this._sonioxTranscripts = [];
+
     recognizer.hints = recognizer.hints || [];
     recognizer.altLanguages = recognizer.altLanguages || [];
   }
@@ -184,8 +190,6 @@ class TaskTranscribe extends Task {
           this._onStartOfSpeech.bind(this, cs, ep, channel));
         ep.addCustomEventListener(NuanceTranscriptionEvents.TranscriptionComplete,
           this._onTranscriptionComplete.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(AzureTranscriptionEvents.Error,
-          this._onNuanceError.bind(this, cs, ep, channel));
         break;
       case 'deepgram':
         this.bugname = 'deepgram_transcribe';
@@ -198,9 +202,8 @@ class TaskTranscribe extends Task {
         break;
       case 'soniox':
         this.bugname = 'soniox_transcribe';
-        ep.addCustomEventListener(SonioxTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
-        ep.addCustomEventListener(SonioxTranscriptionEvents.Error,
-          this._onSonioxError.bind(this, cs, ep));
+        ep.addCustomEventListener(SonioxTranscriptionEvents.Transcription,
+          this._onTranscription.bind(this, cs, ep, channel));
         break;
       case 'ibm':
         this.bugname = 'ibm_transcribe';
@@ -210,8 +213,6 @@ class TaskTranscribe extends Task {
           this._onIbmConnect.bind(this, cs, ep, channel));
         ep.addCustomEventListener(IbmTranscriptionEvents.ConnectFailure,
           this._onIbmConnectFailure.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(IbmTranscriptionEvents.Error,
-          this._onIbmError.bind(this, cs, ep, channel));
         break;
 
       case 'nvidia':
@@ -224,14 +225,13 @@ class TaskTranscribe extends Task {
           this._onTranscriptionComplete.bind(this, cs, ep));
         ep.addCustomEventListener(NvidiaTranscriptionEvents.VadDetected,
           this._onVadDetected.bind(this, cs, ep));
-        ep.addCustomEventListener(NvidiaTranscriptionEvents.Error,
-          this._onNvidiaError.bind(this, cs, ep));
         break;
-
       default:
         throw new Error(`Invalid vendor ${this.vendor}`);
     }
 
+    /* common handler for all stt engine errors */
+    ep.addCustomEventListener(JambonzTranscriptionEvents.Error, this._onJambonzError.bind(this, cs, ep));
     await ep.set(opts)
       .catch((err) => this.logger.info(err, 'Error setting channel variables'));
 
@@ -259,8 +259,11 @@ class TaskTranscribe extends Task {
     this.logger.debug({evt}, 'TaskTranscribe:_onTranscription - before normalization');
 
     evt = this.normalizeTranscription(evt, this.vendor, channel, this.language);
-
     this.logger.debug({evt}, 'TaskTranscribe:_onTranscription');
+    if (evt.alternatives.length === 0) {
+      this.logger.info({evt}, 'TaskTranscribe:_onTranscription - got empty transcript, continue listening');
+      return;
+    }
 
     if (evt.alternatives[0]?.transcript === '' && !cs.callGone && !this.killed) {
       if (['microsoft', 'deepgram'].includes(this.vendor)) {
@@ -273,6 +276,15 @@ class TaskTranscribe extends Task {
       return;
     }
 
+    if (this.vendor === 'soniox') {
+      /* compile transcripts into one */
+      this._sonioxTranscripts.push(evt.vendor.finalWords);
+      if (evt.is_final) {
+        evt = this.compileSonioxTranscripts(this._sonioxTranscripts, 1, this.language);
+        this._sonioxTranscripts = [];
+      }
+    }
+
     if (this.transcriptionHook) {
       const b3 = this.getTracingPropagation();
       const httpHeaders = b3 && {b3};
@@ -321,23 +333,6 @@ class TaskTranscribe extends Task {
       this._timer = null;
     }
   }
-  _onNuanceError(_cs, _ep, _channel, evt) {
-    const {code, error, details} = evt;
-    if (code === 404 && error === 'No speech') {
-      this.logger.debug({code, error, details}, 'TaskTranscribe:_onNuanceError');
-      return this._resolve('timeout');
-    }
-    this.logger.info({code, error, details}, 'TaskTranscribe:_onNuanceError');
-    if (code === 413 && error === 'Too much speech') {
-      return this._resolve('timeout');
-    }
-  }
-  _onSonioxError(cs, ep, evt) {
-    this.logger.info({evt}, 'TaskTranscribe:_onSonioxError');
-  }
-  _onNvidiaError(cs, ep, evt) {
-    this.logger.info({evt}, 'TaskTranscribe:_onNvidiaError');
-  }
   _onDeepgramConnect(_cs, _ep) {
     this.logger.debug('TaskTranscribe:_onDeepgramConnect');
   }
@@ -376,6 +371,24 @@ class TaskTranscribe extends Task {
   _onIbmError(cs, _ep, _channel, evt) {
     this.logger.info({evt}, 'TaskTranscribe:_onIbmError');
   }
+  _onJambonzError(cs, _ep, evt) {
+    this.logger.info({evt}, 'TaskTranscribe:_onJambonzError');
+    const {writeAlerts, AlertType} = cs.srf.locals;
+
+    if (this.vendor === 'nuance') {
+      const {code, error} = evt;
+      if (code === 404 && error === 'No speech') return this._resolve('timeout');
+      if (code === 413 && error === 'Too much speech') return this._resolve('timeout');
+    }
+    this.logger.info({evt}, 'TaskTranscribe:_onJambonzError');
+    writeAlerts({
+      account_sid: cs.accountSid,
+      alert_type: AlertType.STT_FAILURE,
+      message: `Custom speech vendor ${this.vendor} error: ${evt.error}`,
+      vendor: this.vendor,
+    }).catch((err) => this.logger.info({err}, 'Error generating alert for jambonz custom connection failure'));
+    this.notifyError({msg: 'ASR error', details:`Custom speech vendor ${this.vendor} error: ${evt.error}`});
+  }
 
 
 }

lib/utils/amd-utils.js

@@ -267,7 +267,6 @@ module.exports = (logger) => {
       ep.addCustomEventListener(AzureTranscriptionEvents.Transcription, onTranscription.bind(null, cs, ep, task));
       ep.addCustomEventListener(AzureTranscriptionEvents.NoSpeechDetected, onNoSpeechDetected.bind(null, cs, ep, task));
     }
-    logger.debug({sttOpts}, 'startAmd: setting channel vars');
     await ep.set(sttOpts).catch((err) => logger.info(err, 'Error setting channel variables'));
 
     amd

lib/utils/constants.json

@@ -110,6 +110,12 @@
     "NoSpeechDetected": "azure_transcribe::no_speech_detected",
     "VadDetected": "azure_transcribe::vad_detected"
   },
+  "JambonzTranscriptionEvents": {
+    "Transcription": "jambonz_transcribe::transcription",
+    "ConnectFailure": "jambonz_transcribe::connect_failed",
+    "Connect": "jambonz_transcribe::connect",
+    "Error": "jambonz_transcribe::error"
+  },
   "ListenEvents": {
     "Connect": "mod_audio_fork::connect",
     "ConnectFailure": "mod_audio_fork::connect_failed",

lib/utils/db-utils.js

@@ -50,6 +50,8 @@ const speechMapper = (cred) => {
       const o = JSON.parse(decrypt(credential));
       obj.client_id = o.client_id;
       obj.secret = o.secret;
+      obj.nuance_tts_uri = o.nuance_tts_uri;
+      obj.nuance_stt_uri = o.nuance_stt_uri;
     }
     else if ('ibm' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
@@ -66,6 +68,12 @@ const speechMapper = (cred) => {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
     }
+    else if (obj.vendor.startsWith('custom:')) {
+      const o = JSON.parse(decrypt(credential));
+      obj.auth_token = o.auth_token;
+      obj.custom_stt_url = o.custom_stt_url;
+      obj.custom_tts_url = o.custom_tts_url;
+    }
   } catch (err) {
     console.log(err);
   }
@@ -83,53 +91,13 @@ module.exports = (logger, srf) => {
     const [r2] = await pp.query(sqlSpeechCredentials, account_sid);
     const speech = r2.map(speechMapper);
 
-    /* search at the service provider level if we don't find it at the account level */
-    const haveGoogle = speech.find((s) => s.vendor === 'google');
-    const haveAws = speech.find((s) => s.vendor === 'aws');
-    const haveMicrosoft = speech.find((s) => s.vendor === 'microsoft');
-    const haveWellsaid = speech.find((s) => s.vendor === 'wellsaid');
-    const haveNuance = speech.find((s) => s.vendor === 'nuance');
-    const haveDeepgram = speech.find((s) => s.vendor === 'deepgram');
-    const haveSoniox = speech.find((s) => s.vendor === 'soniox');
-    const haveIbm = speech.find((s) => s.vendor === 'ibm');
-    if (!haveGoogle || !haveAws || !haveMicrosoft || !haveWellsaid ||
-      !haveNuance || !haveIbm || !haveDeepgram || !haveSoniox) {
-      const [r3] = await pp.query(sqlSpeechCredentialsForSP, account_sid);
-      if (r3.length) {
-        if (!haveGoogle) {
-          const google = r3.find((s) => s.vendor === 'google');
-          if (google) speech.push(speechMapper(google));
-        }
-        if (!haveAws) {
-          const aws = r3.find((s) => s.vendor === 'aws');
-          if (aws) speech.push(speechMapper(aws));
-        }
-        if (!haveMicrosoft) {
-          const ms = r3.find((s) => s.vendor === 'microsoft');
-          if (ms) speech.push(speechMapper(ms));
-        }
-        if (!haveWellsaid) {
-          const wellsaid = r3.find((s) => s.vendor === 'wellsaid');
-          if (wellsaid) speech.push(speechMapper(wellsaid));
-        }
-        if (!haveNuance) {
-          const nuance = r3.find((s) => s.vendor === 'nuance');
-          if (nuance) speech.push(speechMapper(nuance));
-        }
-        if (!haveDeepgram) {
-          const deepgram = r3.find((s) => s.vendor === 'deepgram');
-          if (deepgram) speech.push(speechMapper(deepgram));
-        }
-        if (!haveSoniox) {
-          const soniox = r3.find((s) => s.vendor === 'soniox');
-          if (soniox) speech.push(speechMapper(soniox));
-        }
-        if (!haveIbm) {
-          const ibm = r3.find((s) => s.vendor === 'ibm');
-          if (ibm) speech.push(speechMapper(ibm));
-        }
+    /* add service provider creds unless we have that vendor at the account level */
+    const [r3] = await pp.query(sqlSpeechCredentialsForSP, account_sid);
+    r3.forEach((s) => {
+      if (!speech.find((s2) => s2.vendor === s.vendor)) {
+        speech.push(speechMapper(s));
       }
-    }
+    });
 
     return {
       ...r[0],

lib/utils/encrypt-decrypt.js

@@ -2,9 +2,9 @@ const crypto = require('crypto');
 const algorithm = process.env.LEGACY_CRYPTO ? 'aes-256-ctr' : 'aes-256-cbc';
 const iv = crypto.randomBytes(16);
 const secretKey = crypto.createHash('sha256')
-  .update(String(process.env.JWT_SECRET))
+  .update(process.env.ENCRYPTION_SECRET || process.env.JWT_SECRET)
   .digest('base64')
-  .substr(0, 32);
+  .substring(0, 32);
 
 const encrypt = (text) => {
   const cipher = crypto.createCipheriv(algorithm, secretKey, iv);
@@ -25,8 +25,8 @@ const decrypt = (data) => {
     throw err;
   }
   const decipher = crypto.createDecipheriv(algorithm, secretKey, Buffer.from(hash.iv, 'hex'));
-  const decrpyted = Buffer.concat([decipher.update(Buffer.from(hash.content, 'hex')), decipher.final()]);
-  return decrpyted.toString();
+  const decrypted = Buffer.concat([decipher.update(Buffer.from(hash.content, 'hex')), decipher.final()]);
+  return decrypted.toString();
 };
 
 module.exports = {

lib/utils/install-srf-locals.js

@@ -138,7 +138,6 @@ function installSrfLocals(srf, logger) {
     retrieveCall,
     listCalls,
     deleteCall,
-    synthAudio,
     createHash,
     retrieveHash,
     deleteKey,
@@ -151,11 +150,17 @@ function installSrfLocals(srf, logger) {
     pushBack,
     popFront,
     removeFromList,
-    lengthOfList,
     getListPosition,
+    lengthOfList,
+  } = require('@jambonz/realtimedb-helpers')({
+    host: process.env.JAMBONES_REDIS_HOST,
+    port: process.env.JAMBONES_REDIS_PORT || 6379
+  }, logger, tracer);
+  const {
+    synthAudio,
     getNuanceAccessToken,
     getIbmAccessToken,
-  } = require('@jambonz/realtimedb-helpers')({
+  } = require('@jambonz/speech-utils')({
     host: process.env.JAMBONES_REDIS_HOST,
     port: process.env.JAMBONES_REDIS_PORT || 6379
   }, logger, tracer);

lib/utils/siprec-utils.js

@@ -242,7 +242,8 @@ const createSipRecPayload = (sdp1, sdp2, logger) => {
     .replace(/a=sendonly\r\n/g, '')
     .replace(/a=direction:both\r\n/g, '');
   */
-  return combinedSdp;
+
+  return combinedSdp.replace(/sendrecv/g, 'recvonly');
 };
 
 module.exports = { parseSiprecPayload, createSipRecPayload } ;

lib/utils/transcription-utils.js

@@ -6,7 +6,8 @@ const {
   NuanceTranscriptionEvents,
   DeepgramTranscriptionEvents,
   SonioxTranscriptionEvents,
-  NvidiaTranscriptionEvents
+  NvidiaTranscriptionEvents,
+  JambonzTranscriptionEvents
 } = require('./constants');
 
 const stickyVars = {
@@ -28,6 +29,7 @@ const stickyVars = {
     'AZURE_SERVICE_ENDPOINT_ID',
     'AZURE_REQUEST_SNR',
     'AZURE_PROFANITY_OPTION',
+    'AZURE_SPEECH_ALTERNATIVE_LANGUAGE_CODES',
     'AZURE_SERVICE_ENDPOINT',
     'AZURE_INITIAL_SPEECH_TIMEOUT_MS',
     'AZURE_USE_OUTPUT_FORMAT_DETAILED',
@@ -223,6 +225,15 @@ const normalizeGoogle = (evt, channel, language) => {
   };
 };
 
+const normalizeCustom = (evt, channel, language) => {
+  return {
+    language_code: language,
+    channel_tag: channel,
+    is_final: evt.is_final,
+    alternatives: [evt.alternatives[0]]
+  };
+};
+
 const normalizeNuance = (evt, channel, language) => {
   const copy = JSON.parse(JSON.stringify(evt));
   return {
@@ -302,6 +313,9 @@ module.exports = (logger) => {
       case 'soniox':
         return normalizeSoniox(evt, channel, language);
       default:
+        if (vendor.startsWith('custom:')) {
+          return normalizeCustom(evt, channel, language);
+        }
         logger.error(`Unknown vendor ${vendor}`);
         return evt;
     }
@@ -311,6 +325,7 @@ module.exports = (logger) => {
     let opts = {};
     const {enable, voiceMs = 0, mode = -1} = rOpts.vad || {};
     const vad = {enable, voiceMs, mode};
+    const vendor = rOpts.vendor;
 
     /* voice activity detection works across vendors */
     opts = {
@@ -320,59 +335,43 @@ module.exports = (logger) => {
       ...(vad.enable && typeof vad.mode === 'number' && {RECOGNIZER_VAD_MODE: vad.mode}),
     };
 
-    if ('google' === rOpts.vendor) {
+    if ('google' === vendor) {
+      let model = 'phone_call';
+      if (rOpts.altLanguages.length > 0) model = task.name === TaskName.Gather ? 'command_and_search' : 'latest_long';
       opts = {
         ...opts,
-        ...(sttCredentials &&
-          {GOOGLE_APPLICATION_CREDENTIALS: JSON.stringify(sttCredentials.credentials)}),
-        ...(rOpts.enhancedModel &&
-            {GOOGLE_SPEECH_USE_ENHANCED: 1}),
-        ...(rOpts.separateRecognitionPerChannel &&
-          {GOOGLE_SPEECH_SEPARATE_RECOGNITION_PER_CHANNEL: 1}),
-        ...(rOpts.profanityFilter &&
-          {GOOGLE_SPEECH_PROFANITY_FILTER: 1}),
-        ...(rOpts.punctuation &&
-          {GOOGLE_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION: 1}),
-        ...(rOpts.words &&
-          {GOOGLE_SPEECH_ENABLE_WORD_TIME_OFFSETS: 1}),
-        ...((rOpts.singleUtterance ||  task.name === TaskName.Gather) &&
-          {GOOGLE_SPEECH_SINGLE_UTTERANCE: 1}),
-        ...(rOpts.diarization &&
-          {GOOGLE_SPEECH_SPEAKER_DIARIZATION: 1}),
+        ...(sttCredentials && {GOOGLE_APPLICATION_CREDENTIALS: JSON.stringify(sttCredentials.credentials)}),
+        ...(rOpts.separateRecognitionPerChannel && {GOOGLE_SPEECH_SEPARATE_RECOGNITION_PER_CHANNEL: 1}),
+        ...(rOpts.separateRecognitionPerChanne === false && {GOOGLE_SPEECH_SEPARATE_RECOGNITION_PER_CHANNEL: 0}),
+        ...(rOpts.profanityFilter && {GOOGLE_SPEECH_PROFANITY_FILTER: 1}),
+        ...(rOpts.punctuation && {GOOGLE_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION: 1}),
+        ...(rOpts.words && {GOOGLE_SPEECH_ENABLE_WORD_TIME_OFFSETS: 1}),
+        ...(rOpts.singleUtterance && {GOOGLE_SPEECH_SINGLE_UTTERANCE: 1}),
+        ...(rOpts.diarization && {GOOGLE_SPEECH_SPEAKER_DIARIZATION: 1}),
         ...(rOpts.diarization && rOpts.diarizationMinSpeakers > 0 &&
           {GOOGLE_SPEECH_SPEAKER_DIARIZATION_MIN_SPEAKER_COUNT: rOpts.diarizationMinSpeakers}),
         ...(rOpts.diarization && rOpts.diarizationMaxSpeakers > 0 &&
           {GOOGLE_SPEECH_SPEAKER_DIARIZATION_MAX_SPEAKER_COUNT: rOpts.diarizationMaxSpeakers}),
-        ...(rOpts.enhancedModel === false &&
-          {GOOGLE_SPEECH_USE_ENHANCED: 0}),
-        ...(rOpts.separateRecognitionPerChannel === false &&
-          {GOOGLE_SPEECH_SEPARATE_RECOGNITION_PER_CHANNEL: 0}),
-        ...(rOpts.profanityFilter === false &&
-          {GOOGLE_SPEECH_PROFANITY_FILTER: 0}),
-        ...(rOpts.punctuation === false &&
-          {GOOGLE_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION: 0}),
-        ...(rOpts.words  == false &&
-          {GOOGLE_SPEECH_ENABLE_WORD_TIME_OFFSETS: 0}),
-        ...((rOpts.singleUtterance === false || task.name === TaskName.Transcribe) &&
-          {GOOGLE_SPEECH_SINGLE_UTTERANCE: 0}),
-        ...(rOpts.diarization === false &&
-          {GOOGLE_SPEECH_SPEAKER_DIARIZATION: 0}),
+        ...(rOpts.enhancedModel && {GOOGLE_SPEECH_USE_ENHANCED: 1}),
+        ...(rOpts.profanityFilter === false && {GOOGLE_SPEECH_PROFANITY_FILTER: 0}),
+        ...(rOpts.punctuation === false && {GOOGLE_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION: 0}),
+        ...(rOpts.words  == false && {GOOGLE_SPEECH_ENABLE_WORD_TIME_OFFSETS: 0}),
+        ...(rOpts.diarization === false && {GOOGLE_SPEECH_SPEAKER_DIARIZATION: 0}),
         ...(rOpts.hints.length > 0 && typeof rOpts.hints[0] === 'string' &&
           {GOOGLE_SPEECH_HINTS: rOpts.hints.join(',')}),
         ...(rOpts.hints.length > 0 && typeof rOpts.hints[0] === 'object' &&
           {GOOGLE_SPEECH_HINTS: JSON.stringify(rOpts.hints)}),
-        ...(typeof rOpts.hintsBoost === 'number' &&
-          {GOOGLE_SPEECH_HINTS_BOOST: rOpts.hintsBoost}),
+        ...(typeof rOpts.hintsBoost === 'number' && {GOOGLE_SPEECH_HINTS_BOOST: rOpts.hintsBoost}),
         ...(rOpts.altLanguages.length > 0 &&
-          {GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES: rOpts.altLanguages.join(',')}),
+          {GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES: [...new Set(rOpts.altLanguages)].join(',')}),
         ...(rOpts.interactionType &&
           {GOOGLE_SPEECH_METADATA_INTERACTION_TYPE: rOpts.interactionType}),
-        ...{GOOGLE_SPEECH_MODEL: rOpts.model || (task.name === TaskName.Gather ? 'latest_short' : 'phone_call')},
-        ...(rOpts.naicsCode > 0 &&
-          {GOOGLE_SPEECH_METADATA_INDUSTRY_NAICS_CODE: rOpts.naicsCode}),
+        ...{GOOGLE_SPEECH_MODEL: rOpts.model || model},
+        ...(rOpts.naicsCode > 0 && {GOOGLE_SPEECH_METADATA_INDUSTRY_NAICS_CODE: rOpts.naicsCode}),
+        GOOGLE_SPEECH_METADATA_RECORDING_DEVICE_TYPE: 'phone_line',
       };
     }
-    else if (['aws', 'polly'].includes(rOpts.vendor)) {
+    else if (['aws', 'polly'].includes(vendor)) {
       opts = {
         ...opts,
         ...(rOpts.vocabularyName && {AWS_VOCABULARY_NAME: rOpts.vocabularyName}),
@@ -385,7 +384,7 @@ module.exports = (logger) => {
         }),
       };
     }
-    else if ('microsoft' === rOpts.vendor) {
+    else if ('microsoft' === vendor) {
       opts = {
         ...opts,
         ...(rOpts.hints.length > 0 && typeof rOpts.hints[0] === 'string' &&
@@ -393,7 +392,7 @@ module.exports = (logger) => {
         ...(rOpts.hints.length > 0 && typeof rOpts.hints[0] === 'object' &&
           {AZURE_SPEECH_HINTS: rOpts.hints.map((h) => h.phrase).join(',')}),
         ...(rOpts.altLanguages && rOpts.altLanguages.length > 0 &&
-          {AZURE_SERVICE_ENDPOINT_ID: rOpts.sttCredentials}),
+          {AZURE_SPEECH_ALTERNATIVE_LANGUAGE_CODES: [...new Set(rOpts.altLanguages)].join(',')}),
         ...(rOpts.requestSnr && {AZURE_REQUEST_SNR: 1}),
         ...(rOpts.profanityOption && {AZURE_PROFANITY_OPTION: rOpts.profanityOption}),
         ...(rOpts.azureServiceEndpoint && {AZURE_SERVICE_ENDPOINT: rOpts.azureServiceEndpoint}),
@@ -410,7 +409,7 @@ module.exports = (logger) => {
           {AZURE_SERVICE_ENDPOINT_ID: sttCredentials.custom_stt_endpoint})
       };
     }
-    else if ('nuance' === rOpts.vendor) {
+    else if ('nuance' === vendor) {
       /**
        * Note: all nuance options are in recognizer.nuanceOptions, should migrate
        * other vendor settings to similar nested structure
@@ -418,12 +417,9 @@ module.exports = (logger) => {
       const {nuanceOptions = {}} = rOpts;
       opts = {
         ...opts,
-        ...(sttCredentials.access_token) &&
-          {NUANCE_ACCESS_TOKEN: sttCredentials.access_token},
-        ...(sttCredentials.krypton_endpoint) &&
-          {NUANCE_KRYPTON_ENDPOINT: sttCredentials.krypton_endpoint},
-        ...(nuanceOptions.topic) &&
-          {NUANCE_TOPIC: nuanceOptions.topic},
+        ...(sttCredentials.access_token) && {NUANCE_ACCESS_TOKEN: sttCredentials.access_token},
+        ...(sttCredentials.nuance_stt_uri) && {NUANCE_KRYPTON_ENDPOINT: sttCredentials.nuance_stt_uri},
+        ...(nuanceOptions.topic) && {NUANCE_TOPIC: nuanceOptions.topic},
         ...(nuanceOptions.utteranceDetectionMode) &&
           {NUANCE_UTTERANCE_DETECTION_MODE: nuanceOptions.utteranceDetectionMode},
         ...(nuanceOptions.punctuation || rOpts.punctuation) && {NUANCE_PUNCTUATION: nuanceOptions.punctuation},
@@ -461,7 +457,7 @@ module.exports = (logger) => {
           {NUANCE_RESOURCES: JSON.stringify(nuanceOptions.resources)},
       };
     }
-    else if ('deepgram' === rOpts.vendor) {
+    else if ('deepgram' === vendor) {
       const {deepgramOptions = {}} = rOpts;
       opts = {
         ...opts,
@@ -505,7 +501,7 @@ module.exports = (logger) => {
           {DEEPGRAM_SPEECH_TAG: deepgramOptions.tag}
       };
     }
-    else if ('soniox' === rOpts.vendor) {
+    else if ('soniox' === vendor) {
       const {sonioxOptions = {}} = rOpts;
       const {storage = {}} = sonioxOptions;
       opts = {
@@ -528,7 +524,7 @@ module.exports = (logger) => {
         ...(storage?.id && storage?.disableSearch && {SONIOX_STORAGE_DISABLE_SEARCH: 1})
       };
     }
-    else if ('ibm' === rOpts.vendor) {
+    else if ('ibm' === vendor) {
       const {ibmOptions = {}} = rOpts;
       opts = {
         ...opts,
@@ -552,7 +548,7 @@ module.exports = (logger) => {
           {IBM_SPEECH_WATSON_LEARNING_OPT_OUT: ibmOptions.watsonLearningOptOut}
       };
     }
-    else if ('nvidia' === rOpts.vendor) {
+    else if ('nvidia' === vendor) {
       const {nvidiaOptions = {}} = rOpts;
       opts = {
         ...opts,
@@ -581,11 +577,29 @@ module.exports = (logger) => {
           {NVIDIA_CUSTOM_CONFIGURATION: JSON.stringify(nvidiaOptions.customConfiguration)}),
       };
     }
+    else if (vendor.startsWith('custom:')) {
+      let {options = {}} = rOpts;
+      const {auth_token, custom_stt_url} = sttCredentials;
+      options = {
+        ...options,
+        ...(rOpts.hints.length > 0 && typeof rOpts.hints[0] === 'string' &&
+        {hints: rOpts.hints}),
+        ...(rOpts.hints.length > 0 && typeof rOpts.hints[0] === 'object' &&
+        {hints: JSON.stringify(rOpts.hints)}),
+        ...(typeof rOpts.hintsBoost === 'number' && {hintsBoost: rOpts.hintsBoost})
+      };
 
-    stickyVars[rOpts.vendor].forEach((key) => {
+      opts = {
+        ...opts,
+        JAMBONZ_STT_API_KEY: auth_token,
+        JAMBONZ_STT_URL: custom_stt_url,
+        ...(Object.keys(options).length > 0 && {JAMBONZ_STT_OPTIONS: JSON.stringify(options)}),
+      };
+    }
+
+    (stickyVars[vendor] || []).forEach((key) => {
       if (!opts[key]) opts[key] = '';
     });
-    //logger.debug({opts}, 'recognizer channel vars');
     return opts;
   };
 
@@ -604,7 +618,6 @@ module.exports = (logger) => {
     ep.removeCustomEventListener(NuanceTranscriptionEvents.Transcription);
     ep.removeCustomEventListener(NuanceTranscriptionEvents.TranscriptionComplete);
     ep.removeCustomEventListener(NuanceTranscriptionEvents.StartOfSpeech);
-    ep.removeCustomEventListener(NuanceTranscriptionEvents.Error);
     ep.removeCustomEventListener(NuanceTranscriptionEvents.VadDetected);
 
     ep.removeCustomEventListener(DeepgramTranscriptionEvents.Transcription);
@@ -612,13 +625,17 @@ module.exports = (logger) => {
     ep.removeCustomEventListener(DeepgramTranscriptionEvents.ConnectFailure);
 
     ep.removeCustomEventListener(SonioxTranscriptionEvents.Transcription);
-    ep.removeCustomEventListener(SonioxTranscriptionEvents.Error);
 
     ep.removeCustomEventListener(NvidiaTranscriptionEvents.Transcription);
     ep.removeCustomEventListener(NvidiaTranscriptionEvents.TranscriptionComplete);
     ep.removeCustomEventListener(NvidiaTranscriptionEvents.StartOfSpeech);
-    ep.removeCustomEventListener(NvidiaTranscriptionEvents.Error);
     ep.removeCustomEventListener(NvidiaTranscriptionEvents.VadDetected);
+
+    ep.removeCustomEventListener(JambonzTranscriptionEvents.Transcription);
+    ep.removeCustomEventListener(JambonzTranscriptionEvents.Connect);
+    ep.removeCustomEventListener(JambonzTranscriptionEvents.ConnectFailure);
+
+    ep.removeCustomEventListener(JambonzTranscriptionEvents.Error);
   };
 
   const setSpeechCredentialsAtRuntime = (recognizer) => {
@@ -626,7 +643,7 @@ module.exports = (logger) => {
     if (recognizer.vendor === 'nuance') {
       const {clientId, secret, kryptonEndpoint} = recognizer.nuanceOptions || {};
       if (clientId && secret) return {client_id: clientId, secret};
-      if (kryptonEndpoint) return {krypton_endpoint: kryptonEndpoint};
+      if (kryptonEndpoint) return {nuance_stt_uri: kryptonEndpoint};
     }
     else if (recognizer.vendor === 'nvidia') {
       const {rivaUri} = recognizer.nvidiaOptions || {};

lib/utils/ws-requestor.js

@@ -219,7 +219,6 @@ class WsRequestor extends BaseRequestor {
   }
 
   _setHandlers(ws) {
-    this.logger.debug('WsRequestor:_setHandlers');
     ws
       .once('open', this._onOpen.bind(this, ws))
       .once('close', this._onClose.bind(this))
@@ -274,6 +273,7 @@ class WsRequestor extends BaseRequestor {
     }, 'WsRequestor - unexpected response');
     this.emit('connection-failure');
     this.emit('not-ready', new Error(`${res.statusCode} ${res.statusMessage}`));
+    this.connections++;
   }
 
   _onSocketClosed() {
@@ -338,7 +338,7 @@ class WsRequestor extends BaseRequestor {
       this.logger.info({url: this.url}, `WsRequestor:_recvAck - ack to unknown msgid ${msgid}, discarding`);
       return;
     }
-    this.logger.debug({url: this.url}, `WsRequestor:_recvAck - received response to ${msgid}`);
+    //this.logger.debug({url: this.url}, `WsRequestor:_recvAck - received response to ${msgid}`);
     this.messagesInFlight.delete(msgid);
     const {success} = obj;
     success && success(data);

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jambonz-feature-server",
-  "version": "v0.8.1",
+  "version": "v0.8.2",
   "main": "app.js",
   "engines": {
     "node": ">= 10.16.0"
@@ -19,15 +19,16 @@
   "bugs": {},
   "scripts": {
     "start": "node app",
-    "test": "NODE_ENV=test JAMBONES_HOSTING=1 HTTP_POOL=1 DRACHTIO_HOST=127.0.0.1 DRACHTIO_PORT=9060 DRACHTIO_SECRET=cymru JAMBONES_MYSQL_HOST=127.0.0.1 JAMBONES_MYSQL_PORT=3360 JAMBONES_MYSQL_USER=jambones_test JAMBONES_MYSQL_PASSWORD=jambones_test JAMBONES_MYSQL_DATABASE=jambones_test JAMBONES_REDIS_HOST=127.0.0.1 JAMBONES_REDIS_PORT=16379 JAMBONES_LOGLEVEL=error ENABLE_METRICS=0 HTTP_PORT=3000 JAMBONES_SBCS=172.38.0.10 JAMBONES_FREESWITCH=127.0.0.1:8022:JambonzR0ck$:docker-host JAMBONES_TIME_SERIES_HOST=127.0.0.1 JAMBONES_NETWORK_CIDR=172.38.0.0/16 node test/ ",
+    "test": "NODE_ENV=test JAMBONES_HOSTING=1 HTTP_POOL=1 ENCRYPTION_SECRET=foobar DRACHTIO_HOST=127.0.0.1 DRACHTIO_PORT=9060 DRACHTIO_SECRET=cymru JAMBONES_MYSQL_HOST=127.0.0.1 JAMBONES_MYSQL_PORT=3360 JAMBONES_MYSQL_USER=jambones_test JAMBONES_MYSQL_PASSWORD=jambones_test JAMBONES_MYSQL_DATABASE=jambones_test JAMBONES_REDIS_HOST=127.0.0.1 JAMBONES_REDIS_PORT=16379 JAMBONES_LOGLEVEL=error ENABLE_METRICS=0 HTTP_PORT=3000 JAMBONES_SBCS=172.38.0.10 JAMBONES_FREESWITCH=127.0.0.1:8022:JambonzR0ck$:docker-host JAMBONES_TIME_SERIES_HOST=127.0.0.1 JAMBONES_NETWORK_CIDR=172.38.0.0/16 node test/ ",
     "coverage": "./node_modules/.bin/nyc --reporter html --report-dir ./coverage npm run test",
     "jslint": "eslint app.js lib"
   },
   "dependencies": {
     "@jambonz/db-helpers": "^0.7.4",
     "@jambonz/http-health-check": "^0.0.1",
-    "@jambonz/realtimedb-helpers": "^0.6.5",
-    "@jambonz/stats-collector": "^0.1.6",
+    "@jambonz/realtimedb-helpers": "^0.7.0",
+    "@jambonz/speech-utils": "^0.0.12",
+    "@jambonz/stats-collector": "^0.1.8",
     "@jambonz/time-series": "^0.2.5",
     "@jambonz/verb-specifications": "^0.0.11",
     "@opentelemetry/api": "^1.4.0",
@@ -43,7 +44,7 @@
     "bent": "^7.3.12",
     "debug": "^4.3.4",
     "deepcopy": "^2.1.0",
-    "drachtio-fsmrf": "^3.0.19",
+    "drachtio-fsmrf": "^3.0.20",
     "drachtio-srf": "^4.5.23",
     "express": "^4.18.2",
     "ip": "^1.1.8",

test/create-test-db.js

@@ -18,30 +18,38 @@ test('creating schema', (t) => {
     if (err) return t.end(err);
     t.pass('schema and test data successfully created');
 
-    if (process.env.GCP_JSON_KEY && process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) {
+    const sql = [];
+    if (process.env.GCP_JSON_KEY) {
       const google_credential = encrypt(process.env.GCP_JSON_KEY);
+      t.pass('adding google credentials');
+      sql.push(`UPDATE speech_credentials SET credential='${google_credential}' WHERE vendor='google';`);
+    }
+    if (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY) {
       const aws_credential = encrypt(JSON.stringify({
         access_key_id: process.env.AWS_ACCESS_KEY_ID,
         secret_access_key: process.env.AWS_SECRET_ACCESS_KEY,
         aws_region: process.env.AWS_REGION
       }));
+      t.pass('adding aws credentials');
+      sql.push(`UPDATE speech_credentials SET credential='${aws_credential}' WHERE vendor='aws';`);
+    }
+    if (process.env.MICROSOFT_REGION && process.env.MICROSOFT_API_KEY) {
       const microsoft_credential = encrypt(JSON.stringify({
-        region: process.env.MICROSOFT_REGION || 'useast',
-        api_key: process.env.MICROSOFT_API_KEY || '1234567890'
+        region: process.env.MICROSOFT_REGION,
+        api_key: process.env.MICROSOFT_API_KEY
       }));
-      const cmd = `
-UPDATE speech_credentials SET credential='${google_credential}' WHERE vendor='google';
-UPDATE speech_credentials SET credential='${aws_credential}' WHERE vendor='aws';
-UPDATE speech_credentials SET credential='${microsoft_credential}' WHERE vendor='microsoft';
-`;
+      t.pass('adding microsoft credentials');
+      sql.push(`UPDATE speech_credentials SET credential='${microsoft_credential}' WHERE vendor='microsoft';`);
+    }
+    if (sql.length > 0) {
       const path = `${__dirname}/.creds.sql`;
-      fs.writeFileSync(path, cmd);
+      const cmd = sql.join('\n');
+      fs.writeFileSync(path, sql.join('\n'));
       exec(`mysql -h 127.0.0.1 -u root --protocol=tcp --port=3360  -D jambones_test < ${path}`, (err, stdout, stderr) => {
         console.log(stdout);
         console.log(stderr);
         if (err) return t.end(err);
         fs.unlinkSync(path)
-        fs.writeFileSync(`${__dirname}/credentials/gcp.json`, process.env.GCP_JSON_KEY);
         t.pass('set account-level speech credentials');
         t.end();
       });

test/remove-test-db.js

@@ -5,7 +5,6 @@ test('dropping jambones_test database', (t) => {
   exec(`mysql -h 127.0.0.1 -u root --protocol=tcp --port=3360 < ${__dirname}/db/remove_test_db.sql`, (err, stdout, stderr) => {
     if (err) return t.end(err);
     t.pass('database successfully dropped');
-    fs.unlinkSync(`${__dirname}/credentials/gcp.json`);
     t.end();
   });
 });