bugfix #192: config with dtmf only followed later by gather with speech not working

Co-authored-by: Guilherme Rauen <g.rauen@cognigy.com>

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 node:18.9.0-alpine3.16 as base
+FROM --platform=linux/amd64 node:18.12.1-alpine3.16 as base
 
 RUN apk --update --no-cache add --virtual .builds-deps build-base python3
 

lib/http-routes/api/messaging.js

@@ -34,6 +34,7 @@ router.post('/:partner', async(req, res) => {
     carrier: req.params.partner,
     messageSid: app.messageSid,
     accountSid: app.accountSid,
+    serviceProviderSid: account.service_provider_sid,
     applicationSid: app.applicationSid,
     from: req.body.from,
     to: req.body.to,

lib/middleware.js

@@ -118,6 +118,7 @@ module.exports = function(srf, logger) {
     const {span} = rootSpan.startChildSpan('lookupAccountDetails');
     try {
       req.locals.accountInfo = await lookupAccountDetails(account_sid);
+      req.locals.service_provider_sid = req.locals.accountInfo?.account?.service_provider_sid;
       span.end();
       if (!req.locals.accountInfo.account.is_active) {
         logger.info(`Account is inactive or suspended ${account_sid}`);
@@ -273,7 +274,9 @@ module.exports = function(srf, logger) {
       }
       /* retrieve the application to execute for this inbound call */
       const params = Object.assign(['POST', 'WS'].includes(app.call_hook.method) ? {sip: req.msg} : {},
-        req.locals.callInfo, {
+        req.locals.callInfo,
+        {service_provider_sid: req.locals.service_provider_sid},
+        {
           defaults: {
             synthesizer: {
               vendor: app.speech_synthesis_vendor,

lib/session/call-session.js

@@ -551,7 +551,9 @@ class CallSession extends Emitter {
             api_key: credential.api_key,
             region: credential.region,
             use_custom_stt: credential.use_custom_stt,
-            custom_stt_endpoint: credential.custom_stt_endpoint
+            custom_stt_endpoint: credential.custom_stt_endpoint,
+            use_custom_tts: credential.use_custom_tts,
+            custom_tts_endpoint: credential.custom_tts_endpoint
           };
         }
         else if ('wellsaid' === vendor) {
@@ -586,13 +588,19 @@ class CallSession extends Emitter {
       this.logger.info(`CallSession:exec starting task #${stackNum}:${taskNum}: ${task.name}`);
       try {
         const resources = await this._evaluatePreconditions(task);
+        let skip = false;
         this.currentTask = task;
         if (TaskName.Gather === task.name && this.isBotModeEnabled) {
-          const timeout = task.timeout;
-          this.logger.info(`CallSession:exec skipping #${stackNum}:${taskNum}: ${task.name}`);
-          this.backgroundGatherTask.updateTimeout(timeout);
+          if (this.backgroundGatherTask.updateTaskInProgress(task)) {
+            this.logger.info(`CallSession:exec skipping #${stackNum}:${taskNum}: ${task.name}`);
+            skip = true;
+          }
+          else {
+            this.logger.info('CallSession:exec disabling bot mode to start gather with new options');
+            this.disableBotMode();
+          }
         }
-        else {
+        if (!skip) {
           const {span, ctx} = this.rootSpan.startChildSpan(`verb:${task.summary}`);
           task.span = span;
           task.ctx = ctx;

lib/tasks/config.js

@@ -146,7 +146,11 @@ class TaskConfig extends Task {
   _onAmdEvent(cs, evt) {
     this.logger.info({evt}, 'Config:_onAmdEvent');
     const {actionHook} = this.data.amd;
-    this.performHook(cs, actionHook, evt);
+    this.performHook(cs, actionHook, evt)
+      .catch((err) => {
+        this.logger.error({err}, 'Config:_onAmdEvent - error calling actionHook');
+      });
+
   }
 }
 

lib/tasks/dial.js

@@ -689,7 +689,10 @@ class TaskDial extends Task {
   _onAmdEvent(cs, evt) {
     this.logger.info({evt}, 'Dial:_onAmdEvent');
     const {actionHook} = this.data.amd;
-    this.performHook(cs, actionHook, evt);
+    this.performHook(cs, actionHook, evt)
+      .catch((err) => {
+        this.logger.error({err}, 'Dial:_onAmdEvent - error calling actionHook');
+      });
   }
 }
 

lib/tasks/gather.js

@@ -143,6 +143,14 @@ class TaskGather extends Task {
       this.logger.debug({hints: this.hints, hintsBoost: this.hintsBoost},
         'Gather:exec - applying global sttHints');
     }
+    if (process.env.JAMBONZ_GATHER_EARLY_HINTS_MATCH &&
+      !this.isContinuousAsr &&
+      this.hints.length > 0 && this.hints.length <= 10) {
+      this.earlyHintsMatch = true;
+      this.interim = true;
+      this.logger.debug('Gather:exec - early hints match enabled');
+    }
+
     if (cs.hasAltLanguages) {
       this.altLanguages = this.altLanguages.concat(cs.altLanguages);
       this.logger.debug({altLanguages: this.altLanguages},
@@ -178,7 +186,8 @@ class TaskGather extends Task {
 
     const startListening = (cs, ep) => {
       this._startTimer();
-      if (this.isContinuousAsr && 0 === this.timeout) this._startAsrTimer();
+      // dont start asr timer until we have a transcription
+      //if (this.isContinuousAsr && 0 === this.timeout) this._startAsrTimer();
       if (this.input.includes('speech') && !this.listenDuringPrompt) {
         this._initSpeech(cs, ep)
           .then(() => {
@@ -251,8 +260,12 @@ class TaskGather extends Task {
     this._resolve('killed');
   }
 
-  updateTimeout(timeout) {
-    this.logger.info(`TaskGather:updateTimeout - updating timeout to ${timeout}`);
+  updateTaskInProgress(opts) {
+    if (!this.needsStt && opts.input.includes('speech')) {
+      this.logger.info('TaskGather:updateTaskInProgress - adding speech to a background gather');
+      return false; // this needs be handled by killing the background gather and starting a new one
+    }
+    const {timeout} = opts;
     this.timeout = timeout;
     this._startTimer();
   }
@@ -526,6 +539,15 @@ class TaskGather extends Task {
       }
     }
 
+    if (this.earlyHintsMatch && evt.is_final === false) {
+      const transcript = evt.alternatives[0].transcript?.toLowerCase();
+      if (this.hints.find((h) => h.toLowerCase() === transcript)) {
+        this.logger.debug({evt}, 'Gather:_onTranscription: early hint match');
+        this._resolve('speech', evt);
+        return;
+      }
+    }
+
     /* count words for bargein feature */
     const words = evt.alternatives[0].transcript.split(' ').length;
     const bufferedWords = this._bufferedTranscripts.reduce((count, e) => {

lib/tasks/say.js

@@ -164,6 +164,10 @@ class TaskSay extends Task {
           'tts.voice': voice
         });
         try {
+          if (vendor === 'microsoft' && this.synthesizer.azureServiceEndpoint) {
+            credentials.use_custom_tts = true;
+            credentials.custom_tts_endpoint = this.synthesizer.azureServiceEndpoint;
+          }
           const {filePath, servedFromCache} = await synthAudio(stats, {
             text,
             vendor,

lib/tasks/sip_refer.js

@@ -36,6 +36,7 @@ class TaskSipRefer extends Task {
         method: 'REFER',
         headers: {
           ...this.headers,
+          ...(this.referToIsUri && {'X-Refer-To-Leave-Untouched': true}),
           'Refer-To': referTo,
           'Referred-By': referredBy
         }
@@ -100,6 +101,7 @@ class TaskSipRefer extends Task {
       /* they may have only provided a phone number/user */
       referTo = `sip:${referTo}@${host}`;
     }
+    else this.referToIsUri = true;
     if (!referredBy) {
       /* default */
       referredBy = cs.req?.callingNumber || dlg.local.uri;

lib/tasks/specs.json

@@ -435,7 +435,8 @@
       "gender": {
         "type": "string",
         "enum": ["MALE", "FEMALE", "NEUTRAL"]
-      }
+      },
+      "azureServiceEndpoint": "string"
     },
     "required": [
       "vendor"

lib/utils/amd-utils.js

@@ -273,26 +273,46 @@ module.exports = (logger) => {
     amd
       .on(AmdEvents.NoSpeechDetected, (evt) => {
         task.emit('amd', {type: AmdEvents.NoSpeechDetected, ...evt});
-        ep.stopTranscription({vendor, bugname});
+        try {
+          ep.connected && ep.stopTranscription({vendor, bugname});
+        } catch (err) {
+          logger.info({err}, 'Error stopping transcription');
+        }
       })
       .on(AmdEvents.HumanDetected, (evt) => {
         task.emit('amd', {type: AmdEvents.HumanDetected, ...evt});
-        ep.stopTranscription({vendor, bugname});
+        try {
+          ep.connected && ep.stopTranscription({vendor, bugname});
+        } catch (err) {
+          logger.info({err}, 'Error stopping transcription');
+        }
       })
       .on(AmdEvents.MachineDetected, (evt) => {
         task.emit('amd', {type: AmdEvents.MachineDetected, ...evt});
       })
       .on(AmdEvents.DecisionTimeout, (evt) => {
         task.emit('amd', {type: AmdEvents.DecisionTimeout, ...evt});
-        ep.stopTranscription({vendor, bugname});
+        try {
+          ep.connected && ep.stopTranscription({vendor, bugname});
+        } catch (err) {
+          logger.info({err}, 'Error stopping transcription');
+        }
       })
       .on(AmdEvents.ToneTimeout, (evt) => {
         //task.emit('amd', {type: AmdEvents.ToneTimeout, ...evt});
-        ep.execute('avmd_stop').catch((err) => logger.info(err, 'Error stopping avmd'));
+        try {
+          ep.connected && ep.execute('avmd_stop').catch((err) => logger.info(err, 'Error stopping avmd'));
+        } catch (err) {
+          logger.info({err}, 'Error stopping avmd');
+        }
       })
       .on(AmdEvents.MachineStoppedSpeaking, () => {
         task.emit('amd', {type: AmdEvents.MachineStoppedSpeaking});
-        ep.stopTranscription({vendor, bugname});
+        try {
+          ep.connected && ep.stopTranscription({vendor, bugname});
+        } catch (err) {
+          logger.info({err}, 'Error stopping transcription');
+        }
       });
 
     /* start transcribing, and also listening for beep */

lib/utils/db-utils.js

@@ -39,6 +39,8 @@ const speechMapper = (cred) => {
       obj.region = o.region;
       obj.use_custom_stt = o.use_custom_stt;
       obj.custom_stt_endpoint = o.custom_stt_endpoint;
+      obj.use_custom_tts = o.use_custom_tts;
+      obj.custom_tts_endpoint = o.custom_tts_endpoint;
     }
     else if ('wellsaid' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));

lib/utils/http-listener.js

@@ -21,6 +21,7 @@ const handleErrors = (logger, app, resolve, reject, e) => {
     server.on('error', handleErrors.bind(null, logger, app, resolve, reject));
     return;
   }
+  logger.info({err: e, port: PORT}, 'httpListener error');
   reject(e);
 };
 
@@ -30,7 +31,7 @@ const createHttpListener = (logger, srf) => {
   app.use(express.urlencoded({ extended: true }));
   app.use(express.json());
   app.use('/', httpRoutes);
-  app.use((err, req, res, next) => {
+  app.use((err, _req, res, _next) => {
     logger.error(err, 'burped error');
     res.status(err.status || 500).json({msg: err.message});
   });

package.json

@@ -26,7 +26,7 @@
   "dependencies": {
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/db-helpers": "^0.7.3",
-    "@jambonz/realtimedb-helpers": "^0.4.35",
+    "@jambonz/realtimedb-helpers": "^0.5.9",
     "@jambonz/stats-collector": "^0.1.6",
     "@jambonz/time-series": "^0.2.5",
     "@opentelemetry/api": "^1.1.0",