add recognizer.azureServiceEndpoint for custom azure voices

Co-authored-by: Guilherme Rauen <g.rauen@cognigy.com>

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 node:18.9.0-alpine3.16 as base
+FROM --platform=linux/amd64 node:18.12.1-alpine3.16 as base
 
 RUN apk --update --no-cache add --virtual .builds-deps build-base python3
 

lib/session/call-session.js

@@ -551,7 +551,9 @@ class CallSession extends Emitter {
             api_key: credential.api_key,
             region: credential.region,
             use_custom_stt: credential.use_custom_stt,
-            custom_stt_endpoint: credential.custom_stt_endpoint
+            custom_stt_endpoint: credential.custom_stt_endpoint,
+            use_custom_tts: credential.use_custom_tts,
+            custom_tts_endpoint: credential.custom_tts_endpoint
           };
         }
         else if ('wellsaid' === vendor) {

lib/tasks/config.js

@@ -146,7 +146,11 @@ class TaskConfig extends Task {
   _onAmdEvent(cs, evt) {
     this.logger.info({evt}, 'Config:_onAmdEvent');
     const {actionHook} = this.data.amd;
-    this.performHook(cs, actionHook, evt);
+    this.performHook(cs, actionHook, evt)
+      .catch((err) => {
+        this.logger.error({err}, 'Config:_onAmdEvent - error calling actionHook');
+      });
+
   }
 }
 

lib/tasks/dial.js

@@ -689,7 +689,10 @@ class TaskDial extends Task {
   _onAmdEvent(cs, evt) {
     this.logger.info({evt}, 'Dial:_onAmdEvent');
     const {actionHook} = this.data.amd;
-    this.performHook(cs, actionHook, evt);
+    this.performHook(cs, actionHook, evt)
+      .catch((err) => {
+        this.logger.error({err}, 'Dial:_onAmdEvent - error calling actionHook');
+      });
   }
 }
 

lib/tasks/gather.js

@@ -143,6 +143,14 @@ class TaskGather extends Task {
       this.logger.debug({hints: this.hints, hintsBoost: this.hintsBoost},
         'Gather:exec - applying global sttHints');
     }
+    if (process.env.JAMBONZ_GATHER_EARLY_HINTS_MATCH &&
+      !this.isContinuousAsr &&
+      this.hints.length > 0 && this.hints.length <= 10) {
+      this.earlyHintsMatch = true;
+      this.interim = true;
+      this.logger.debug('Gather:exec - early hints match enabled');
+    }
+
     if (cs.hasAltLanguages) {
       this.altLanguages = this.altLanguages.concat(cs.altLanguages);
       this.logger.debug({altLanguages: this.altLanguages},
@@ -526,6 +534,15 @@ class TaskGather extends Task {
       }
     }
 
+    if (this.earlyHintsMatch && evt.is_final === false) {
+      const transcript = evt.alternatives[0].transcript?.toLowerCase();
+      if (this.hints.find((h) => h.toLowerCase() === transcript)) {
+        this.logger.debug({evt}, 'Gather:_onTranscription: early hint match');
+        this._resolve('speech', evt);
+        return;
+      }
+    }
+
     /* count words for bargein feature */
     const words = evt.alternatives[0].transcript.split(' ').length;
     const bufferedWords = this._bufferedTranscripts.reduce((count, e) => {

lib/tasks/say.js

@@ -164,6 +164,10 @@ class TaskSay extends Task {
           'tts.voice': voice
         });
         try {
+          if (vendor === 'microsoft' && this.synthesizer.azureServiceEndpoint) {
+            credentials.use_custom_tts = true;
+            credentials.custom_tts_endpoint = this.synthesizer.azureServiceEndpoint;
+          }
           const {filePath, servedFromCache} = await synthAudio(stats, {
             text,
             vendor,

lib/tasks/specs.json

@@ -435,7 +435,8 @@
       "gender": {
         "type": "string",
         "enum": ["MALE", "FEMALE", "NEUTRAL"]
-      }
+      },
+      "azureServiceEndpoint": "string"
     },
     "required": [
       "vendor"

lib/utils/amd-utils.js

@@ -273,26 +273,46 @@ module.exports = (logger) => {
     amd
       .on(AmdEvents.NoSpeechDetected, (evt) => {
         task.emit('amd', {type: AmdEvents.NoSpeechDetected, ...evt});
-        ep.stopTranscription({vendor, bugname});
+        try {
+          ep.connected && ep.stopTranscription({vendor, bugname});
+        } catch (err) {
+          logger.info({err}, 'Error stopping transcription');
+        }
       })
       .on(AmdEvents.HumanDetected, (evt) => {
         task.emit('amd', {type: AmdEvents.HumanDetected, ...evt});
-        ep.stopTranscription({vendor, bugname});
+        try {
+          ep.connected && ep.stopTranscription({vendor, bugname});
+        } catch (err) {
+          logger.info({err}, 'Error stopping transcription');
+        }
       })
       .on(AmdEvents.MachineDetected, (evt) => {
         task.emit('amd', {type: AmdEvents.MachineDetected, ...evt});
       })
       .on(AmdEvents.DecisionTimeout, (evt) => {
         task.emit('amd', {type: AmdEvents.DecisionTimeout, ...evt});
-        ep.stopTranscription({vendor, bugname});
+        try {
+          ep.connected && ep.stopTranscription({vendor, bugname});
+        } catch (err) {
+          logger.info({err}, 'Error stopping transcription');
+        }
       })
       .on(AmdEvents.ToneTimeout, (evt) => {
         //task.emit('amd', {type: AmdEvents.ToneTimeout, ...evt});
-        ep.execute('avmd_stop').catch((err) => logger.info(err, 'Error stopping avmd'));
+        try {
+          ep.connected && ep.execute('avmd_stop').catch((err) => logger.info(err, 'Error stopping avmd'));
+        } catch (err) {
+          logger.info({err}, 'Error stopping avmd');
+        }
       })
       .on(AmdEvents.MachineStoppedSpeaking, () => {
         task.emit('amd', {type: AmdEvents.MachineStoppedSpeaking});
-        ep.stopTranscription({vendor, bugname});
+        try {
+          ep.connected && ep.stopTranscription({vendor, bugname});
+        } catch (err) {
+          logger.info({err}, 'Error stopping transcription');
+        }
       });
 
     /* start transcribing, and also listening for beep */

lib/utils/db-utils.js

@@ -39,6 +39,8 @@ const speechMapper = (cred) => {
       obj.region = o.region;
       obj.use_custom_stt = o.use_custom_stt;
       obj.custom_stt_endpoint = o.custom_stt_endpoint;
+      obj.use_custom_tts = o.use_custom_tts;
+      obj.custom_tts_endpoint = o.custom_tts_endpoint;
     }
     else if ('wellsaid' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));

lib/utils/http-listener.js

@@ -21,6 +21,7 @@ const handleErrors = (logger, app, resolve, reject, e) => {
     server.on('error', handleErrors.bind(null, logger, app, resolve, reject));
     return;
   }
+  logger.info({err: e, port: PORT}, 'httpListener error');
   reject(e);
 };
 
@@ -30,7 +31,7 @@ const createHttpListener = (logger, srf) => {
   app.use(express.urlencoded({ extended: true }));
   app.use(express.json());
   app.use('/', httpRoutes);
-  app.use((err, req, res, next) => {
+  app.use((err, _req, res, _next) => {
     logger.error(err, 'burped error');
     res.status(err.status || 500).json({msg: err.message});
   });

package.json

@@ -25,8 +25,8 @@
   },
   "dependencies": {
     "@jambonz/http-health-check": "^0.0.1",
-    "@jambonz/db-helpers": "^0.7.2",
-    "@jambonz/realtimedb-helpers": "^0.4.35",
+    "@jambonz/db-helpers": "^0.7.3",
+    "@jambonz/realtimedb-helpers": "^0.5.9",
     "@jambonz/stats-collector": "^0.1.6",
     "@jambonz/time-series": "^0.2.5",
     "@opentelemetry/api": "^1.1.0",