gather: fix bug where empty deepgram transcript saved incorrectly

* support mod_rimelabs_tts

* update speech utils

lib/session/call-session.js

@@ -702,7 +702,7 @@ class CallSession extends Emitter {
       task = await this.backgroundTaskManager.newTask('bargeIn', gather);
       task.sticky = autoEnable;
       // listen to the bargein-done from background manager
-      this.backgroundTaskManager.once('bargeIn-done', () => {
+      this.backgroundTaskManager.on('bargeIn-done', () => {
         if (this.requestor instanceof WsRequestor) {
           try {
             this.kill(true);
@@ -855,6 +855,19 @@ class CallSession extends Emitter {
             model_id: credential.model_id,
             options: credential.options
           };
+        } else if ('playht' === vendor) {
+          return {
+            api_key: credential.api_key,
+            user_id: credential.user_id,
+            voice_engine: credential.voice_engine,
+            options: credential.options
+          };
+        } else if ('rimelabs' === vendor) {
+          return {
+            api_key: credential.api_key,
+            model_id: credential.model_id,
+            options: credential.options
+          };
         } else if ('assemblyai' === vendor) {
           return {
             speech_credential_sid: credential.speech_credential_sid,

lib/tasks/gather.js

@@ -837,7 +837,7 @@ class TaskGather extends SttTask {
         if (this.bargein && (words + bufferedWords) < this.minBargeinWordCount) {
           this.logger.debug({evt, words, bufferedWords},
             'TaskGather:_onTranscription - final transcript but < min barge words');
-          this._bufferedTranscripts.push(evt);
+          if (!emptyTranscript) this._bufferedTranscripts.push(evt);
           if (!['soniox', 'aws', 'microsoft', 'deepgram'].includes(this.vendor)) this._startTranscribing(ep);
           return;
         }

lib/tasks/listen.js

@@ -8,6 +8,10 @@ const DTMF_SPAN_NAME = 'dtmf';
 class TaskListen extends Task {
   constructor(logger, opts, parentTask) {
     super(logger, opts);
+    /**
+     * @deprecated
+     * use bidirectionalAudio.enabled
+     */
     this.disableBidirectionalAudio = opts.disableBidirectionalAudio;
     this.preconditions = TaskPreconditions.Endpoint;
 
@@ -25,6 +29,15 @@ class TaskListen extends Task {
     this.results = {};
     this.playAudioQueue = [];
     this.isPlayingAudioFromQueue = false;
+    this.bidirectionalAudio = {
+      enabled: this.disableBidirectionalAudio === true ? false : true,
+      ...(this.data['bidirectionalAudio']),
+    };
+
+    // From drachtio-version 3.0.40, forkAudioStart will send empty bugname, metadata together with
+    // bidirectionalAudio params that cause old version of freeswitch missunderstand between bugname and
+    // bidirectionalAudio params
+    this._bugname = 'audio_fork';
 
     if (this.transcribe) this.transcribeTask = makeTask(logger, {'transcribe': opts.transcribe}, this);
   }
@@ -133,7 +146,8 @@ class TaskListen extends Task {
       mixType: this.mixType,
       sampling: this.sampleRate,
       ...(this._bugname && {bugname: this._bugname}),
-      metadata
+      metadata,
+      bidirectionalAudio: this.bidirectionalAudio || {}
     });
     this.recordStartTime = moment();
     if (this.maxLength) {
@@ -153,7 +167,7 @@ class TaskListen extends Task {
     }
 
     /* support bi-directional audio */
-    if (!this.disableBidirectionalAudio) {
+    if (this.bidirectionalAudio.enabled) {
       ep.addCustomEventListener(ListenEvents.PlayAudio, this._onPlayAudio.bind(this, ep));
     }
     ep.addCustomEventListener(ListenEvents.KillAudio, this._onKillAudio.bind(this, ep));

lib/utils/background-task-manager.js

@@ -26,7 +26,7 @@ class BackgroundTaskManager extends Emitter {
     return this.tasks.size;
   }
 
-  async newTask(type, opts) {
+  async newTask(type, opts, sticky = false) {
     this.logger.info({opts}, `initiating Background task ${type}`);
     if (this.tasks.has(type)) {
       this.logger.info(`Background task ${type} is running, skipped`);
@@ -52,6 +52,7 @@ class BackgroundTaskManager extends Emitter {
     if (task) {
       this.tasks.set(type, task);
     }
+    if (task && sticky) task.sticky = true;
     return task;
   }
 
@@ -116,7 +117,7 @@ class BackgroundTaskManager extends Emitter {
           this._taskCompleted('bargeIn', task);
           if (task.sticky && !this.cs.callGone && !this.cs._stopping) {
             this.logger.info('BackgroundTaskManager:_initBargeIn: restarting background bargeIn');
-            this.newTask('bargeIn', opts);
+            this.newTask('bargeIn', opts, true);
           }
           return;
         })

lib/utils/db-utils.js

@@ -94,6 +94,17 @@ const speechMapper = (cred) => {
       obj.api_key = o.api_key;
       obj.model_id = o.model_id;
       obj.options = o.options;
+    } else if ('playht' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = o.api_key;
+      obj.user_id = o.user_id;
+      obj.voice_engine = o.voice_engine;
+      obj.options = o.options;
+    } else if ('rimelabs' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = o.api_key;
+      obj.model_id = o.model_id;
+      obj.options = o.options;
     } else if ('assemblyai' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jambonz-feature-server",
-  "version": "0.8.6",
+  "version": "0.9.0",
   "main": "app.js",
   "engines": {
     "node": ">= 18.x"
@@ -25,57 +25,57 @@
     "jslint:fix": "eslint app.js tracer.js lib --fix"
   },
   "dependencies": {
-    "@aws-sdk/client-auto-scaling": "^3.360.0",
-    "@aws-sdk/client-sns": "^3.360.0",
+    "@aws-sdk/client-auto-scaling": "^3.549.0",
+    "@aws-sdk/client-sns": "^3.549.0",
     "@jambonz/db-helpers": "^0.9.3",
     "@jambonz/http-health-check": "^0.0.1",
-    "@jambonz/mw-registrar": "^0.2.4",
-    "@jambonz/realtimedb-helpers": "^0.8.7",
-    "@jambonz/speech-utils": "^0.0.47",
+    "@jambonz/mw-registrar": "^0.2.7",
+    "@jambonz/realtimedb-helpers": "^0.8.8",
+    "@jambonz/speech-utils": "^0.0.51",
     "@jambonz/stats-collector": "^0.1.9",
     "@jambonz/time-series": "^0.2.8",
     "@jambonz/verb-specifications": "^0.0.69",
-    "@opentelemetry/api": "^1.4.0",
-    "@opentelemetry/exporter-jaeger": "^1.9.0",
-    "@opentelemetry/exporter-trace-otlp-http": "^0.35.0",
-    "@opentelemetry/exporter-zipkin": "^1.9.0",
-    "@opentelemetry/instrumentation": "^0.35.0",
-    "@opentelemetry/resources": "^1.9.0",
-    "@opentelemetry/sdk-trace-base": "^1.9.0",
-    "@opentelemetry/sdk-trace-node": "^1.9.0",
-    "@opentelemetry/semantic-conventions": "^1.9.0",
+    "@opentelemetry/api": "^1.8.0",
+    "@opentelemetry/exporter-jaeger": "^1.23.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
+    "@opentelemetry/exporter-zipkin": "^1.23.0",
+    "@opentelemetry/instrumentation": "^0.50.0",
+    "@opentelemetry/resources": "^1.23.0",
+    "@opentelemetry/sdk-trace-base": "^1.23.0",
+    "@opentelemetry/sdk-trace-node": "^1.23.0",
+    "@opentelemetry/semantic-conventions": "^1.23.0",
     "bent": "^7.3.12",
     "debug": "^4.3.4",
     "deepcopy": "^2.1.0",
-    "drachtio-fsmrf": "^3.0.39",
+    "drachtio-fsmrf": "^3.0.40",
     "drachtio-srf": "^4.5.31",
-    "express": "^4.18.2",
+    "express": "^4.19.2",
     "express-validator": "^7.0.1",
-    "ip": "^1.1.9",
-    "moment": "^2.29.4",
-    "parse-url": "^8.1.0",
-    "pino": "^8.8.0",
+    "ip": "^2.0.1",
+    "moment": "^2.30.1",
+    "parse-url": "^9.2.0",
+    "pino": "^8.20.0",
     "polly-ssml-split": "^0.1.0",
     "proxyquire": "^2.1.3",
-    "sdp-transform": "^2.14.1",
+    "sdp-transform": "^2.14.2",
     "short-uuid": "^4.2.2",
-    "sinon": "^15.0.1",
+    "sinon": "^17.0.1",
     "to-snake-case": "^1.0.0",
-    "undici": "^5.28.3",
+    "undici": "^6.11.1",
     "uuid-random": "^1.3.2",
     "verify-aws-sns-signature": "^0.1.0",
-    "ws": "^8.9.0",
+    "ws": "^8.16.0",
     "xml2js": "^0.6.2"
   },
   "devDependencies": {
     "clear-module": "^4.1.2",
-    "eslint": "^7.32.0",
-    "eslint-plugin-promise": "^4.3.1",
+    "eslint": "7.32.0",
+    "eslint-plugin-promise": "^6.1.1",
     "nyc": "^15.1.0",
-    "tape": "^5.6.1"
+    "tape": "^5.7.5"
   },
   "optionalDependencies": {
-    "bufferutil": "^4.0.6",
-    "utf-8-validate": "^5.0.8"
+    "bufferutil": "^4.0.8",
+    "utf-8-validate": "^6.0.3"
   }
 }

test/docker-compose-testbed.yaml

@@ -57,7 +57,7 @@ services:
         condition: service_healthy
 
   freeswitch:
-    image: drachtio/drachtio-freeswitch-mrf:0.6.2
+    image: drachtio/drachtio-freeswitch-mrf:0.7.3
     restart: always
     command: freeswitch --rtp-range-start 20000 --rtp-range-end 20100
     environment: