wip

support google tts mod
ignore google errors with error_code 0
2026-01-25 02:07:56 +00:00 · 2024-04-17 23:11:03 +07:00 · 2024-04-17 22:09:55 +07:00 · 2024-04-16 20:06:26 -04:00 · 2024-04-15 16:25:12 -04:00 · 2024-04-15 15:38:10 -04:00
10 changed files with 3691 additions and 5969 deletions
--- a/lib/session/call-session.js
+++ b/lib/session/call-session.js
@@ -702,7 +702,7 @@ class CallSession extends Emitter {
      task = await this.backgroundTaskManager.newTask('bargeIn', gather);
      task.sticky = autoEnable;
      // listen to the bargein-done from background manager
-      this.backgroundTaskManager.once('bargeIn-done', () => {
+      this.backgroundTaskManager.on('bargeIn-done', () => {
        if (this.requestor instanceof WsRequestor) {
          try {
            this.kill(true);
@@ -855,6 +855,19 @@ class CallSession extends Emitter {
            model_id: credential.model_id,
            options: credential.options
          };
+        } else if ('playht' === vendor) {
+          return {
+            api_key: credential.api_key,
+            user_id: credential.user_id,
+            voice_engine: credential.voice_engine,
+            options: credential.options
+          };
+        } else if ('rimelabs' === vendor) {
+          return {
+            api_key: credential.api_key,
+            model_id: credential.model_id,
+            options: credential.options
+          };
        } else if ('assemblyai' === vendor) {
          return {
            speech_credential_sid: credential.speech_credential_sid,
@@ -1585,7 +1598,23 @@ Duration=${duration} `
  }

  _preCacheAudio(newTasks) {
-    for (const task of newTasks) {
+    /**
+     * only precache audio for the a queued say if we have one or more non-Config verbs
+     * ahead of it in the queue.  This is because the Config verb returns immediately
+     * and would not give us enough time to generate the audio.  The point of precaching
+     * is to take advantage of getting the audio in advance of being needed, so we need
+     * to be confident we have some time before the say verb is executed, and the Config
+     * does not give us that confidence since it returns immediately.
+     */
+    const haveQueuedNonConfig = this.tasks.findIndex((t) => t.name !== TaskName.Config) !== -1;
+    let tasks = haveQueuedNonConfig ? newTasks : [];
+    if (!haveQueuedNonConfig) {
+      const idxFirstNotConfig = newTasks.findIndex((t) => t.name !== TaskName.Config);
+      if (-1 === idxFirstNotConfig) return;
+      tasks = newTasks.slice(idxFirstNotConfig + 1);
+    }
+
+    for (const task of tasks) {
      if (task.name === TaskName.Config && task.hasSynthesizer) {
        /* if they change synthesizer settings don't try to precache */
        break;
--- a/lib/tasks/gather.js
+++ b/lib/tasks/gather.js
@@ -837,7 +837,7 @@ class TaskGather extends SttTask {
        if (this.bargein && (words + bufferedWords) < this.minBargeinWordCount) {
          this.logger.debug({evt, words, bufferedWords},
            'TaskGather:_onTranscription - final transcript but < min barge words');
-          this._bufferedTranscripts.push(evt);
+          if (!emptyTranscript) this._bufferedTranscripts.push(evt);
          if (!['soniox', 'aws', 'microsoft', 'deepgram'].includes(this.vendor)) this._startTranscribing(ep);
          return;
        }
@@ -933,6 +933,10 @@ class TaskGather extends SttTask {
    this.logger.debug('TaskGather:_onTranscriptionComplete');
  }
  async _onJambonzError(cs, ep, evt) {
+    if (this.vendor === 'google' && evt.error_code === 0) {
+      this.logger.info({evt}, 'TaskTranscribe:_onJambonzError - ignoring google error code 0');
+      return;
+    }
    this.logger.info({evt}, 'TaskGather:_onJambonzError');
    if (this.isHandledByPrimaryProvider && this.fallbackVendor) {
      ep.stopTranscription({
--- a/lib/tasks/say.js
+++ b/lib/tasks/say.js
@@ -187,6 +187,9 @@ class TaskSay extends TtsTask {
      if (key.startsWith('variable_tts_')) {
        let newKey = key.substring('variable_tts_'.length)
          .replace('whisper_', 'whisper.')
+          .replace('deepgram_', 'deepgram.')
+          .replace('playht_', 'playht.')
+          .replace('rimelabs_', 'rimelabs.')
          .replace('elevenlabs_', 'elevenlabs.');
        if (spanMapping[newKey]) newKey = spanMapping[newKey];
        attrs[newKey] = value;
@@ -198,6 +201,9 @@ class TaskSay extends TtsTask {
 }

 const spanMapping = {
+  // IMPORTANT!!! JAMBONZ WEBAPP WILL SHOW TEXT PERFECTLY IF THE SPAN NAME IS SMALLER OR EQUAL 25 CHARACTERS.
+  // EX: whisper.ratelim_reqs has length 20 <= 25 which is perfect
+  // Elevenlabs
  'elevenlabs.reported_latency_ms': 'elevenlabs.latency_ms',
  'elevenlabs.request_id': 'elevenlabs.req_id',
  'elevenlabs.history_item_id': 'elevenlabs.item_id',
@@ -205,11 +211,33 @@ const spanMapping = {
  'elevenlabs.name_lookup_time_ms': 'name_lookup_ms',
  'elevenlabs.connect_time_ms': 'connect_ms',
  'elevenlabs.final_response_time_ms': 'final_response_ms',
+  // Whisper
  'whisper.reported_latency_ms': 'whisper.latency_ms',
  'whisper.request_id': 'whisper.req_id',
+  'whisper.reported_organization': 'whisper.organization',
+  'whisper.reported_ratelimit_requests': 'whisper.ratelimit',
+  'whisper.reported_ratelimit_remaining_requests': 'whisper.ratelimit_remain',
+  'whisper.reported_ratelimit_reset_requests': 'whisper.ratelimit_reset',
  'whisper.name_lookup_time_ms': 'name_lookup_ms',
  'whisper.connect_time_ms': 'connect_ms',
  'whisper.final_response_time_ms': 'final_response_ms',
+  // Deepgram
+  'deepgram.request_id': 'deepgram.req_id',
+  'deepgram.reported_model_name': 'deepgram.model_name',
+  'deepgram.reported_model_uuid': 'deepgram.model_uuid',
+  'deepgram.reported_char_count': 'deepgram.char_count',
+  'deepgram.name_lookup_time_ms': 'name_lookup_ms',
+  'deepgram.connect_time_ms': 'connect_ms',
+  'deepgram.final_response_time_ms': 'final_response_ms',
+  // Playht
+  'playht.request_id': 'playht.req_id',
+  'playht.name_lookup_time_ms': 'name_lookup_ms',
+  'playht.connect_time_ms': 'connect_ms',
+  'playht.final_response_time_ms': 'final_response_ms',
+  // Rimelabs
+  'rimelabs.name_lookup_time_ms': 'name_lookup_ms',
+  'rimelabs.connect_time_ms': 'connect_ms',
+  'rimelabs.final_response_time_ms': 'final_response_ms',
 };

 module.exports = TaskSay;
--- a/lib/tasks/transcribe.js
+++ b/lib/tasks/transcribe.js
@@ -535,6 +535,10 @@ class TaskTranscribe extends SttTask {
  }

  async _onJambonzError(cs, _ep, evt) {
+    if (this.vendor === 'google' && evt.error_code === 0) {
+      this.logger.info({evt}, 'TaskTranscribe:_onJambonzError - ignoring google error code 0');
+      return;
+    }
    this.logger.info({evt}, 'TaskTranscribe:_onJambonzError');
    if (this.paused) return;
    if (this.isHandledByPrimaryProvider && this.fallbackVendor) {
--- a/lib/tasks/tts-task.js
+++ b/lib/tasks/tts-task.js
@@ -68,6 +68,9 @@ class TtsTask extends Task {
      tts_engine: vendor,
      tts_voice: voice,
      cache_speech_handles: 1,
+      ...(vendor === 'google' && {
+        GOOGLE_TTS_APPLICATION_CREDENTIALS: JSON.stringify(credentials.credentials)
+      })
    }).catch((err) => this.logger.info({err}, `${this.name}: Error setting tts_engine on endpoint`));

    if (!preCache) this.logger.info({vendor, language, voice, model}, `${this.name}:exec`);
--- a/lib/utils/background-task-manager.js
+++ b/lib/utils/background-task-manager.js
@@ -26,7 +26,7 @@ class BackgroundTaskManager extends Emitter {
    return this.tasks.size;
  }

-  async newTask(type, opts) {
+  async newTask(type, opts, sticky = false) {
    this.logger.info({opts}, `initiating Background task ${type}`);
    if (this.tasks.has(type)) {
      this.logger.info(`Background task ${type} is running, skipped`);
@@ -52,6 +52,7 @@ class BackgroundTaskManager extends Emitter {
    if (task) {
      this.tasks.set(type, task);
    }
+    if (task && sticky) task.sticky = true;
    return task;
  }

@@ -116,7 +117,7 @@ class BackgroundTaskManager extends Emitter {
          this._taskCompleted('bargeIn', task);
          if (task.sticky && !this.cs.callGone && !this.cs._stopping) {
            this.logger.info('BackgroundTaskManager:_initBargeIn: restarting background bargeIn');
-            this.newTask('bargeIn', opts);
+            this.newTask('bargeIn', opts, true);
          }
          return;
        })
--- a/lib/utils/db-utils.js
+++ b/lib/utils/db-utils.js
@@ -94,6 +94,17 @@ const speechMapper = (cred) => {
      obj.api_key = o.api_key;
      obj.model_id = o.model_id;
      obj.options = o.options;
+    } else if ('playht' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = o.api_key;
+      obj.user_id = o.user_id;
+      obj.voice_engine = o.voice_engine;
+      obj.options = o.options;
+    } else if ('rimelabs' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = o.api_key;
+      obj.model_id = o.model_id;
+      obj.options = o.options;
    } else if ('assemblyai' === obj.vendor) {
      const o = JSON.parse(decrypt(credential));
      obj.api_key = o.api_key;
--- a/lib/utils/transcription-utils.js
+++ b/lib/utils/transcription-utils.js
@@ -690,7 +690,9 @@ module.exports = (logger) => {
        ...(deepgramOptions.keywords) &&
          {DEEPGRAM_SPEECH_KEYWORDS: deepgramOptions.keywords.join(',')},
        ...('endpointing' in deepgramOptions) &&
-          {DEEPGRAM_SPEECH_ENDPOINTING: deepgramOptions.endpointing === false ? 'false' : deepgramOptions.endpointing},
+          {DEEPGRAM_SPEECH_ENDPOINTING: deepgramOptions.endpointing === false ? 'false' : deepgramOptions.endpointing,
+            // default DEEPGRAM_SPEECH_UTTERANCE_END_MS is 1000, will be override by user settings later if there is.
+            DEEPGRAM_SPEECH_UTTERANCE_END_MS: 1000},
        ...(deepgramOptions.utteranceEndMs) &&
          {DEEPGRAM_SPEECH_UTTERANCE_END_MS: deepgramOptions.utteranceEndMs},
        ...(deepgramOptions.vadTurnoff) &&
--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
  "name": "jambonz-feature-server",
-  "version": "0.8.6",
+  "version": "0.9.0",
  "main": "app.js",
  "engines": {
    "node": ">= 18.x"
@@ -25,57 +25,57 @@
    "jslint:fix": "eslint app.js tracer.js lib --fix"
  },
  "dependencies": {
-    "@aws-sdk/client-auto-scaling": "^3.360.0",
-    "@aws-sdk/client-sns": "^3.360.0",
+    "@aws-sdk/client-auto-scaling": "^3.549.0",
+    "@aws-sdk/client-sns": "^3.549.0",
    "@jambonz/db-helpers": "^0.9.3",
    "@jambonz/http-health-check": "^0.0.1",
-    "@jambonz/mw-registrar": "^0.2.4",
-    "@jambonz/realtimedb-helpers": "^0.8.7",
-    "@jambonz/speech-utils": "^0.0.48",
+    "@jambonz/mw-registrar": "^0.2.7",
+    "@jambonz/realtimedb-helpers": "^0.8.8",
+    "@jambonz/speech-utils": "^0.0.51",
    "@jambonz/stats-collector": "^0.1.9",
    "@jambonz/time-series": "^0.2.8",
    "@jambonz/verb-specifications": "^0.0.69",
-    "@opentelemetry/api": "^1.4.0",
-    "@opentelemetry/exporter-jaeger": "^1.9.0",
-    "@opentelemetry/exporter-trace-otlp-http": "^0.35.0",
-    "@opentelemetry/exporter-zipkin": "^1.9.0",
-    "@opentelemetry/instrumentation": "^0.35.0",
-    "@opentelemetry/resources": "^1.9.0",
-    "@opentelemetry/sdk-trace-base": "^1.9.0",
-    "@opentelemetry/sdk-trace-node": "^1.9.0",
-    "@opentelemetry/semantic-conventions": "^1.9.0",
+    "@opentelemetry/api": "^1.8.0",
+    "@opentelemetry/exporter-jaeger": "^1.23.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
+    "@opentelemetry/exporter-zipkin": "^1.23.0",
+    "@opentelemetry/instrumentation": "^0.50.0",
+    "@opentelemetry/resources": "^1.23.0",
+    "@opentelemetry/sdk-trace-base": "^1.23.0",
+    "@opentelemetry/sdk-trace-node": "^1.23.0",
+    "@opentelemetry/semantic-conventions": "^1.23.0",
    "bent": "^7.3.12",
    "debug": "^4.3.4",
    "deepcopy": "^2.1.0",
    "drachtio-fsmrf": "^3.0.40",
    "drachtio-srf": "^4.5.31",
-    "express": "^4.18.2",
+    "express": "^4.19.2",
    "express-validator": "^7.0.1",
-    "ip": "^1.1.9",
-    "moment": "^2.29.4",
-    "parse-url": "^8.1.0",
-    "pino": "^8.8.0",
+    "ip": "^2.0.1",
+    "moment": "^2.30.1",
+    "parse-url": "^9.2.0",
+    "pino": "^8.20.0",
    "polly-ssml-split": "^0.1.0",
    "proxyquire": "^2.1.3",
-    "sdp-transform": "^2.14.1",
+    "sdp-transform": "^2.14.2",
    "short-uuid": "^4.2.2",
-    "sinon": "^15.0.1",
+    "sinon": "^17.0.1",
    "to-snake-case": "^1.0.0",
    "undici": "^6.11.1",
    "uuid-random": "^1.3.2",
    "verify-aws-sns-signature": "^0.1.0",
-    "ws": "^8.9.0",
+    "ws": "^8.16.0",
    "xml2js": "^0.6.2"
  },
  "devDependencies": {
    "clear-module": "^4.1.2",
-    "eslint": "^7.32.0",
-    "eslint-plugin-promise": "^4.3.1",
+    "eslint": "7.32.0",
+    "eslint-plugin-promise": "^6.1.1",
    "nyc": "^15.1.0",
-    "tape": "^5.6.1"
+    "tape": "^5.7.5"
  },
  "optionalDependencies": {
-    "bufferutil": "^4.0.6",
-    "utf-8-validate": "^5.0.8"
+    "bufferutil": "^4.0.8",
+    "utf-8-validate": "^6.0.3"
  }
 }
Author	SHA1	Message	Date
Quan HL	b7f6c5b20c	wip	2024-04-17 23:11:03 +07:00
Quan HL	ad5df4b7f7	support google tts mod	2024-04-17 22:09:55 +07:00
Dave Horton	bd002ede48	ignore google errors with error_code 0	2024-04-16 20:06:26 -04:00
Dave Horton	1a2aa91973	proper fix for precache (#721 ) * proper fix for precache * wip	2024-04-15 16:25:12 -04:00
Dave Horton	e322b7d8d3	be more cautious about pre-caching prompts; in particular, a Config verb will not give us time to precache so avoid in that scenario (#720 )	2024-04-15 15:38:10 -04:00
Hoan Luu Huu	7da11df88e	default DEEPGRAM_SPEECH_UTTERANCE_END_MS is 1000 (#719 )	2024-04-14 19:39:07 -04:00
Hoan Luu Huu	09cf1345f6	tts span for whisper (#718 ) * tts span for whisper * support deepgram tts span * support playht tts span * support rimelabs tts span * wip	2024-04-14 09:14:49 -04:00
Dave Horton	2595f527ff	gather: fix bug where empty deepgram transcript saved incorrectly	2024-04-13 09:59:02 -04:00
Dave Horton	1d77c0cd20	bugfx: bargein after first when config bargein with sticky=true fails	2024-04-12 20:08:21 -04:00
Hoan Luu Huu	9eab81268b	support mod_rimelabs_tts (#716 ) * support mod_rimelabs_tts * update speech utils	2024-04-12 07:28:45 -04:00
Dave Horton	ecf3d140d6	fix #714 (#715 )	2024-04-10 16:23:22 -04:00
Hoan Luu Huu	4a52be9171	support mod_playht_tts (#711 ) * support mod_playht_tts * update speech utils version	2024-04-08 10:21:54 -04:00
Dave Horton	9b722ae36d	update deps (#709 ) * update deps * version	2024-04-07 18:22:31 -04:00
Dave Horton	370b046fac	update to speech utils with azure 1.36.0	2024-04-07 12:16:35 -04:00