0.0.48

Feat/mod deepgram tts

lib/synth-audio.js

@@ -141,12 +141,12 @@ async function synthAudio(client, logger, stats, { account_sid,
   let filePath;
   if (['nuance', 'nvidia'].includes(vendor) ||
     (
-      process.env.JAMBONES_TTS_TRIM_SILENCE &&
+      (process.env.JAMBONES_TTS_TRIM_SILENCE || !process.env.JAMBONES_DISABLE_TTS_STREAMING) &&
       ['microsoft', 'azure'].includes(vendor)
     ) ||
     (
       !process.env.JAMBONES_DISABLE_TTS_STREAMING &&
-      vendor === 'elevenlabs'
+      ['elevenlabs', 'deepgram'].includes(vendor)
     )
   ) {
     filePath = `${TMP_FOLDER}/${key.replace('tts:', `tts-${salt || ''}`)}.r8`;
@@ -183,7 +183,9 @@ async function synthAudio(client, logger, stats, { account_sid,
       case 'azure':
       case 'microsoft':
         vendorLabel = 'microsoft';
-        audioBuffer = await synthMicrosoft(logger, {credentials, stats, language, voice, text, deploymentId, filePath});
+        audioBuffer = await synthMicrosoft(logger, {credentials, stats, language, voice, text, deploymentId,
+          filePath, renderForCaching, disableTtsStreaming});
+        if (audioBuffer?.filePath) return audioBuffer;
         break;
       case 'nuance':
         model = model || 'enhanced';
@@ -210,7 +212,9 @@ async function synthAudio(client, logger, stats, { account_sid,
         if (audioBuffer?.filePath) return audioBuffer;
         break;
       case 'deepgram':
-        audioBuffer = await synthDeepgram(logger, {credentials, stats, model, text});
+        audioBuffer = await synthDeepgram(logger, {credentials, stats, model, text,
+          renderForCaching, disableTtsStreaming});
+        if (audioBuffer?.filePath) return audioBuffer;
         break;
       case vendor.startsWith('custom') ? vendor : 'cant_match_value':
         ({ audioBuffer, filePath } = await synthCustomVendor(logger,
@@ -381,10 +385,47 @@ const synthMicrosoft = async(logger, {
   language,
   voice,
   text,
-  filePath
+  filePath,
+  renderForCaching,
+  disableTtsStreaming
 }) => {
   try {
     const {api_key: apiKey, region, use_custom_tts, custom_tts_endpoint, custom_tts_endpoint_url} = credentials;
+    // let clean up the text
+    let content = text;
+    if (use_custom_tts && !content.startsWith('<speak')) {
+      /**
+       * Note: it seems that to use custom voice ssml is required with the voice attribute
+       * Otherwise sending plain text we get "Voice does not match"
+       */
+      content = `<speak>${text}</speak>`;
+    }
+
+    if (content.startsWith('<speak>')) {
+      /* microsoft enforces some properties and uses voice xml element so if the user did not supply do it for them */
+      const words = content.slice(7, -8).trim().replace(/(\r\n|\n|\r)/gm, ' ');
+      // eslint-disable-next-line max-len
+      content = `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xml:lang="${language}"><voice name="${voice}">${words}</voice></speak>`;
+      logger.info({content}, 'synthMicrosoft');
+    }
+    if (!process.env.JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
+      let params = '';
+      params += `{api_key=${apiKey}`;
+      params += `,language=${language}`;
+      params += ',vendor=microsoft';
+      params += `,voice=${voice}`;
+      params += ',write_cache_file=1';
+      if (region) params += `,region=${region}`;
+      if (custom_tts_endpoint) params += `,endpointId=${custom_tts_endpoint}`;
+      if (process.env.JAMBONES_HTTP_PROXY_IP) params += `,http_proxy_ip=${process.env.JAMBONES_HTTP_PROXY_IP}`;
+      if (process.env.JAMBONES_HTTP_PROXY_PORT) params += `,http_proxy_port=${process.env.JAMBONES_HTTP_PROXY_PORT}`;
+      params += '}';
+      return {
+        filePath: `say:${params}${content.replace(/\n/g, ' ')}`,
+        servedFromCache: false,
+        rtt: 0
+      };
+    }
     if (use_custom_tts && custom_tts_endpoint_url) {
       return await _synthOnPremMicrosoft(logger, {
         credentials,
@@ -396,20 +437,12 @@ const synthMicrosoft = async(logger, {
       });
     }
     const trimSilence = filePath.endsWith('.r8');
-    let content = text;
     const speechConfig = SpeechConfig.fromSubscription(apiKey, region);
     speechConfig.speechSynthesisLanguage = language;
     speechConfig.speechSynthesisVoiceName = voice;
     if (use_custom_tts && custom_tts_endpoint) {
       speechConfig.endpointId = custom_tts_endpoint;
     }
-    if (use_custom_tts && !content.startsWith('<speak')) {
-      /**
-       * Note: it seems that to use custom voice ssml is required with the voice attribute
-       * Otherwise sending plain text we get "Voice does not match"
-       */
-      content = `<speak>${text}</speak>`;
-    }
     speechConfig.speechSynthesisOutputFormat = trimSilence ?
       SpeechSynthesisOutputFormat.Raw8Khz16BitMonoPcm :
       SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3;
@@ -421,14 +454,6 @@ const synthMicrosoft = async(logger, {
     }
     const synthesizer = new SpeechSynthesizer(speechConfig);
 
-    if (content.startsWith('<speak>')) {
-      /* microsoft enforces some properties and uses voice xml element so if the user did not supply do it for them */
-      const words = content.slice(7, -8).trim().replace(/(\r\n|\n|\r)/gm, ' ');
-      // eslint-disable-next-line max-len
-      content = `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xml:lang="${language}"><voice name="${voice}">${words}</voice></speak>`;
-      logger.info({content}, 'synthMicrosoft');
-    }
-
     return new Promise((resolve, reject) => {
       const speakAsync = content.startsWith('<speak') ?
         synthesizer.speakSsmlAsync.bind(synthesizer) :
@@ -697,8 +722,22 @@ const synthWhisper = async(logger, {credentials, stats, voice, text, renderForCa
   }
 };
 
-const synthDeepgram = async(logger, {credentials, stats, model, text}) => {
+const synthDeepgram = async(logger, {credentials, stats, model, text, renderForCaching, disableTtsStreaming}) => {
   const {api_key} = credentials;
+  if (!process.env.JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
+    let params = '';
+    params += `{api_key=${api_key}`;
+    params += ',vendor=deepgram';
+    params += `,voice=${model}`;
+    params += ',write_cache_file=1';
+    params += '}';
+
+    return {
+      filePath: `say:${params}${text.replace(/\n/g, ' ')}`,
+      servedFromCache: false,
+      rtt: 0
+    };
+  }
   try {
     const post = bent('https://api.deepgram.com', 'POST', 'buffer', {
       'Authorization': `Token ${api_key}`,

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.0.44",
+  "version": "0.0.48",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@jambonz/speech-utils",
-      "version": "0.0.44",
+      "version": "0.0.48",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-polly": "^3.496.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.0.44",
+  "version": "0.0.48",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",

test/synth.js

@@ -188,6 +188,7 @@ test('Azure speech synth tests', async(t) => {
       language: 'en-US',
       voice: 'en-US-ChristopherNeural',
       text: longText,
+      renderForCaching: true
     });
     t.ok(!opts.servedFromCache, `successfully synthesized microsoft audio to ${opts.filePath}`);
     if (process.env.JAMBONES_HTTP_PROXY_IP && process.env.JAMBONES_HTTP_PROXY_PORT) {
@@ -203,6 +204,7 @@ test('Azure speech synth tests', async(t) => {
       language: 'en-US',
       voice: 'en-US-ChristopherNeural',
       text: longText,
+      renderForCaching: true
     });
     t.ok(opts.servedFromCache, `successfully retrieved microsoft audio from cache ${opts.filePath}`);
   } catch (err) {
@@ -237,6 +239,7 @@ test('Azure SSML tests', async(t) => {
       language: 'en-US',
       voice: 'en-US-ChristopherNeural',
       text,
+      renderForCaching: true
     });
     t.ok(!opts.servedFromCache, `successfully synthesized microsoft audio to ${opts.filePath}`);
     if (process.env.JAMBONES_HTTP_PROXY_IP && process.env.JAMBONES_HTTP_PROXY_PORT) {
@@ -252,6 +255,7 @@ test('Azure SSML tests', async(t) => {
       language: 'en-US',
       voice: 'en-US-ChristopherNeural',
       text,
+      renderForCaching: true
     });
     t.ok(opts.servedFromCache, `successfully retrieved microsoft audio from cache ${opts.filePath}`);
   } catch (err) {
@@ -283,6 +287,7 @@ test('Azure custom voice speech synth tests', async(t) => {
       language: 'en-US',
       voice: process.env.MICROSOFT_CUSTOM_VOICE,
       text,
+      renderForCaching: true
     });
     t.ok(!opts.servedFromCache, `successfully synthesized microsoft audio to ${opts.filePath}`);
 
@@ -297,6 +302,7 @@ test('Azure custom voice speech synth tests', async(t) => {
       language: 'en-US',
       voice: process.env.MICROSOFT_CUSTOM_VOICE,
       text,
+      renderForCaching: true
     });
     t.ok(opts.servedFromCache, `successfully retrieved microsoft custom voice audio from cache ${opts.filePath}`);
   } catch (err) {
@@ -589,6 +595,7 @@ test('Deepgram speech synth tests', async(t) => {
       },
       model: 'aura-asteria-en',
       text,
+      renderForCaching: true
     });
     t.ok(!opts.servedFromCache, `successfully synthesized deepgram audio to ${opts.filePath}`);