0.0.42

support whisper streaming

index.js

@@ -12,6 +12,7 @@ module.exports = (opts, logger) => {
     client,
     getTtsSize: require('./lib/get-tts-size').bind(null, client, logger),
     purgeTtsCache: require('./lib/purge-tts-cache').bind(null, client, logger),
+    addFileToCache: require('./lib/add-file-to-cache').bind(null, client, logger),
     synthAudio: require('./lib/synth-audio').bind(null, client, logger),
     getNuanceAccessToken: require('./lib/get-nuance-access-token').bind(null, client, logger),
     getIbmAccessToken: require('./lib/get-ibm-access-token').bind(null, client, logger),

lib/add-file-to-cache.js

@@ -0,0 +1,30 @@
+const fs = require('fs/promises');
+const {noopLogger, makeSynthKey} = require('./utils');
+const EXPIRES = (process.env.JAMBONES_TTS_CACHE_DURATION_MINS || 4 * 60) * 60; // cache tts for 4 hours
+
+async function addFileToCache(client, logger, path,
+  {account_sid, vendor, language, voice, deploymentId, engine, text}) {
+  let key;
+  logger = logger || noopLogger;
+
+  try {
+    key = makeSynthKey({
+      account_sid,
+      vendor,
+      language: language || '',
+      voice: voice || deploymentId,
+      engine,
+      text,
+    });
+    const audioBuffer = await fs.readFile(path);
+    await client.setex(key, EXPIRES, audioBuffer.toString('base64'));
+  } catch (err) {
+    logger.error(err, 'addFileToCache: Error');
+    return;
+  }
+
+  logger.debug(`addFileToCache: added ${path} to cache with key ${key}`);
+  return key;
+}
+
+module.exports = addFileToCache;

lib/get-tts-voices.js

@@ -97,7 +97,7 @@ const getAwsVoices = async(_client, logger, credentials) => {
         secretAccessKey
       }
     });
-    const command = new DescribeVoicesCommand({LanguageCode: 'en-US'});
+    const command = new DescribeVoicesCommand({});
     const response = await client.send(command);
     return response;
   } catch (err) {

lib/synth-audio.js

@@ -76,7 +76,8 @@ const trimTrailingSilence = (buffer) => {
  * the synthesized audio, and a variable indicating whether it was served from cache
  */
 async function synthAudio(client, logger, stats, { account_sid,
-  vendor, language, voice, gender, text, engine, salt, model, credentials, deploymentId, disableTtsCache, options
+  vendor, language, voice, gender, text, engine, salt, model, credentials, deploymentId,
+  disableTtsCache, renderForCaching, options
 }) {
   let audioBuffer;
   let servedFromCache = false;
@@ -84,7 +85,7 @@ async function synthAudio(client, logger, stats, { account_sid,
   logger = logger || noopLogger;
 
   assert.ok(['google', 'aws', 'polly', 'microsoft',
-    'wellsaid', 'nuance', 'nvidia', 'ibm', 'elevenlabs', 'whisper'].includes(vendor) ||
+    'wellsaid', 'nuance', 'nvidia', 'ibm', 'elevenlabs', 'whisper', 'deepgram'].includes(vendor) ||
   vendor.startsWith('custom'),
   `synthAudio supported vendors are google, aws, microsoft, nuance, nvidia and wellsaid, not ${vendor}`);
   if ('google' === vendor) {
@@ -142,6 +143,10 @@ async function synthAudio(client, logger, stats, { account_sid,
     (
       process.env.JAMBONES_TTS_TRIM_SILENCE &&
       ['microsoft', 'azure'].includes(vendor)
+    ) ||
+    (
+      !process.env.JAMBONES_DISABLE_TTS_STREAMING &&
+      vendor === 'elevenlabs'
     )
   ) {
     filePath = `${TMP_FOLDER}/${key.replace('tts:', `tts-${salt || ''}`)}.r8`;
@@ -194,11 +199,26 @@ async function synthAudio(client, logger, stats, { account_sid,
         audioBuffer = await synthWellSaid(logger, {credentials, stats, language, voice, text, filePath});
         break;
       case 'elevenlabs':
-        audioBuffer = await synthElevenlabs(logger, {credentials, options, stats, language, voice, text, filePath});
+        audioBuffer = await synthElevenlabs(logger, {
+          credentials, options, stats, language, voice, text, renderForCaching, filePath
+        });
+        if (typeof audioBuffer === 'object' && audioBuffer.filePath) {
+          return audioBuffer;
+        }
+        else {
+          audioBuffer = await synthElevenlabs(logger, {credentials, options, stats, language, voice, text, filePath});
+        }
         break;
       case 'whisper':
+        audioBuffer = await synthWhisper(logger, {credentials, stats, voice, text, renderForCaching});
+        if (typeof audioBuffer === 'object' && audioBuffer.filePath) {
+          return audioBuffer;
+        }
         audioBuffer = await synthWhisper(logger, {credentials, stats, voice, text});
         break;
+      case 'deepgram':
+        audioBuffer = await synthDeepgram(logger, {credentials, stats, model, text});
+        break;
       case vendor.startsWith('custom') ? vendor : 'cant_match_value':
         ({ audioBuffer, filePath } = await synthCustomVendor(logger,
           {credentials, stats, language, voice, text, filePath}));
@@ -591,9 +611,30 @@ const synthCustomVendor = async(logger, {credentials, stats, language, voice, te
   }
 };
 
-const synthElevenlabs = async(logger, {credentials, options, stats, language, voice, text}) => {
+const synthElevenlabs = async(logger, {credentials, options, stats, language, voice, text, renderForCaching}) => {
   const {api_key, model_id, options: credOpts} = credentials;
   const opts = !!options && Object.keys(options).length !== 0 ? options : JSON.parse(credOpts || '{}');
+
+  /* default to using the streaming interface, unless disabled by env var OR we want just a cache file */
+  if (!process.env.JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching) {
+    let params = '';
+    params += `{api_key=${api_key}`;
+    params += `,model_id=${model_id}`;
+    params += `,optimize_streaming_latency=${opts.optimize_streaming_latency || 2}`;
+    params += ',write_cache_file=1';
+    if (opts.voice_settings?.similarity_boost) params += `,similarity_boost=${opts.voice_settings.similarity_boost}`;
+    if (opts.voice_settings?.stability) params += `,stability=${opts.voice_settings.stability}`;
+    if (opts.voice_settings?.style) params += `,style=${opts.voice_settings.style}`;
+    if (opts.voice_settings?.use_speaker_boost === false) params += ',use_speaker_boost=false';
+    params += '}';
+
+    return {
+      filePath: `say:${params}${text.replace(/\n/g, ' ').replace(/\r/g, ' ')}`,
+      servedFromCache: false,
+      rtt: 0
+    };
+  }
+
   const optimize_streaming_latency = opts.optimize_streaming_latency ?
     `?optimize_streaming_latency=${opts.optimize_streaming_latency}` : '';
   try {
@@ -619,8 +660,24 @@ const synthElevenlabs = async(logger, {credentials, options, stats, language, vo
   }
 };
 
-const synthWhisper = async(logger, {credentials, stats, voice, text}) => {
-  const {api_key, model_id, baseURL, timeout} = credentials;
+const synthWhisper = async(logger, {credentials, stats, voice, text, renderForCaching}) => {
+  const {api_key, model_id, baseURL, timeout, speed} = credentials;
+  /* if the env is set to stream then bag out, unless we are specifically rendering to generate a cache file */
+  if (!process.env.JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching) {
+    let params = '';
+    params += `{api_key=${api_key}`;
+    params += `,model_id=${model_id}`;
+    params += `,voice=${voice}`;
+    params += ',write_cache_file=1';
+    if (speed) params += `,speed=${speed}`;
+    params += '}';
+
+    return {
+      filePath: `say:${params}${text.replace(/\n/g, ' ')}`,
+      servedFromCache: false,
+      rtt: 0
+    };
+  }
   try {
     const openai = new OpenAI.OpenAI({
       apiKey: api_key,
@@ -640,8 +697,27 @@ const synthWhisper = async(logger, {credentials, stats, voice, text}) => {
     stats.increment('tts.count', ['vendor:openai', 'accepted:no']);
     throw err;
   }
-}
-;
+};
+
+const synthDeepgram = async(logger, {credentials, stats, model, text}) => {
+  const {api_key} = credentials;
+  try {
+    const post = bent('https://api.beta.deepgram.com', 'POST', 'buffer', {
+      'Authorization': `Token ${api_key}`,
+      'Accept': 'audio/mpeg',
+      'Content-Type': 'application/json'
+    });
+    const mp3 = await post(`/v1/speak?model=${model}`, {
+      text
+    });
+    return mp3;
+  } catch (err) {
+    logger.info({err}, 'synth Deepgram returned error');
+    stats.increment('tts.count', ['vendor:deepgram', 'accepted:no']);
+    throw err;
+  }
+};
+
 const getFileExtFromMime = (mime) => {
   switch (mime) {
     case 'audio/wav':

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.0.32",
+  "version": "0.0.42",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",
@@ -24,26 +24,26 @@
   },
   "homepage": "https://github.com/jambonz/speech-utils#readme",
   "dependencies": {
-    "@aws-sdk/client-polly": "^3.359.0",
-    "@aws-sdk/client-sts": "^3.458.0",
-    "@google-cloud/text-to-speech": "^4.2.1",
-    "@grpc/grpc-js": "^1.8.13",
+    "@aws-sdk/client-polly": "^3.496.0",
+    "@aws-sdk/client-sts": "^3.496.0",
+    "@google-cloud/text-to-speech": "^5.0.2",
+    "@grpc/grpc-js": "^1.9.14",
     "@jambonz/realtimedb-helpers": "^0.8.7",
     "bent": "^7.3.12",
     "debug": "^4.3.4",
-    "form-urlencoded": "^6.1.0",
+    "form-urlencoded": "^6.1.4",
     "google-protobuf": "^3.21.2",
     "ibm-watson": "^8.0.0",
-    "microsoft-cognitiveservices-speech-sdk": "1.32.0",
-    "openai": "^4.16.2",
-    "undici": "^5.21.0"
+    "microsoft-cognitiveservices-speech-sdk": "1.34.0",
+    "openai": "^4.25.0",
+    "undici": "^6.4.0"
   },
   "devDependencies": {
-    "config": "^3.3.9",
-    "eslint": "^8.33.0",
+    "config": "^3.3.10",
+    "eslint": "^8.56.0",
     "eslint-plugin-promise": "^6.1.1",
     "nyc": "^15.1.0",
-    "pino": "^7.2.0",
-    "tape": "^5.1.1"
+    "pino": "^8.17.0",
+    "tape": "^5.7.3"
   }
 }

test/synth.js

@@ -20,7 +20,7 @@ const stats = {
 
 test('Google speech synth tests', async(t) => {
   const fn = require('..');
-  const {synthAudio, client} = fn(opts, logger);
+  const {synthAudio, addFileToCache, client} = fn(opts, logger);
 
   if (!process.env.GCP_FILE && !process.env.GCP_JSON_KEY) {
     t.pass('skipping google speech synth tests since neither GCP_FILE nor GCP_JSON_KEY provided');
@@ -58,6 +58,14 @@ test('Google speech synth tests', async(t) => {
     });
     t.ok(opts.servedFromCache, `successfully retrieved cached google audio from ${opts.filePath}`);
 
+    const success = await addFileToCache(opts.filePath, {
+      vendor: 'google',
+      language: 'en-GB',
+      gender: 'FEMALE',
+      text: 'This is a test.  This is only a test'
+    });
+    t.ok(success, `successfully added ${opts.filePath} to cache`);
+
     opts = await synthAudio(stats, {
       vendor: 'google',
       credentials: {
@@ -514,6 +522,33 @@ test('whisper speech synth tests', async(t) => {
   client.quit();
 })
 
+test('Deepgram speech synth tests', async(t) => {
+  const fn = require('..');
+  const {synthAudio, client} = fn(opts, logger);
+
+  if (!process.env.DEEPGRAM_API_KEY) {
+    t.pass('skipping Deepgram speech synth tests since DEEPGRAM_API_KEY');
+    return t.end();
+  }
+  const text = 'Hi there and welcome to jambones!';
+  try {
+    let opts = await synthAudio(stats, {
+      vendor: 'deepgram',
+      credentials: {
+        api_key: process.env.DEEPGRAM_API_KEY
+      },
+      model: 'alpha-asteria-en-v2',
+      text,
+    });
+    t.ok(!opts.servedFromCache, `successfully synthesized deepgram audio to ${opts.filePath}`);
+
+  } catch (err) {
+    console.error(JSON.stringify(err));
+    t.end(err);
+  }
+  client.quit();
+})
+
 test('TTS Cache tests', async(t) => {
   const fn = require('..');
   const {purgeTtsCache, getTtsSize, client} = fn(opts, logger);