cartesia should have speed if speed value is 0.0

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: lts/*
       - run: npm install
       - name: Install Docker Compose
         run: |

lib/add-file-to-cache.js

@@ -3,29 +3,8 @@ const {noopLogger, makeSynthKey} = require('./utils');
 const {JAMBONES_TTS_CACHE_DURATION_MINS} = require('./config');
 const EXPIRES = JAMBONES_TTS_CACHE_DURATION_MINS;
 
-function getExtensionAndSampleRate(path) {
-  const match = path.match(/\.([^.]*)$/);
-  if (!match) {
-    //default should be wav file.
-    return ['wav', 8000];
-  }
-
-  const extension = match[1];
-  const sampleRateMap = {
-    r8: 8000,
-    r16: 16000,
-    r24: 24000,
-    r44: 44100,
-    r48: 48000,
-    r96: 96000,
-  };
-
-  const sampleRate = sampleRateMap[extension] || 8000;
-  return [extension, sampleRate];
-}
-
 async function addFileToCache(client, logger, path,
-  {account_sid, vendor, language, voice, deploymentId, engine, model, text, instructions}) {
+  {account_sid, vendor, language, voice, deploymentId, engine, text}) {
   let key;
   logger = logger || noopLogger;
 
@@ -36,19 +15,10 @@ async function addFileToCache(client, logger, path,
       language: language || '',
       voice: voice || deploymentId,
       engine,
-      model,
       text,
-      instructions
     });
-    const [extension, sampleRate] = getExtensionAndSampleRate(path);
     const audioBuffer = await fs.readFile(path);
-    await client.setex(key, EXPIRES, JSON.stringify(
-      {
-        audioContent: audioBuffer.toString('base64'),
-        extension,
-        sampleRate
-      }
-    ));
+    await client.setex(key, EXPIRES, audioBuffer.toString('base64'));
   } catch (err) {
     logger.error(err, 'addFileToCache: Error');
     return;

lib/config.js

@@ -2,14 +2,13 @@ const JAMBONES_TTS_TRIM_SILENCE = process.env.JAMBONES_TTS_TRIM_SILENCE;
 const JAMBONES_DISABLE_TTS_STREAMING = process.env.JAMBONES_DISABLE_TTS_STREAMING;
 const JAMBONES_DISABLE_AZURE_TTS_STREAMING = process.env.JAMBONES_DISABLE_AZURE_TTS_STREAMING;
 const JAMBONES_EAGERLY_PRE_CACHE_AUDIO = process.env.JAMBONES_EAGERLY_PRE_CACHE_AUDIO;
-const JAMBONES_AZURE_ENABLE_SSML = process.env.JAMBONES_AZURE_ENABLE_SSML;
 
 const JAMBONES_HTTP_PROXY_IP = process.env.JAMBONES_HTTP_PROXY_IP;
 const JAMBONES_HTTP_PROXY_PORT = process.env.JAMBONES_HTTP_PROXY_PORT;
 const JAMBONES_TTS_CACHE_DURATION_MINS =
   (parseInt(process.env.JAMBONES_TTS_CACHE_DURATION_MINS) || 4 * 60) * 60; // cache tts for 4 hours
 
-const TMP_FOLDER = process.env.JAMBONES_TMP_FOLDER || '/tmp';
+const TMP_FOLDER = '/tmp';
 
 const HTTP_TIMEOUT = 5000;
 
@@ -22,6 +21,5 @@ module.exports = {
   JAMBONES_TTS_CACHE_DURATION_MINS,
   JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
   TMP_FOLDER,
-  HTTP_TIMEOUT,
-  JAMBONES_AZURE_ENABLE_SSML
+  HTTP_TIMEOUT
 };

lib/purge-tts-cache.js

@@ -12,7 +12,7 @@ const debug = require('debug')('jambonz:realtimedb-helpers');
  * @returns {object} result - {error, purgedCount}
  */
 async function purgeTtsCache(client, logger, {all, account_sid, vendor,
-  language, voice, deploymentId, engine, model, text, instructions} = {all: true}) {
+  language, voice, deploymentId, engine, text} = {all: true}) {
   logger = logger || noopLogger;
 
   let purgedCount = 0, error;
@@ -33,9 +33,7 @@ async function purgeTtsCache(client, logger, {all, account_sid, vendor,
         language: language || '',
         voice: voice || deploymentId,
         engine,
-        model,
         text,
-        instructions
       });
       purgedCount = await client.del(key);
       if (purgedCount === 0) error = 'Specified item not found';

lib/utils.js

@@ -6,7 +6,7 @@ const pool = new Pool('https://auth.crt.nuance.com');
 const NUANCE_AUTH_ENDPOINT = 'tts.api.nuance.com:443';
 const grpc = require('@grpc/grpc-js');
 const formurlencoded = require('form-urlencoded');
-const { TMP_FOLDER, HTTP_TIMEOUT } = require('./config');
+const { JAMBONES_DISABLE_TTS_STREAMING, JAMBONES_TTS_TRIM_SILENCE, TMP_FOLDER, HTTP_TIMEOUT } = require('./config');
 
 const debug = require('debug')('jambonz:realtimedb-helpers');
 /**
@@ -17,27 +17,68 @@ const debug = require('debug')('jambonz:realtimedb-helpers');
 //const nuanceClientMap = new Map();
 
 function makeSynthKey({
-  account_sid = '',
-  vendor,
-  language,
-  voice,
-  engine = '',
-  model = '',
-  text,
-  instructions = '',
-}) {
+  account_sid = '', vendor, language, voice, engine = '', text,
+  renderForCaching = false}) {
   const hash = crypto.createHash('sha1');
-  hash.update(`${language}:${vendor}:${voice}:${engine}:${model}:${text}:${instructions}`);
+  hash.update(`${language}:${vendor}:${voice}:${engine}:${text}`);
   const hexHashKey = hash.digest('hex');
   const accountKey = account_sid ? `:${account_sid}` : '';
-  const key = `tts${accountKey}:${hexHashKey}`;
+  const namespace = vendor.startsWith('custom') ? vendor : getFileExtension({vendor, voice, renderForCaching});
+  const key = `tts${accountKey}:${namespace}:${hexHashKey}`;
   return key;
 }
 
-function makeFilePath({key, salt = '', extension}) {
+function makeFilePath({vendor, voice, key, salt = '', renderForCaching = false}) {
+  const extension = getFileExtension({vendor, renderForCaching, voice});
   return `${TMP_FOLDER}/${key.replace('tts:', `tts-${salt}`)}.${extension}`;
 }
 
+function getFileExtension({vendor, voice, renderForCaching = false}) {
+  const mp3Extension = 'mp3';
+  const r8Extension = 'r8';
+  const wavExtension = 'wav';
+
+  switch (vendor) {
+    case 'azure':
+    case 'microsoft':
+      if (!renderForCaching && !JAMBONES_DISABLE_TTS_STREAMING || JAMBONES_TTS_TRIM_SILENCE) {
+        return r8Extension;
+      } else {
+        return mp3Extension;
+      }
+    case 'deepgram':
+    case 'elevenlabs':
+    case 'rimelabs':
+    case 'playht':
+    case 'cartesia':
+      if (renderForCaching || JAMBONES_DISABLE_TTS_STREAMING) {
+        return mp3Extension;
+      } else {
+        return r8Extension;
+      }
+    case 'nuance':
+    case 'nvidia':
+    case 'verbio':
+      return r8Extension;
+    case 'google':
+      // google voice cloning just support wav.
+      if (typeof voice === 'object' && voice.voice_cloning_key) {
+        return wavExtension;
+      } else {
+        return mp3Extension;
+      }
+    default:
+      // If vendor is custom
+      if (vendor.startsWith('custom')) {
+        if (renderForCaching || JAMBONES_DISABLE_TTS_STREAMING) {
+          return mp3Extension;
+        } else {
+          return r8Extension;
+        }
+      }
+      return mp3Extension;
+  }
+}
 
 const noopLogger = {
   info: () => {},

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.2.26",
+  "version": "0.1.24",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",
@@ -26,10 +26,9 @@
   },
   "homepage": "https://github.com/jambonz/speech-utils#readme",
   "dependencies": {
-    "23": "^0.0.0",
     "@aws-sdk/client-polly": "^3.496.0",
     "@aws-sdk/client-sts": "^3.496.0",
-    "@cartesia/cartesia-js": "^2.2.7",
+    "@cartesia/cartesia-js": "^2.1.0",
     "@google-cloud/text-to-speech": "^5.5.0",
     "@grpc/grpc-js": "^1.9.14",
     "@jambonz/realtimedb-helpers": "^0.8.7",
@@ -37,10 +36,10 @@
     "debug": "^4.3.4",
     "form-urlencoded": "^6.1.4",
     "google-protobuf": "^3.21.2",
-    "ibm-watson": "^11.0.0",
+    "ibm-watson": "^8.0.0",
     "microsoft-cognitiveservices-speech-sdk": "1.38.0",
-    "openai": "^4.98.0",
-    "undici": "^7.5.0"
+    "openai": "^4.25.0",
+    "undici": "^6.4.0"
   },
   "devDependencies": {
     "config": "^3.3.11",

test/synth.js

@@ -5,7 +5,7 @@ const fs = require('fs');
 const {makeSynthKey} = require('../lib/utils');
 const logger = require('pino')();
 const bent = require('bent');
-const getJSON = bent('json');
+const getJSON = bent('json')
 
 process.on('unhandledRejection', (reason, p) => {
   console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
@@ -185,7 +185,6 @@ test('AWS speech synth tests', async(t) => {
       language: 'en-US',
       voice: 'Joey',
       text: 'This is a test.  This is only a test',
-      renderForCaching: true,
     });
     t.ok(!opts.servedFromCache, `successfully synthesized aws audio to ${opts.filePath}`);
 
@@ -199,7 +198,6 @@ test('AWS speech synth tests', async(t) => {
       language: 'en-US',
       voice: 'Joey',
       text: 'This is a test.  This is only a test',
-      renderForCaching: true,
     });
     t.ok(opts.servedFromCache, `successfully retrieved aws audio from cache ${opts.filePath}`);
   } catch (err) {
@@ -554,7 +552,6 @@ test('Custom Vendor speech synth tests', async(t) => {
       language: 'en-US',
       voice: 'English-US.Female-1',
       text: 'This is a test.  This is only a test',
-      renderForCaching: true
     });
     t.ok(!opts.servedFromCache, `successfully synthesized custom vendor audio to ${opts.filePath}`);
     t.ok(opts.filePath.endsWith('wav'), 'audio is cached as wav file');
@@ -576,7 +573,6 @@ test('Custom Vendor speech synth tests', async(t) => {
       language: 'en-US',
       voice: 'English-US.Female-1',
       text: 'This is a test.  This is only a test',
-      renderForCaching: true
     });
     t.ok(opts.servedFromCache, `successfully get custom vendor cached audio to ${opts.filePath}`);
     t.ok(opts.filePath.endsWith('wav'), 'audio is cached as wav file');
@@ -591,7 +587,6 @@ test('Custom Vendor speech synth tests', async(t) => {
       language: 'en-US',
       voice: 'English-US.Female-1',
       text: '<speak>This is a test.  This is only a test</speak>',
-      renderForCaching: true
     });
     t.ok(!opts.servedFromCache, `successfully synthesized Custom Vendor audio to ${opts.filePath}`);
     obj = await getJSON(`http://127.0.0.1:3100/lastRequest/somethingnew2`);
@@ -725,66 +720,7 @@ test('Cartesia speech synth tests', async(t) => {
   client.quit();
 });
 
-test('inworld speech synth', async(t) => {
-  const fn = require('..');
-  const {synthAudio, client} = fn(opts, logger);
-
-  if (!process.env.INWORLD_API_KEY) {
-    t.pass('skipping inworld speech synth tests since INWORLD_API_KEY is not provided');
-    return t.end();
-  }
-  const text = 'Hi there and welcome to jambones!';
-  try {
-    const opts = await synthAudio(stats, {
-      vendor: 'inworld',
-      credentials: {
-        api_key: process.env.INWORLD_API_KEY,
-        model_id: 'inworld-tts-1'
-      },
-      language: 'en',
-      voice: 'Ashley',
-      text,
-      renderForCaching: true
-    });
-    t.ok(!opts.servedFromCache, `successfully synthesized inworld audio to ${opts.filePath}`);
-
-  } catch (err) {
-    console.error(JSON.stringify(err));
-    t.end(err);
-  }
-  client.quit();
-});
-
-test('resemble speech synth', async(t) => {
-  const fn = require('..');
-  const {synthAudio, client} = fn(opts, logger);
-
-  if (!process.env.RESEMBLE_API_KEY) {
-    t.pass('skipping resemble speech synth tests since RESEMBLE_API_KEY is not provided');
-    return t.end();
-  }
-  const text = '<speak prompt="Speak in an excited, upbeat tone">Hello from Resemble!</speak>';
-  try {
-    const opts = await synthAudio(stats, {
-      vendor: 'resemble',
-      credentials: {
-        api_key: process.env.RESEMBLE_API_KEY,
-      },
-      language: 'en',
-      voice: '3f5fb9f1',
-      text,
-      renderForCaching: true
-    });
-    t.ok(!opts.servedFromCache, `successfully synthesized resemble audio to ${opts.filePath}`);
-
-  } catch (err) {
-    console.error(JSON.stringify(err));
-    t.end(err);
-  }
-  client.quit();
-});
-
-test('rimelabs speech synth tests mist', async(t) => {
+test('rimelabs speech synth tests', async(t) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);
 
@@ -794,7 +730,7 @@ test('rimelabs speech synth tests mist', async(t) => {
   }
   const text = 'Hi there and welcome to jambones!';
   try {
-    const opts = await synthAudio(stats, {
+    let opts = await synthAudio(stats, {
       vendor: 'rimelabs',
       credentials: {
         api_key: process.env.RIMELABS_API_KEY,
@@ -804,7 +740,7 @@ test('rimelabs speech synth tests mist', async(t) => {
           reduceLatency: false
         })
       },
-      language: 'eng',
+      language: 'en-US',
       voice: 'amber',
       text,
       renderForCaching: true
@@ -818,40 +754,6 @@ test('rimelabs speech synth tests mist', async(t) => {
   client.quit();
 });
 
-test('rimelabs speech synth tests mistv2', async(t) => {
-  const fn = require('..');
-  const {synthAudio, client} = fn(opts, logger);
-
-  if (!process.env.RIMELABS_API_KEY) {
-    t.pass('skipping rimelabs speech synth tests since RIMELABS_API_KEY is not provided');
-    return t.end();
-  }
-  const text = 'Hi there and welcome to jambones!';
-  try {
-    const opts = await synthAudio(stats, {
-      vendor: 'rimelabs',
-      credentials: {
-        api_key: process.env.RIMELABS_API_KEY,
-        model_id: 'mistv2',
-        options: JSON.stringify({
-          speedAlpha: 1.0,
-          reduceLatency: false
-        })
-      },
-      language: 'spa',
-      voice: 'pablo',
-      text,
-      renderForCaching: true
-    });
-    t.ok(!opts.servedFromCache, `successfully synthesized rimelabs mistv2 audio to ${opts.filePath}`);
-
-  } catch (err) {
-    console.error(JSON.stringify(err));
-    t.end(err);
-  }
-  client.quit();
-});
-
 test('whisper speech synth tests', async(t) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);
@@ -951,12 +853,11 @@ test('TTS Cache tests', async(t) => {
     // save some random tts keys to cache
     const minRecords = 8;
     for (const i in Array(minRecords).fill(0)) {
-      await client.set(makeSynthKey({vendor: i, language: i, voice: i, engine: i, model: i, text: i,
-        instructions: i}), i);
+      await client.set(makeSynthKey({vendor: i, language: i, voice: i, engine: i, text: i}), i);
     }
     const count = await getTtsSize();
     t.ok(count >= minRecords, 'getTtsSize worked.');
-
+    
     const {purgedCount} = await purgeTtsCache();
     t.ok(purgedCount >= minRecords, `successfully purged at least ${minRecords} tts records from cache`);
 
@@ -971,7 +872,7 @@ test('TTS Cache tests', async(t) => {
   try {
     // save some random tts keys to cache
     for (const i in Array(10).fill(0)) {
-      await client.set(makeSynthKey({vendor: i, language: i, voice: i, engine: i, text: i, instructions: i}), i);
+      await client.set(makeSynthKey({vendor: i, language: i, voice: i, engine: i, text: i}), i);
     }
     // save a specific key to tts cache
     const opts = {vendor: 'aws', language: 'en-US', voice: 'MALE', engine: 'Engine', text: 'Hello World!'};
@@ -987,15 +888,24 @@ test('TTS Cache tests', async(t) => {
       language: 'non-existing',
       voice: 'non-existing',
     });
-    t.ok(purgedCountWhenErrored === 0, 'purged no records when specified key was not found');
-    t.ok(error, 'error returned when specified key was not found');
+    t.ok(purgedCountWhenErrored === 0, `purged no records when specified key was not found`);
+    t.ok(error, `error returned when specified key was not found`);
 
     // make sure other tts keys are still there
-    const cached = await client.keys('tts:*');
-    t.ok(cached.length >= 1, 'successfully kept all non-specified tts records in cache');
+    const cached = await client.keys('tts:*')
+    t.ok(cached.length >= 1, `successfully kept all non-specified tts records in cache`);
 
-    process.env.VG_TRIM_TTS_SILENCE = 'true';
+    // retrieve keys from cache and check the key contains the file extension    
+    let key = cached[0];
+    t.ok(key.includes('mp3'), `tts cache extension shoult be part of the key and equal mp3`);
+
+    process.env.VG_TRIM_TTS_SILENCE = 'true';    
     await client.set(makeSynthKey({ vendor: 'azure' }), 'value');
+
+    const r8Keys = await client.keys('tts:r8*');
+    key = r8Keys[0];
+    t.ok(key.includes('r8'), `tts cache extension shoult be part of the key and equal r8`);
+
   } catch (err) {
     console.error(JSON.stringify(err));
     t.end(err);
@@ -1009,10 +919,10 @@ test('TTS Cache tests', async(t) => {
     const account_sid = "12412512_cabc_5aff"
     const account_sid2 = "22412512_cabc_5aff"
     for (const i in Array(minRecords).fill(0)) {
-      await client.set(makeSynthKey({account_sid, vendor: i, language: i, voice: i, engine: i, text: i, instructions: i}), i);
+      await client.set(makeSynthKey({account_sid, vendor: i, language: i, voice: i, engine: i, text: i}), i);
     }
     for (const i in Array(minRecords).fill(0)) {
-      await client.set(makeSynthKey({account_sid: account_sid2, vendor: i, language: i, voice: i, engine: i, text: i, instructions: i}), i);
+      await client.set(makeSynthKey({account_sid: account_sid2, vendor: i, language: i, voice: i, engine: i, text: i}), i);
     }
     const {purgedCount} = await purgeTtsCache({account_sid});
     t.equal(purgedCount, minRecords, `successfully purged at least ${minRecords} tts records from cache for account_sid:${account_sid}`);