0.2.9

support openai whisper instructions

lib/add-file-to-cache.js

@@ -25,7 +25,7 @@ function getExtensionAndSampleRate(path) {
 }
 
 async function addFileToCache(client, logger, path,
-  {account_sid, vendor, language, voice, deploymentId, engine, model, text}) {
+  {account_sid, vendor, language, voice, deploymentId, engine, model, text, instructions}) {
   let key;
   logger = logger || noopLogger;
 
@@ -38,6 +38,7 @@ async function addFileToCache(client, logger, path,
       engine,
       model,
       text,
+      instructions
     });
     const [extension, sampleRate] = getExtensionAndSampleRate(path);
     const audioBuffer = await fs.readFile(path);

lib/purge-tts-cache.js

@@ -12,7 +12,7 @@ const debug = require('debug')('jambonz:realtimedb-helpers');
  * @returns {object} result - {error, purgedCount}
  */
 async function purgeTtsCache(client, logger, {all, account_sid, vendor,
-  language, voice, deploymentId, engine, model, text} = {all: true}) {
+  language, voice, deploymentId, engine, model, text, instructions} = {all: true}) {
   logger = logger || noopLogger;
 
   let purgedCount = 0, error;
@@ -35,6 +35,7 @@ async function purgeTtsCache(client, logger, {all, account_sid, vendor,
         engine,
         model,
         text,
+        instructions
       });
       purgedCount = await client.del(key);
       if (purgedCount === 0) error = 'Specified item not found';

lib/synth-audio.js

@@ -89,7 +89,7 @@ const trimTrailingSilence = (buffer) => {
  */
 async function synthAudio(client, createHash, retrieveHash, logger, stats, { account_sid,
   vendor, language, voice, gender, text, engine, salt, model, credentials, deploymentId,
-  disableTtsCache, renderForCaching = false, disableTtsStreaming, options
+  disableTtsCache, renderForCaching = false, disableTtsStreaming, options, instructions
 }) {
   let audioData;
   let servedFromCache = false;
@@ -171,7 +171,8 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     engine,
     // model or model_id is used to identify the tts cache.
     model: model || credentials.model_id,
-    text
+    text,
+    instructions
   });
 
   debug(`synth key is ${key}`);
@@ -215,7 +216,8 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
         audioData = await synthNuance(client, logger, {credentials, stats, voice, model, text});
         break;
       case 'nvidia':
-        audioData = await synthNvidia(client, logger, {credentials, stats, language, voice, model, text});
+        audioData = await synthNvidia(client, logger, {credentials, stats, language, voice, model, text,
+          renderForCaching, disableTtsStreaming});
         break;
       case 'ibm':
         audioData = await synthIbm(logger, {credentials, stats, voice, text});
@@ -241,7 +243,7 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
         break;
       case 'whisper':
         audioData = await synthWhisper(logger, {
-          credentials, stats, voice, text, renderForCaching, disableTtsStreaming});
+          credentials, stats, voice, text, instructions, renderForCaching, disableTtsStreaming});
         break;
       case 'verbio':
         audioData = await synthVerbio(client, logger, {
@@ -708,8 +710,24 @@ const synthNuance = async(client, logger, {credentials, stats, voice, model, tex
   });
 };
 
-const synthNvidia = async(client, logger, {credentials, stats, language,  voice, model, text}) => {
+const synthNvidia = async(client, logger, {
+  credentials, stats, language,  voice, model, text, renderForCaching, disableTtsStreaming
+}) => {
   const {riva_server_uri} = credentials;
+  if (!JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
+    let params = '';
+    params += `{riva_server_uri=${riva_server_uri}`;
+    params += `,voice=${voice}`;
+    params += `,language=${language}`;
+    params += ',write_cache_file=1';
+    params += '}';
+
+    return {
+      filePath: `say:${params}${text.replace(/\n/g, ' ')}`,
+      servedFromCache: false,
+      rtt: 0
+    };
+  }
   let rivaClient, request;
   const sampleRate = 8000;
   try {
@@ -789,9 +807,13 @@ const synthElevenlabs = async(logger, {
     if (opts.voice_settings?.similarity_boost) params += `,similarity_boost=${opts.voice_settings.similarity_boost}`;
     if (opts.voice_settings?.stability) params += `,stability=${opts.voice_settings.stability}`;
     if (opts.voice_settings?.style) params += `,style=${opts.voice_settings.style}`;
+    if (opts.voice_settings?.speed !== null && opts.voice_settings?.speed !== undefined)
+      params += `,speed=${opts.voice_settings.speed}`;
     if (opts.voice_settings?.use_speaker_boost === false) params += ',use_speaker_boost=false';
     if (opts.previous_text) params += `,previous_text=${opts.previous_text}`;
     if (opts.next_text) params += `,next_text=${opts.next_text}`;
+    if (opts.pronunciation_dictionary_locators && Array.isArray(opts.pronunciation_dictionary_locators))
+      params += `,pronunciation_dictionary_locators=${JSON.stringify(opts.pronunciation_dictionary_locators)}`;
     params += '}';
 
     return {
@@ -942,6 +964,11 @@ const synthRimelabs = async(logger, {
     params += ',write_cache_file=1';
     if (opts.speedAlpha) params += `,speed_alpha=${opts.speedAlpha}`;
     if (opts.reduceLatency) params += `,reduce_latency=${opts.reduceLatency}`;
+    // Arcana model parameters
+    if (opts.temperature) params += `,temperature=${opts.temperature}`;
+    if (opts.repetition_penalty) params += `,repetition_penalty=${opts.repetition_penalty}`;
+    if (opts.top_p) params += `,top_p=${opts.top_p}`;
+    if (opts.max_tokens) params += `,max_tokens=${opts.max_tokens}`;
     params += '}';
 
     return {
@@ -1022,7 +1049,8 @@ const synthVerbio = async(client, logger, {credentials, stats, voice, text, rend
   }
 };
 
-const synthWhisper = async(logger, {credentials, stats, voice, text, renderForCaching, disableTtsStreaming}) => {
+const synthWhisper = async(logger, {credentials, stats, voice, text, instructions,
+  renderForCaching, disableTtsStreaming}) => {
   const {api_key, model_id, baseURL, timeout, speed} = credentials;
   /* if the env is set to stream then bag out, unless we are specifically rendering to generate a cache file */
   if (!JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
@@ -1033,6 +1061,8 @@ const synthWhisper = async(logger, {credentials, stats, voice, text, renderForCa
     params += `,voice=${voice}`;
     params += ',write_cache_file=1';
     if (speed) params += `,speed=${speed}`;
+    // comma is used to separated parameters in freeswitch tts module
+    if (instructions) params += `,instructions=${instructions.replace(/\n/g, ' ').replace(/,/g, ';')}`;
     params += '}';
 
     return {
@@ -1052,6 +1082,7 @@ const synthWhisper = async(logger, {credentials, stats, voice, text, renderForCa
       model: model_id,
       voice,
       input: text,
+      ...(instructions && {instructions}),
       response_format: 'mp3'
     });
     return {

lib/utils.js

@@ -23,10 +23,11 @@ function makeSynthKey({
   voice,
   engine = '',
   model = '',
-  text
+  text,
+  instructions = '',
 }) {
   const hash = crypto.createHash('sha1');
-  hash.update(`${language}:${vendor}:${voice}:${engine}:${model}:${text}`);
+  hash.update(`${language}:${vendor}:${voice}:${engine}:${model}:${text}:${instructions}`);
   const hexHashKey = hash.digest('hex');
   const accountKey = account_sid ? `:${account_sid}` : '';
   const key = `tts${accountKey}:${hexHashKey}`;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.2.3",
+  "version": "0.2.9",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",
@@ -26,6 +26,7 @@
   },
   "homepage": "https://github.com/jambonz/speech-utils#readme",
   "dependencies": {
+    "23": "^0.0.0",
     "@aws-sdk/client-polly": "^3.496.0",
     "@aws-sdk/client-sts": "^3.496.0",
     "@cartesia/cartesia-js": "^2.1.0",
@@ -38,8 +39,8 @@
     "google-protobuf": "^3.21.2",
     "ibm-watson": "^8.0.0",
     "microsoft-cognitiveservices-speech-sdk": "1.38.0",
-    "openai": "^4.25.0",
-    "undici": "^6.4.0"
+    "openai": "^4.98.0",
+    "undici": "^7.5.0"
   },
   "devDependencies": {
     "config": "^3.3.11",

test/synth.js

@@ -887,7 +887,8 @@ test('TTS Cache tests', async(t) => {
     // save some random tts keys to cache
     const minRecords = 8;
     for (const i in Array(minRecords).fill(0)) {
-      await client.set(makeSynthKey({vendor: i, language: i, voice: i, engine: i, model: i, text: i}), i);
+      await client.set(makeSynthKey({vendor: i, language: i, voice: i, engine: i, model: i, text: i,
+        instructions: i}), i);
     }
     const count = await getTtsSize();
     t.ok(count >= minRecords, 'getTtsSize worked.');
@@ -906,7 +907,7 @@ test('TTS Cache tests', async(t) => {
   try {
     // save some random tts keys to cache
     for (const i in Array(10).fill(0)) {
-      await client.set(makeSynthKey({vendor: i, language: i, voice: i, engine: i, text: i}), i);
+      await client.set(makeSynthKey({vendor: i, language: i, voice: i, engine: i, text: i, instructions: i}), i);
     }
     // save a specific key to tts cache
     const opts = {vendor: 'aws', language: 'en-US', voice: 'MALE', engine: 'Engine', text: 'Hello World!'};
@@ -944,10 +945,10 @@ test('TTS Cache tests', async(t) => {
     const account_sid = "12412512_cabc_5aff"
     const account_sid2 = "22412512_cabc_5aff"
     for (const i in Array(minRecords).fill(0)) {
-      await client.set(makeSynthKey({account_sid, vendor: i, language: i, voice: i, engine: i, text: i}), i);
+      await client.set(makeSynthKey({account_sid, vendor: i, language: i, voice: i, engine: i, text: i, instructions: i}), i);
     }
     for (const i in Array(minRecords).fill(0)) {
-      await client.set(makeSynthKey({account_sid: account_sid2, vendor: i, language: i, voice: i, engine: i, text: i}), i);
+      await client.set(makeSynthKey({account_sid: account_sid2, vendor: i, language: i, voice: i, engine: i, text: i, instructions: i}), i);
     }
     const {purgedCount} = await purgeTtsCache({account_sid});
     t.equal(purgedCount, minRecords, `successfully purged at least ${minRecords} tts records from cache for account_sid:${account_sid}`);