0.1.22

Fix custom tts vendor cached file can not be played

lib/synth-audio.js

@@ -20,7 +20,8 @@ const {
   createKryptonClient,
   createRivaClient,
   noopLogger,
-  makeFilePath
+  makeFilePath,
+  makePlayhtKey
 } = require('./utils');
 const getNuanceAccessToken = require('./get-nuance-access-token');
 const getVerbioAccessToken = require('./get-verbio-token');
@@ -169,7 +170,9 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     renderForCaching
   });
   let filePath;
-  filePath = makeFilePath({vendor, key, salt, renderForCaching});
+  // used only for custom vendor
+  let fileExtension;
+  filePath = makeFilePath({vendor, voice, key, salt, renderForCaching});
   debug(`synth key is ${key}`);
   let cached;
   if (!disableTtsCache) {
@@ -191,7 +194,7 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
       cached = await client.get(preCachekey);
       if (cached) {
         // Precache audio is available update filpath with precache file extension.
-        filePath = makeFilePath({vendor, key, salt, renderForCaching: true});
+        filePath = makeFilePath({vendor, voice, key, salt, renderForCaching: true});
       }
     }
   }
@@ -200,7 +203,17 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     debug('result WAS found in cache');
     servedFromCache = true;
     stats.increment('tts.cache.requests', ['found:yes']);
-    audioBuffer = Buffer.from(cached, 'base64');
+    if (vendor.startsWith('custom')) {
+      // custom vendors support multiple mime types such as: mp3, wav, r8, r16 ...etc,
+      // mime type/file extension is available when http response has header Content-type.
+      // In cache, file extension is store together with audiBuffer in a json.
+      // Normal cache audio will be base64 string
+      const payload = JSON.parse(cached);
+      filePath = filePath.replace(/\.[^\.]*$/g, payload.fileExtension);
+      audioBuffer = Buffer.from(payload.audioBuffer, 'base64');
+    } else {
+      audioBuffer = Buffer.from(cached, 'base64');
+    }
     client.expire(key, EXPIRES).catch((err) => logger.info(err, 'Error setting expires'));
   }
   if (!cached) {
@@ -244,7 +257,7 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
         });
         break;
       case 'playht':
-        audioBuffer = await synthPlayHT(logger, {
+        audioBuffer = await synthPlayHT(client, logger, {
           credentials, options, stats, language, voice, text, renderForCaching, disableTtsStreaming, filePath
         });
         break;
@@ -267,7 +280,7 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
           renderForCaching, disableTtsStreaming});
         break;
       case vendor.startsWith('custom') ? vendor : 'cant_match_value':
-        ({ audioBuffer, filePath } = await synthCustomVendor(logger,
+        ({ audioBuffer, filePath, fileExtension } = await synthCustomVendor(logger,
           {credentials, stats, language, voice, text, filePath}));
         break;
       default:
@@ -281,7 +294,14 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     debug(`tts rtt time for ${text.length} chars on ${vendorLabel}: ${rtt}`);
     logger.info(`tts rtt time for ${text.length} chars on ${vendorLabel}: ${rtt}`);
 
-    client.setex(key, EXPIRES, audioBuffer.toString('base64'))
+    const base64Audio = audioBuffer.toString('base64');
+    const cacheContent = vendor.startsWith('custom') ?
+      JSON.stringify({
+        audioBuffer: base64Audio,
+        fileExtension
+      }) : base64Audio;
+
+    client.setex(key, EXPIRES, cacheContent)
       .catch((err) => logger.error(err, `error calling setex on key ${key}`));
   }
 
@@ -352,6 +372,44 @@ const synthPolly = async(createHash, retrieveHash, logger,
 
 const synthGoogle = async(logger, {credentials, stats, language, voice, gender, text}) => {
   const client = new ttsGoogle.TextToSpeechClient(credentials);
+  // If google custom voice cloning is used.
+  // At this time 31 Oct 2024, google node sdk has not support voice cloning yet.
+  if (typeof voice === 'object' && voice.voice_cloning_key) {
+    try {
+      const accessToken = await client.auth.getAccessToken();
+      const projectId = await client.getProjectId();
+
+      const post = bent('https://texttospeech.googleapis.com', 'POST', 'json', {
+        'Authorization': `Bearer ${accessToken}`,
+        'x-goog-user-project': projectId,
+        'Content-Type': 'application/json; charset=utf-8'
+      });
+
+      const payload = {
+        input: {
+          text
+        },
+        voice: {
+          language_code: language,
+          voice_clone: {
+            voice_cloning_key: voice.voice_cloning_key
+          }
+        },
+        audioConfig: {
+          // Cloning voice at this time still in v1 beta version, and it support LINEAR16 in Wav format, 24.000Hz
+          audioEncoding: 'LINEAR16',
+          sample_rate_hertz: 24000
+        }
+      };
+
+      const wav = await post('/v1beta1/text:synthesize', payload);
+      return Buffer.from(wav.audioContent, 'base64');
+    } catch (err) {
+      logger.info({err: await err.text()}, 'synthGoogle returned error');
+      throw err;
+    }
+  }
+
   const opts = {
     voice: {
       ...(typeof voice === 'string' && {name: voice}),
@@ -690,9 +748,11 @@ const synthCustomVendor = async(logger, {credentials, stats, language, voice, te
     const regex = /\.[^\.]*$/g;
     const mime = response.headers['content-type'];
     const buffer = await response.arrayBuffer();
+    const fileExtension = getFileExtFromMime(mime);
     return {
       audioBuffer: buffer,
-      filePath: filePath.replace(regex, getFileExtFromMime(mime))
+      filePath: filePath.replace(regex, fileExtension),
+      fileExtension
     };
   } catch (err) {
     logger.info({err}, `Vendor ${vendor} returned error`);
@@ -755,12 +815,40 @@ const synthElevenlabs = async(logger, {
   }
 };
 
-const synthPlayHT = async(logger, {
+const synthPlayHT = async(client, logger, {
-  credentials, options, stats, voice, text, renderForCaching, disableTtsStreaming
+  credentials, options, stats, voice, language, text, renderForCaching, disableTtsStreaming
 }) => {
   const {api_key, user_id, voice_engine, options: credOpts} = credentials;
   const opts = !!options && Object.keys(options).length !== 0 ? options : JSON.parse(credOpts || '{}');
 
+  let synthesizeUrl = 'https://api.play.ht/api/v2/tts/stream';
+
+  // If model is play3.0, the synthesizeUrl is got from authentication endpoint
+  if (voice_engine === 'Play3.0') {
+    try {
+      const post = bent('https://api.play.ht', 'POST', 'json', 201, {
+        'AUTHORIZATION': api_key,
+        'X-USER-ID': user_id,
+        'Accept': 'application/json'
+      });
+      const key = makePlayhtKey(api_key);
+      const url = await client.get(key);
+      if (!url) {
+        const {inference_address, expires_at_ms} = await post('/api/v3/auth');
+        synthesizeUrl = inference_address;
+        const expiry =  Math.floor((expires_at_ms - Date.now()) / 1000 - 30);
+        await client.set(key, inference_address, 'EX', expiry);
+      } else {
+        // Use cached URL
+        synthesizeUrl = url;
+      }
+    } catch (err) {
+      logger.info({err}, 'synth PlayHT returned error for authentication version 3.0');
+      stats.increment('tts.count', ['vendor:playht', 'accepted:no']);
+      throw err;
+    }
+  }
+
   /* default to using the streaming interface, unless disabled by env var OR we want just a cache file */
   if (!JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
     let params = '';
@@ -769,7 +857,9 @@ const synthPlayHT = async(logger, {
     params += ',vendor=playht';
     params += `,voice=${voice}`;
     params += `,voice_engine=${voice_engine}`;
+    params += `,synthesize_url=${synthesizeUrl}`;
     params += ',write_cache_file=1';
+    params += `,language=${language}`;
     if (opts.quality) params += `,quality=${opts.quality}`;
     if (opts.speed) params += `,speed=${opts.speed}`;
     if (opts.seed) params += `,style=${opts.seed}`;
@@ -778,6 +868,8 @@ const synthPlayHT = async(logger, {
     if (opts.voice_guidance) params += `,voice_guidance=${opts.voice_guidance}`;
     if (opts.style_guidance) params += `,style_guidance=${opts.style_guidance}`;
     if (opts.text_guidance) params += `,text_guidance=${opts.text_guidance}`;
+    if (opts.top_p) params += `,top_p=${opts.top_p}`;
+    if (opts.repetition_penalty) params += `,repetition_penalty=${opts.repetition_penalty}`;
     params += '}';
 
     return {
@@ -788,14 +880,18 @@ const synthPlayHT = async(logger, {
   }
 
   try {
-    const post = bent('https://api.play.ht', 'POST', 'buffer', {
+    const post = bent('POST', 'buffer', {
-      'AUTHORIZATION': api_key,
+      ...(voice_engine !== 'Play3.0' && {
-      'X-USER-ID': user_id,
+        'AUTHORIZATION': api_key,
+        'X-USER-ID': user_id,
+      }),
       'Accept': 'audio/mpeg',
       'Content-Type': 'application/json'
     });
-    const mp3 = await post('/api/v2/tts/stream', {
+
+    const mp3 = await post(synthesizeUrl, {
       text,
+      ...(voice_engine === 'Play3.0' && { language }),
       voice,
       voice_engine,
       output_format: 'mp3',

lib/utils.js

@@ -23,19 +23,20 @@ function makeSynthKey({
   hash.update(`${language}:${vendor}:${voice}:${engine}:${text}`);
   const hexHashKey = hash.digest('hex');
   const accountKey = account_sid ? `:${account_sid}` : '';
-  const namespace = vendor.startsWith('custom') ? vendor : getFileExtension({vendor, renderForCaching});
+  const namespace = vendor.startsWith('custom') ? vendor : getFileExtension({vendor, voice, renderForCaching});
   const key = `tts${accountKey}:${namespace}:${hexHashKey}`;
   return key;
 }
 
-function makeFilePath({vendor, key, salt = '', renderForCaching = false}) {
+function makeFilePath({vendor, voice, key, salt = '', renderForCaching = false}) {
-  const extension = getFileExtension({vendor, renderForCaching});
+  const extension = getFileExtension({vendor, renderForCaching, voice});
   return `${TMP_FOLDER}/${key.replace('tts:', `tts-${salt}`)}.${extension}`;
 }
 
-function getFileExtension({vendor, renderForCaching = false}) {
+function getFileExtension({vendor, voice, renderForCaching = false}) {
   const mp3Extension = 'mp3';
   const r8Extension = 'r8';
+  const wavExtension = 'wav';
 
   switch (vendor) {
     case 'azure':
@@ -47,7 +48,7 @@ function getFileExtension({vendor, renderForCaching = false}) {
       }
     case 'deepgram':
     case 'elevenlabs':
-    case 'rimlabs':
+    case 'rimelabs':
     case 'playht':
       if (renderForCaching || JAMBONES_DISABLE_TTS_STREAMING) {
         return mp3Extension;
@@ -58,6 +59,13 @@ function getFileExtension({vendor, renderForCaching = false}) {
     case 'nvidia':
     case 'verbio':
       return r8Extension;
+    case 'google':
+      // google voice cloning just support wav.
+      if (typeof voice === 'object' && voice.voice_cloning_key) {
+        return wavExtension;
+      } else {
+        return mp3Extension;
+      }
     default:
       // If vendor is custom
       if (vendor.startsWith('custom')) {
@@ -98,6 +106,11 @@ function makeAwsKey(awsAccessKeyId) {
   return `aws:${hash.digest('hex')}`;
 }
 
+function makePlayhtKey(apiKey) {
+  const hash = crypto.createHash('sha1');
+  hash.update(apiKey);
+  return `playht:${hash.digest('hex')}`;
+}
 function makeVerbioKey(client_id) {
   const hash = crypto.createHash('sha1');
   hash.update(client_id);
@@ -171,6 +184,7 @@ module.exports = {
   makeSynthKey,
   makeNuanceKey,
   makeIbmKey,
+  makePlayhtKey,
   makeAwsKey,
   makeVerbioKey,
   getNuanceAccessToken,

package-lock.json

@@ -1,17 +1,17 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.1.16",
+  "version": "0.1.22",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@jambonz/speech-utils",
-      "version": "0.1.16",
+      "version": "0.1.22",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-polly": "^3.496.0",
         "@aws-sdk/client-sts": "^3.496.0",
-        "@google-cloud/text-to-speech": "^5.0.2",
+        "@google-cloud/text-to-speech": "^5.5.0",
         "@grpc/grpc-js": "^1.9.14",
         "@jambonz/realtimedb-helpers": "^0.8.7",
         "bent": "^7.3.12",
@@ -1230,9 +1230,10 @@
       }
     },
     "node_modules/@google-cloud/text-to-speech": {
-      "version": "5.0.2",
+      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-5.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-5.5.0.tgz",
-      "integrity": "sha512-Q11Ddh9eHKSDA3E/KSqMITgVprXb0XgIKuJP9F5ScJ1T9h+DNrbgIU7shd0QOlPqb8ruQRiTOqL08+Mq5R89Ow==",
+      "integrity": "sha512-Cw/UK2Y3l31Vsuozu8cxsmVS/09fShimes0tRLgDbOY2ZMG1Dckb6Zf/Q3Nxg4X0feFep44pvwNmyHKrOnl9SQ==",
+      "license": "Apache-2.0",
       "dependencies": {
         "google-gax": "^4.0.3"
       },
@@ -8338,9 +8339,9 @@
       "dev": true
     },
     "@google-cloud/text-to-speech": {
-      "version": "5.0.2",
+      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-5.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-5.5.0.tgz",
-      "integrity": "sha512-Q11Ddh9eHKSDA3E/KSqMITgVprXb0XgIKuJP9F5ScJ1T9h+DNrbgIU7shd0QOlPqb8ruQRiTOqL08+Mq5R89Ow==",
+      "integrity": "sha512-Cw/UK2Y3l31Vsuozu8cxsmVS/09fShimes0tRLgDbOY2ZMG1Dckb6Zf/Q3Nxg4X0feFep44pvwNmyHKrOnl9SQ==",
       "requires": {
         "google-gax": "^4.0.3"
       }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.1.16",
+  "version": "0.1.22",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",
@@ -28,7 +28,7 @@
   "dependencies": {
     "@aws-sdk/client-polly": "^3.496.0",
     "@aws-sdk/client-sts": "^3.496.0",
-    "@google-cloud/text-to-speech": "^5.0.2",
+    "@google-cloud/text-to-speech": "^5.5.0",
     "@grpc/grpc-js": "^1.9.14",
     "@jambonz/realtimedb-helpers": "^0.8.7",
     "bent": "^7.3.12",

test/synth.js

@@ -91,14 +91,17 @@ test('Google speech Custom voice synth tests', async(t) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);
 
-  if (!process.env.GCP_CUSTOM_VOICE_FILE && !process.env.GCP_CUSTOM_VOICE_JSON_KEY || !process.env.GCP_CUSTOM_VOICE_MODEL) {
+  if (!process.env.GCP_CUSTOM_VOICE_FILE &&
-    t.pass('skipping google speech synth tests since neither GCP_CUSTOM_VOICE_FILE nor GCP_CUSTOM_VOICE_JSON_KEY provided, GCP_CUSTOM_VOICE_MODEL is not provided');
+    !process.env.GCP_CUSTOM_VOICE_JSON_KEY ||
+    !process.env.GCP_CUSTOM_VOICE_MODEL) {
+    t.pass(`skipping google speech synth tests since neither 
+GCP_CUSTOM_VOICE_FILE nor GCP_CUSTOM_VOICE_JSON_KEY provided, GCP_CUSTOM_VOICE_MODEL is not provided`);
     return t.end();
   }
   try {
     const str = process.env.GCP_CUSTOM_VOICE_JSON_KEY || fs.readFileSync(process.env.GCP_CUSTOM_VOICE_FILE);
     const creds = JSON.parse(str);
-    let opts = await synthAudio(stats, {
+    const opts = await synthAudio(stats, {
       vendor: 'google',
       credentials: {
         credentials: {
@@ -109,7 +112,7 @@ test('Google speech Custom voice synth tests', async(t) => {
       language: 'en-AU',
       text: 'This is a test.  This is only a test',
       voice: {
-        reportedUsage:"REALTIME",
+        reportedUsage: 'REALTIME',
         model: process.env.GCP_CUSTOM_VOICE_MODEL
       }
     });
@@ -121,6 +124,48 @@ test('Google speech Custom voice synth tests', async(t) => {
   client.quit();
 });
 
+test('Google speech voice cloning synth tests', async(t) => {
+  const fn = require('..');
+  const {synthAudio, client} = fn(opts, logger);
+
+  if (!process.env.GCP_CUSTOM_VOICE_FILE &&
+    !process.env.GCP_CUSTOM_VOICE_JSON_KEY ||
+    !process.env.GCP_VOICE_CLONING_FILE &&
+    !process.env.GCP_VOICE_CLONING_JSON_KEY) {
+    t.pass(`skipping google speech synth tests since neither 
+GCP_CUSTOM_VOICE_FILE nor GCP_CUSTOM_VOICE_JSON_KEY provided,
+GCP_VOICE_CLONING_FILE nor GCP_VOICE_CLONING_JSON_KEY is not provided`);
+    return t.end();
+  }
+  try {
+    const googleKey = process.env.GCP_CUSTOM_VOICE_JSON_KEY ||
+      fs.readFileSync(process.env.GCP_CUSTOM_VOICE_FILE);
+    const voice_cloning_key = process.env.GCP_VOICE_CLONING_JSON_KEY ||
+      fs.readFileSync(process.env.GCP_VOICE_CLONING_FILE).toString();
+    const creds = JSON.parse(googleKey);
+    const opts = await synthAudio(stats, {
+      vendor: 'google',
+      credentials: {
+        credentials: {
+          client_email: creds.client_email,
+          private_key: creds.private_key,
+          project_id: creds.project_id
+        },
+      },
+      language: 'en-US',
+      text: 'This is a test. This is only a test. This is a test. This is only a test. This is a test. This is only a test',
+      voice: {
+        voice_cloning_key
+      }
+    });
+    t.ok(!opts.servedFromCache, `successfully synthesized google voice cloning audio to ${opts.filePath}`);
+  } catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+  client.quit();
+});
+
 test('AWS speech synth tests', async(t) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);
@@ -509,6 +554,7 @@ test('Custom Vendor speech synth tests', async(t) => {
       text: 'This is a test.  This is only a test',
     });
     t.ok(!opts.servedFromCache, `successfully synthesized custom vendor audio to ${opts.filePath}`);
+    t.ok(opts.filePath.endsWith('wav'), 'audio is cached as wav file');
     let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/somethingnew`);
     t.ok(obj.headers.Authorization == 'Bearer some_jwt_token', 'Custom Vendor Authentication Header is correct');
     t.ok(obj.body.language == 'en-US', 'Custom Vendor Language is correct');
@@ -516,6 +562,21 @@ test('Custom Vendor speech synth tests', async(t) => {
     t.ok(obj.body.type == 'text', 'Custom Vendor type is correct');
     t.ok(obj.body.text == 'This is a test.  This is only a test', 'Custom Vendor text is correct');
 
+    // Checking if cache is stored with wav format
+    opts = await synthAudio(stats, {
+      vendor: 'custom:somethingnew',
+      credentials: {
+        use_for_tts: 1,
+        custom_tts_url: "http://127.0.0.1:3100/somethingnew",
+        auth_token: 'some_jwt_token'
+      },
+      language: 'en-US',
+      voice: 'English-US.Female-1',
+      text: 'This is a test.  This is only a test',
+    });
+    t.ok(opts.servedFromCache, `successfully get custom vendor cached audio to ${opts.filePath}`);
+    t.ok(opts.filePath.endsWith('wav'), 'audio is cached as wav file');
+
     opts = await synthAudio(stats, {
       vendor: 'custom:somethingnew2',
       credentials: {
@@ -574,9 +635,9 @@ test('Elevenlabs speech synth tests', async(t) => {
     t.end(err);
   }
   client.quit();
-})
+});
 
-test('PlayHT speech synth tests', async(t) => {
+const testPlayHT = async(t, voice_engine) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);
 
@@ -584,26 +645,26 @@ test('PlayHT speech synth tests', async(t) => {
     t.pass('skipping PlayHT speech synth tests since PLAYHT_API_KEY or PLAYHT_USER_ID is/are not provided');
     return t.end();
   }
-  const text = 'Hi there and welcome to jambones!';
+  const text = 'Hi there and welcome to jambones! ' + Date.now();
   try {
-    let opts = await synthAudio(stats, {
+    const opts = await synthAudio(stats, {
       vendor: 'playht',
       credentials: {
         api_key: process.env.PLAYHT_API_KEY,
         user_id: process.env.PLAYHT_USER_ID,
-        voice_engine: 'PlayHT2.0-turbo',
+        voice_engine,
         options: JSON.stringify({
-          quality: "medium",
+          quality: 'medium',
           speed: 1,
           seed: 1,
           temperature: 1,
-          emotion: "female_happy",
+          emotion: 'female_happy',
           voice_guidance: 3,
           style_guidance: 20,
           text_guidance: 1,
         })
       },
-      language: 'en-US',
+      language: 'english',
       voice: 's3://voice-cloning-zero-shot/d9ff78ba-d016-47f6-b0ef-dd630f59414e/female-cs/manifest.json',
       text,
       renderForCaching: true
@@ -615,6 +676,14 @@ test('PlayHT speech synth tests', async(t) => {
     t.end(err);
   }
   client.quit();
+};
+
+test('PlayHT speech synth tests', async(t) => {
+  await testPlayHT(t, 'PlayHT2.0-turbo');
+});
+
+test('PlayHT3.0 speech synth tests', async(t) => {
+  await testPlayHT(t, 'Play3.0');
 });
 
 test('rimelabs speech synth tests', async(t) => {