0.1.22

Fix custom tts vendor cached file can not be played

lib/synth-audio.js

@@ -170,7 +170,9 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     renderForCaching
   });
   let filePath;
-  filePath = makeFilePath({vendor, key, salt, renderForCaching});
+  // used only for custom vendor
+  let fileExtension;
+  filePath = makeFilePath({vendor, voice, key, salt, renderForCaching});
   debug(`synth key is ${key}`);
   let cached;
   if (!disableTtsCache) {
@@ -192,7 +194,7 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
       cached = await client.get(preCachekey);
       if (cached) {
         // Precache audio is available update filpath with precache file extension.
-        filePath = makeFilePath({vendor, key, salt, renderForCaching: true});
+        filePath = makeFilePath({vendor, voice, key, salt, renderForCaching: true});
       }
     }
   }
@@ -201,7 +203,17 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     debug('result WAS found in cache');
     servedFromCache = true;
     stats.increment('tts.cache.requests', ['found:yes']);
-    audioBuffer = Buffer.from(cached, 'base64');
+    if (vendor.startsWith('custom')) {
+      // custom vendors support multiple mime types such as: mp3, wav, r8, r16 ...etc,
+      // mime type/file extension is available when http response has header Content-type.
+      // In cache, file extension is store together with audiBuffer in a json.
+      // Normal cache audio will be base64 string
+      const payload = JSON.parse(cached);
+      filePath = filePath.replace(/\.[^\.]*$/g, payload.fileExtension);
+      audioBuffer = Buffer.from(payload.audioBuffer, 'base64');
+    } else {
+      audioBuffer = Buffer.from(cached, 'base64');
+    }
     client.expire(key, EXPIRES).catch((err) => logger.info(err, 'Error setting expires'));
   }
   if (!cached) {
@@ -268,7 +280,7 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
           renderForCaching, disableTtsStreaming});
         break;
       case vendor.startsWith('custom') ? vendor : 'cant_match_value':
-        ({ audioBuffer, filePath } = await synthCustomVendor(logger,
+        ({ audioBuffer, filePath, fileExtension } = await synthCustomVendor(logger,
           {credentials, stats, language, voice, text, filePath}));
         break;
       default:
@@ -282,7 +294,14 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     debug(`tts rtt time for ${text.length} chars on ${vendorLabel}: ${rtt}`);
     logger.info(`tts rtt time for ${text.length} chars on ${vendorLabel}: ${rtt}`);
 
-    client.setex(key, EXPIRES, audioBuffer.toString('base64'))
+    const base64Audio = audioBuffer.toString('base64');
+    const cacheContent = vendor.startsWith('custom') ?
+      JSON.stringify({
+        audioBuffer: base64Audio,
+        fileExtension
+      }) : base64Audio;
+
+    client.setex(key, EXPIRES, cacheContent)
       .catch((err) => logger.error(err, `error calling setex on key ${key}`));
   }
 
@@ -353,6 +372,44 @@ const synthPolly = async(createHash, retrieveHash, logger,
 
 const synthGoogle = async(logger, {credentials, stats, language, voice, gender, text}) => {
   const client = new ttsGoogle.TextToSpeechClient(credentials);
+  // If google custom voice cloning is used.
+  // At this time 31 Oct 2024, google node sdk has not support voice cloning yet.
+  if (typeof voice === 'object' && voice.voice_cloning_key) {
+    try {
+      const accessToken = await client.auth.getAccessToken();
+      const projectId = await client.getProjectId();
+
+      const post = bent('https://texttospeech.googleapis.com', 'POST', 'json', {
+        'Authorization': `Bearer ${accessToken}`,
+        'x-goog-user-project': projectId,
+        'Content-Type': 'application/json; charset=utf-8'
+      });
+
+      const payload = {
+        input: {
+          text
+        },
+        voice: {
+          language_code: language,
+          voice_clone: {
+            voice_cloning_key: voice.voice_cloning_key
+          }
+        },
+        audioConfig: {
+          // Cloning voice at this time still in v1 beta version, and it support LINEAR16 in Wav format, 24.000Hz
+          audioEncoding: 'LINEAR16',
+          sample_rate_hertz: 24000
+        }
+      };
+
+      const wav = await post('/v1beta1/text:synthesize', payload);
+      return Buffer.from(wav.audioContent, 'base64');
+    } catch (err) {
+      logger.info({err: await err.text()}, 'synthGoogle returned error');
+      throw err;
+    }
+  }
+
   const opts = {
     voice: {
       ...(typeof voice === 'string' && {name: voice}),
@@ -691,9 +748,11 @@ const synthCustomVendor = async(logger, {credentials, stats, language, voice, te
     const regex = /\.[^\.]*$/g;
     const mime = response.headers['content-type'];
     const buffer = await response.arrayBuffer();
+    const fileExtension = getFileExtFromMime(mime);
     return {
       audioBuffer: buffer,
-      filePath: filePath.replace(regex, getFileExtFromMime(mime))
+      filePath: filePath.replace(regex, fileExtension),
+      fileExtension
     };
   } catch (err) {
     logger.info({err}, `Vendor ${vendor} returned error`);
@@ -779,6 +838,9 @@ const synthPlayHT = async(client, logger, {
         synthesizeUrl = inference_address;
         const expiry =  Math.floor((expires_at_ms - Date.now()) / 1000 - 30);
         await client.set(key, inference_address, 'EX', expiry);
+      } else {
+        // Use cached URL
+        synthesizeUrl = url;
       }
     } catch (err) {
       logger.info({err}, 'synth PlayHT returned error for authentication version 3.0');
@@ -797,6 +859,7 @@ const synthPlayHT = async(client, logger, {
     params += `,voice_engine=${voice_engine}`;
     params += `,synthesize_url=${synthesizeUrl}`;
     params += ',write_cache_file=1';
+    params += `,language=${language}`;
     if (opts.quality) params += `,quality=${opts.quality}`;
     if (opts.speed) params += `,speed=${opts.speed}`;
     if (opts.seed) params += `,style=${opts.seed}`;

lib/utils.js

@@ -23,19 +23,20 @@ function makeSynthKey({
   hash.update(`${language}:${vendor}:${voice}:${engine}:${text}`);
   const hexHashKey = hash.digest('hex');
   const accountKey = account_sid ? `:${account_sid}` : '';
-  const namespace = vendor.startsWith('custom') ? vendor : getFileExtension({vendor, renderForCaching});
+  const namespace = vendor.startsWith('custom') ? vendor : getFileExtension({vendor, voice, renderForCaching});
   const key = `tts${accountKey}:${namespace}:${hexHashKey}`;
   return key;
 }
 
-function makeFilePath({vendor, key, salt = '', renderForCaching = false}) {
-  const extension = getFileExtension({vendor, renderForCaching});
+function makeFilePath({vendor, voice, key, salt = '', renderForCaching = false}) {
+  const extension = getFileExtension({vendor, renderForCaching, voice});
   return `${TMP_FOLDER}/${key.replace('tts:', `tts-${salt}`)}.${extension}`;
 }
 
-function getFileExtension({vendor, renderForCaching = false}) {
+function getFileExtension({vendor, voice, renderForCaching = false}) {
   const mp3Extension = 'mp3';
   const r8Extension = 'r8';
+  const wavExtension = 'wav';
 
   switch (vendor) {
     case 'azure':
@@ -47,7 +48,7 @@ function getFileExtension({vendor, renderForCaching = false}) {
       }
     case 'deepgram':
     case 'elevenlabs':
-    case 'rimlabs':
+    case 'rimelabs':
     case 'playht':
       if (renderForCaching || JAMBONES_DISABLE_TTS_STREAMING) {
         return mp3Extension;
@@ -58,6 +59,13 @@ function getFileExtension({vendor, renderForCaching = false}) {
     case 'nvidia':
     case 'verbio':
       return r8Extension;
+    case 'google':
+      // google voice cloning just support wav.
+      if (typeof voice === 'object' && voice.voice_cloning_key) {
+        return wavExtension;
+      } else {
+        return mp3Extension;
+      }
     default:
       // If vendor is custom
       if (vendor.startsWith('custom')) {

package-lock.json

@@ -1,17 +1,17 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.1.17",
+  "version": "0.1.22",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@jambonz/speech-utils",
-      "version": "0.1.17",
+      "version": "0.1.22",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-polly": "^3.496.0",
         "@aws-sdk/client-sts": "^3.496.0",
-        "@google-cloud/text-to-speech": "^5.0.2",
+        "@google-cloud/text-to-speech": "^5.5.0",
         "@grpc/grpc-js": "^1.9.14",
         "@jambonz/realtimedb-helpers": "^0.8.7",
         "bent": "^7.3.12",
@@ -1230,9 +1230,10 @@
       }
     },
     "node_modules/@google-cloud/text-to-speech": {
-      "version": "5.0.2",
-      "resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-5.0.2.tgz",
-      "integrity": "sha512-Q11Ddh9eHKSDA3E/KSqMITgVprXb0XgIKuJP9F5ScJ1T9h+DNrbgIU7shd0QOlPqb8ruQRiTOqL08+Mq5R89Ow==",
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-5.5.0.tgz",
+      "integrity": "sha512-Cw/UK2Y3l31Vsuozu8cxsmVS/09fShimes0tRLgDbOY2ZMG1Dckb6Zf/Q3Nxg4X0feFep44pvwNmyHKrOnl9SQ==",
+      "license": "Apache-2.0",
       "dependencies": {
         "google-gax": "^4.0.3"
       },
@@ -8338,9 +8339,9 @@
       "dev": true
     },
     "@google-cloud/text-to-speech": {
-      "version": "5.0.2",
-      "resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-5.0.2.tgz",
-      "integrity": "sha512-Q11Ddh9eHKSDA3E/KSqMITgVprXb0XgIKuJP9F5ScJ1T9h+DNrbgIU7shd0QOlPqb8ruQRiTOqL08+Mq5R89Ow==",
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-5.5.0.tgz",
+      "integrity": "sha512-Cw/UK2Y3l31Vsuozu8cxsmVS/09fShimes0tRLgDbOY2ZMG1Dckb6Zf/Q3Nxg4X0feFep44pvwNmyHKrOnl9SQ==",
       "requires": {
         "google-gax": "^4.0.3"
       }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.1.17",
+  "version": "0.1.22",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",
@@ -28,7 +28,7 @@
   "dependencies": {
     "@aws-sdk/client-polly": "^3.496.0",
     "@aws-sdk/client-sts": "^3.496.0",
-    "@google-cloud/text-to-speech": "^5.0.2",
+    "@google-cloud/text-to-speech": "^5.5.0",
     "@grpc/grpc-js": "^1.9.14",
     "@jambonz/realtimedb-helpers": "^0.8.7",
     "bent": "^7.3.12",

test/synth.js

@@ -91,14 +91,17 @@ test('Google speech Custom voice synth tests', async(t) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);
 
-  if (!process.env.GCP_CUSTOM_VOICE_FILE && !process.env.GCP_CUSTOM_VOICE_JSON_KEY || !process.env.GCP_CUSTOM_VOICE_MODEL) {
-    t.pass('skipping google speech synth tests since neither GCP_CUSTOM_VOICE_FILE nor GCP_CUSTOM_VOICE_JSON_KEY provided, GCP_CUSTOM_VOICE_MODEL is not provided');
+  if (!process.env.GCP_CUSTOM_VOICE_FILE &&
+    !process.env.GCP_CUSTOM_VOICE_JSON_KEY ||
+    !process.env.GCP_CUSTOM_VOICE_MODEL) {
+    t.pass(`skipping google speech synth tests since neither 
+GCP_CUSTOM_VOICE_FILE nor GCP_CUSTOM_VOICE_JSON_KEY provided, GCP_CUSTOM_VOICE_MODEL is not provided`);
     return t.end();
   }
   try {
     const str = process.env.GCP_CUSTOM_VOICE_JSON_KEY || fs.readFileSync(process.env.GCP_CUSTOM_VOICE_FILE);
     const creds = JSON.parse(str);
-    let opts = await synthAudio(stats, {
+    const opts = await synthAudio(stats, {
       vendor: 'google',
       credentials: {
         credentials: {
@@ -109,7 +112,7 @@ test('Google speech Custom voice synth tests', async(t) => {
       language: 'en-AU',
       text: 'This is a test.  This is only a test',
       voice: {
-        reportedUsage:"REALTIME",
+        reportedUsage: 'REALTIME',
         model: process.env.GCP_CUSTOM_VOICE_MODEL
       }
     });
@@ -121,6 +124,48 @@ test('Google speech Custom voice synth tests', async(t) => {
   client.quit();
 });
 
+test('Google speech voice cloning synth tests', async(t) => {
+  const fn = require('..');
+  const {synthAudio, client} = fn(opts, logger);
+
+  if (!process.env.GCP_CUSTOM_VOICE_FILE &&
+    !process.env.GCP_CUSTOM_VOICE_JSON_KEY ||
+    !process.env.GCP_VOICE_CLONING_FILE &&
+    !process.env.GCP_VOICE_CLONING_JSON_KEY) {
+    t.pass(`skipping google speech synth tests since neither 
+GCP_CUSTOM_VOICE_FILE nor GCP_CUSTOM_VOICE_JSON_KEY provided,
+GCP_VOICE_CLONING_FILE nor GCP_VOICE_CLONING_JSON_KEY is not provided`);
+    return t.end();
+  }
+  try {
+    const googleKey = process.env.GCP_CUSTOM_VOICE_JSON_KEY ||
+      fs.readFileSync(process.env.GCP_CUSTOM_VOICE_FILE);
+    const voice_cloning_key = process.env.GCP_VOICE_CLONING_JSON_KEY ||
+      fs.readFileSync(process.env.GCP_VOICE_CLONING_FILE).toString();
+    const creds = JSON.parse(googleKey);
+    const opts = await synthAudio(stats, {
+      vendor: 'google',
+      credentials: {
+        credentials: {
+          client_email: creds.client_email,
+          private_key: creds.private_key,
+          project_id: creds.project_id
+        },
+      },
+      language: 'en-US',
+      text: 'This is a test. This is only a test. This is a test. This is only a test. This is a test. This is only a test',
+      voice: {
+        voice_cloning_key
+      }
+    });
+    t.ok(!opts.servedFromCache, `successfully synthesized google voice cloning audio to ${opts.filePath}`);
+  } catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+  client.quit();
+});
+
 test('AWS speech synth tests', async(t) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);
@@ -509,6 +554,7 @@ test('Custom Vendor speech synth tests', async(t) => {
       text: 'This is a test.  This is only a test',
     });
     t.ok(!opts.servedFromCache, `successfully synthesized custom vendor audio to ${opts.filePath}`);
+    t.ok(opts.filePath.endsWith('wav'), 'audio is cached as wav file');
     let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/somethingnew`);
     t.ok(obj.headers.Authorization == 'Bearer some_jwt_token', 'Custom Vendor Authentication Header is correct');
     t.ok(obj.body.language == 'en-US', 'Custom Vendor Language is correct');
@@ -516,6 +562,21 @@ test('Custom Vendor speech synth tests', async(t) => {
     t.ok(obj.body.type == 'text', 'Custom Vendor type is correct');
     t.ok(obj.body.text == 'This is a test.  This is only a test', 'Custom Vendor text is correct');
 
+    // Checking if cache is stored with wav format
+    opts = await synthAudio(stats, {
+      vendor: 'custom:somethingnew',
+      credentials: {
+        use_for_tts: 1,
+        custom_tts_url: "http://127.0.0.1:3100/somethingnew",
+        auth_token: 'some_jwt_token'
+      },
+      language: 'en-US',
+      voice: 'English-US.Female-1',
+      text: 'This is a test.  This is only a test',
+    });
+    t.ok(opts.servedFromCache, `successfully get custom vendor cached audio to ${opts.filePath}`);
+    t.ok(opts.filePath.endsWith('wav'), 'audio is cached as wav file');
+
     opts = await synthAudio(stats, {
       vendor: 'custom:somethingnew2',
       credentials: {