fixed playht3.0 cannot be played if credential is cached

support playht3.0

.github/workflows/ci.yml

@@ -13,6 +13,11 @@ jobs:
         with:
           node-version: lts/*
       - run: npm install
+      - name: Install Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+          docker-compose --version
       - run: npm run jslint
       - run: sudo apt update && sudo apt install -y squid
       - run: sudo cp test/squid.conf /etc/squid/squid.conf

lib/config.js

@@ -1,6 +1,7 @@
 const JAMBONES_TTS_TRIM_SILENCE = process.env.JAMBONES_TTS_TRIM_SILENCE;
 const JAMBONES_DISABLE_TTS_STREAMING = process.env.JAMBONES_DISABLE_TTS_STREAMING;
 const JAMBONES_DISABLE_AZURE_TTS_STREAMING = process.env.JAMBONES_DISABLE_AZURE_TTS_STREAMING;
+const JAMBONES_EAGERLY_PRE_CACHE_AUDIO = process.env.JAMBONES_EAGERLY_PRE_CACHE_AUDIO;
 
 const JAMBONES_HTTP_PROXY_IP = process.env.JAMBONES_HTTP_PROXY_IP;
 const JAMBONES_HTTP_PROXY_PORT = process.env.JAMBONES_HTTP_PROXY_PORT;
@@ -18,6 +19,7 @@ module.exports = {
   JAMBONES_HTTP_PROXY_IP,
   JAMBONES_HTTP_PROXY_PORT,
   JAMBONES_TTS_CACHE_DURATION_MINS,
+  JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
   TMP_FOLDER,
   HTTP_TIMEOUT
 };

lib/synth-audio.js

@@ -20,7 +20,8 @@ const {
   createKryptonClient,
   createRivaClient,
   noopLogger,
-  makeFilePath
+  makeFilePath,
+  makePlayhtKey
 } = require('./utils');
 const getNuanceAccessToken = require('./get-nuance-access-token');
 const getVerbioAccessToken = require('./get-verbio-token');
@@ -44,6 +45,7 @@ const {
   JAMBONES_HTTP_PROXY_IP,
   JAMBONES_HTTP_PROXY_PORT,
   JAMBONES_TTS_CACHE_DURATION_MINS,
+  JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
 } = require('./config');
 const EXPIRES = JAMBONES_TTS_CACHE_DURATION_MINS;
 const OpenAI = require('openai');
@@ -86,7 +88,7 @@ const trimTrailingSilence = (buffer) => {
  */
 async function synthAudio(client, createHash, retrieveHash, logger, stats, { account_sid,
   vendor, language, voice, gender, text, engine, salt, model, credentials, deploymentId,
-  disableTtsCache, renderForCaching, disableTtsStreaming, options
+  disableTtsCache, renderForCaching = false, disableTtsStreaming, options
 }) {
   let audioBuffer;
   let servedFromCache = false;
@@ -151,21 +153,48 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
     assert.ok(voice, 'synthAudio requires voice when verbio is used');
     assert.ok(credentials.client_id, 'synthAudio requires client_id when verbio is used');
     assert.ok(credentials.client_secret, 'synthAudio requires client_secret when verbio is used');
+  } else if ('deepgram' === vendor) {
+    if (!credentials.deepgram_tts_uri) {
+      assert.ok(credentials.api_key, 'synthAudio requires api_key when deepgram is used');
+    }
+
   }
+
   const key = makeSynthKey({
     account_sid,
     vendor,
     language: language || '',
     voice: voice || deploymentId,
     engine,
-    text
+    text,
+    renderForCaching
   });
   let filePath;
-  filePath = makeFilePath(vendor, key, salt);
+  filePath = makeFilePath({vendor, key, salt, renderForCaching});
   debug(`synth key is ${key}`);
   let cached;
   if (!disableTtsCache) {
     cached = await client.get(key);
+    /**
+    *  If we are using tts streaming and also precaching audio, audio could have been cached by streaming (r8)
+    * or here in speech-utils due to precaching (mp3), so we need to check for both keys.
+    */
+    if (!cached && JAMBONES_EAGERLY_PRE_CACHE_AUDIO) {
+      const preCachekey = makeSynthKey({
+        account_sid,
+        vendor,
+        language: language || '',
+        voice: voice || deploymentId,
+        engine,
+        text,
+        renderForCaching: true
+      });
+      cached = await client.get(preCachekey);
+      if (cached) {
+        // Precache audio is available update filpath with precache file extension.
+        filePath = makeFilePath({vendor, key, salt, renderForCaching: true});
+      }
+    }
   }
   if (cached) {
     // found in cache - extend the expiry and use it
@@ -216,7 +245,7 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc
         });
         break;
       case 'playht':
-        audioBuffer = await synthPlayHT(logger, {
+        audioBuffer = await synthPlayHT(client, logger, {
           credentials, options, stats, language, voice, text, renderForCaching, disableTtsStreaming, filePath
         });
         break;
@@ -727,12 +756,40 @@ const synthElevenlabs = async(logger, {
   }
 };
 
-const synthPlayHT = async(logger, {
+const synthPlayHT = async(client, logger, {
-  credentials, options, stats, voice, text, renderForCaching, disableTtsStreaming
+  credentials, options, stats, voice, language, text, renderForCaching, disableTtsStreaming
 }) => {
   const {api_key, user_id, voice_engine, options: credOpts} = credentials;
   const opts = !!options && Object.keys(options).length !== 0 ? options : JSON.parse(credOpts || '{}');
 
+  let synthesizeUrl = 'https://api.play.ht/api/v2/tts/stream';
+
+  // If model is play3.0, the synthesizeUrl is got from authentication endpoint
+  if (voice_engine === 'Play3.0') {
+    try {
+      const post = bent('https://api.play.ht', 'POST', 'json', 201, {
+        'AUTHORIZATION': api_key,
+        'X-USER-ID': user_id,
+        'Accept': 'application/json'
+      });
+      const key = makePlayhtKey(api_key);
+      const url = await client.get(key);
+      if (!url) {
+        const {inference_address, expires_at_ms} = await post('/api/v3/auth');
+        synthesizeUrl = inference_address;
+        const expiry =  Math.floor((expires_at_ms - Date.now()) / 1000 - 30);
+        await client.set(key, inference_address, 'EX', expiry);
+      } else {
+        // Use cached URL
+        synthesizeUrl = url;
+      }
+    } catch (err) {
+      logger.info({err}, 'synth PlayHT returned error for authentication version 3.0');
+      stats.increment('tts.count', ['vendor:playht', 'accepted:no']);
+      throw err;
+    }
+  }
+
   /* default to using the streaming interface, unless disabled by env var OR we want just a cache file */
   if (!JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
     let params = '';
@@ -741,6 +798,7 @@ const synthPlayHT = async(logger, {
     params += ',vendor=playht';
     params += `,voice=${voice}`;
     params += `,voice_engine=${voice_engine}`;
+    params += `,synthesize_url=${synthesizeUrl}`;
     params += ',write_cache_file=1';
     if (opts.quality) params += `,quality=${opts.quality}`;
     if (opts.speed) params += `,speed=${opts.speed}`;
@@ -750,6 +808,8 @@ const synthPlayHT = async(logger, {
     if (opts.voice_guidance) params += `,voice_guidance=${opts.voice_guidance}`;
     if (opts.style_guidance) params += `,style_guidance=${opts.style_guidance}`;
     if (opts.text_guidance) params += `,text_guidance=${opts.text_guidance}`;
+    if (opts.top_p) params += `,top_p=${opts.top_p}`;
+    if (opts.repetition_penalty) params += `,repetition_penalty=${opts.repetition_penalty}`;
     params += '}';
 
     return {
@@ -760,14 +820,18 @@ const synthPlayHT = async(logger, {
   }
 
   try {
-    const post = bent('https://api.play.ht', 'POST', 'buffer', {
+    const post = bent('POST', 'buffer', {
-      'AUTHORIZATION': api_key,
+      ...(voice_engine !== 'Play3.0' && {
-      'X-USER-ID': user_id,
+        'AUTHORIZATION': api_key,
+        'X-USER-ID': user_id,
+      }),
       'Accept': 'audio/mpeg',
       'Content-Type': 'application/json'
     });
-    const mp3 = await post('/api/v2/tts/stream', {
+
+    const mp3 = await post(synthesizeUrl, {
       text,
+      ...(voice_engine === 'Play3.0' && { language }),
       voice,
       voice_engine,
       output_format: 'mp3',
@@ -909,13 +973,14 @@ const synthWhisper = async(logger, {credentials, stats, voice, text, renderForCa
 };
 
 const synthDeepgram = async(logger, {credentials, stats, model, text, renderForCaching, disableTtsStreaming}) => {
-  const {api_key} = credentials;
+  const {api_key, deepgram_tts_uri} = credentials;
   if (!JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
     let params = '';
     params += `{api_key=${api_key}`;
     params += ',vendor=deepgram';
     params += `,voice=${model}`;
     params += ',write_cache_file=1';
+    if (deepgram_tts_uri) params += `,endpoint=${deepgram_tts_uri}`;
     params += '}';
 
     return {
@@ -925,8 +990,9 @@ const synthDeepgram = async(logger, {credentials, stats, model, text, renderForC
     };
   }
   try {
-    const post = bent('https://api.deepgram.com', 'POST', 'buffer', {
+    const post = bent(deepgram_tts_uri || 'https://api.deepgram.com', 'POST', 'buffer', {
-      'Authorization': `Token ${api_key}`,
+      // on-premise deepgram does not  require to have api_key
+      ...(api_key && {'Authorization': `Token ${api_key}`}),
       'Accept': 'audio/mpeg',
       'Content-Type': 'application/json'
     });

lib/utils.js

@@ -16,29 +16,31 @@ const debug = require('debug')('jambonz:realtimedb-helpers');
  */
 //const nuanceClientMap = new Map();
 
-function makeSynthKey({account_sid = '', vendor, language, voice, engine = '', text}) {
+function makeSynthKey({
+  account_sid = '', vendor, language, voice, engine = '', text,
+  renderForCaching = false}) {
   const hash = crypto.createHash('sha1');
   hash.update(`${language}:${vendor}:${voice}:${engine}:${text}`);
   const hexHashKey = hash.digest('hex');
   const accountKey = account_sid ? `:${account_sid}` : '';
-  const namespace = vendor.startsWith('custom') ? vendor : getFileExtension(vendor);
+  const namespace = vendor.startsWith('custom') ? vendor : getFileExtension({vendor, renderForCaching});
   const key = `tts${accountKey}:${namespace}:${hexHashKey}`;
   return key;
 }
 
-function makeFilePath(vendor, key, salt = '') {
+function makeFilePath({vendor, key, salt = '', renderForCaching = false}) {
-  const extension = getFileExtension(vendor);
+  const extension = getFileExtension({vendor, renderForCaching});
   return `${TMP_FOLDER}/${key.replace('tts:', `tts-${salt}`)}.${extension}`;
 }
 
-function getFileExtension(vendor) {
+function getFileExtension({vendor, renderForCaching = false}) {
   const mp3Extension = 'mp3';
   const r8Extension = 'r8';
 
   switch (vendor) {
     case 'azure':
     case 'microsoft':
-      if (!JAMBONES_DISABLE_TTS_STREAMING || JAMBONES_TTS_TRIM_SILENCE) {
+      if (!renderForCaching && !JAMBONES_DISABLE_TTS_STREAMING || JAMBONES_TTS_TRIM_SILENCE) {
         return r8Extension;
       } else {
         return mp3Extension;
@@ -46,16 +48,25 @@ function getFileExtension(vendor) {
     case 'deepgram':
     case 'elevenlabs':
     case 'rimlabs':
-      if (!JAMBONES_DISABLE_TTS_STREAMING) {
+    case 'playht':
-        return r8Extension;
+      if (renderForCaching || JAMBONES_DISABLE_TTS_STREAMING) {
-      } else {
         return mp3Extension;
+      } else {
+        return r8Extension;
       }
     case 'nuance':
     case 'nvidia':
     case 'verbio':
       return r8Extension;
     default:
+      // If vendor is custom
+      if (vendor.startsWith('custom')) {
+        if (renderForCaching || JAMBONES_DISABLE_TTS_STREAMING) {
+          return mp3Extension;
+        } else {
+          return r8Extension;
+        }
+      }
       return mp3Extension;
   }
 }
@@ -87,6 +98,11 @@ function makeAwsKey(awsAccessKeyId) {
   return `aws:${hash.digest('hex')}`;
 }
 
+function makePlayhtKey(apiKey) {
+  const hash = crypto.createHash('sha1');
+  hash.update(apiKey);
+  return `playht:${hash.digest('hex')}`;
+}
 function makeVerbioKey(client_id) {
   const hash = crypto.createHash('sha1');
   hash.update(client_id);
@@ -160,6 +176,7 @@ module.exports = {
   makeSynthKey,
   makeNuanceKey,
   makeIbmKey,
+  makePlayhtKey,
   makeAwsKey,
   makeVerbioKey,
   getNuanceAccessToken,

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.1.12",
+  "version": "0.1.17",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@jambonz/speech-utils",
-      "version": "0.1.12",
+      "version": "0.1.17",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-polly": "^3.496.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.1.12",
+  "version": "0.1.17",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",

test/synth.js

@@ -574,9 +574,9 @@ test('Elevenlabs speech synth tests', async(t) => {
     t.end(err);
   }
   client.quit();
-})
+});
 
-test('PlayHT speech synth tests', async(t) => {
+const testPlayHT = async(t, voice_engine) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);
 
@@ -584,26 +584,26 @@ test('PlayHT speech synth tests', async(t) => {
     t.pass('skipping PlayHT speech synth tests since PLAYHT_API_KEY or PLAYHT_USER_ID is/are not provided');
     return t.end();
   }
-  const text = 'Hi there and welcome to jambones!';
+  const text = 'Hi there and welcome to jambones! ' + Date.now();
   try {
-    let opts = await synthAudio(stats, {
+    const opts = await synthAudio(stats, {
       vendor: 'playht',
       credentials: {
         api_key: process.env.PLAYHT_API_KEY,
         user_id: process.env.PLAYHT_USER_ID,
-        voice_engine: 'PlayHT2.0-turbo',
+        voice_engine,
         options: JSON.stringify({
-          quality: "medium",
+          quality: 'medium',
           speed: 1,
           seed: 1,
           temperature: 1,
-          emotion: "female_happy",
+          emotion: 'female_happy',
           voice_guidance: 3,
           style_guidance: 20,
           text_guidance: 1,
         })
       },
-      language: 'en-US',
+      language: 'english',
       voice: 's3://voice-cloning-zero-shot/d9ff78ba-d016-47f6-b0ef-dd630f59414e/female-cs/manifest.json',
       text,
       renderForCaching: true
@@ -615,6 +615,14 @@ test('PlayHT speech synth tests', async(t) => {
     t.end(err);
   }
   client.quit();
+};
+
+test('PlayHT speech synth tests', async(t) => {
+  await testPlayHT(t, 'PlayHT2.0-turbo');
+});
+
+test('PlayHT3.0 speech synth tests', async(t) => {
+  await testPlayHT(t, 'Play3.0');
 });
 
 test('rimelabs speech synth tests', async(t) => {