0.0.50

support mod_playht_tts

lib/synth-audio.js

@@ -85,7 +85,7 @@ async function synthAudio(client, logger, stats, { account_sid,
   logger = logger || noopLogger;
 
   assert.ok(['google', 'aws', 'polly', 'microsoft',
-    'wellsaid', 'nuance', 'nvidia', 'ibm', 'elevenlabs', 'whisper', 'deepgram'].includes(vendor) ||
+    'wellsaid', 'nuance', 'nvidia', 'ibm', 'elevenlabs', 'whisper', 'deepgram', 'playht'].includes(vendor) ||
   vendor.startsWith('custom'),
   `synthAudio supported vendors are google, aws, microsoft, nuance, nvidia and wellsaid, not ${vendor}`);
   if ('google' === vendor) {
@@ -123,6 +123,11 @@ async function synthAudio(client, logger, stats, { account_sid,
     assert.ok(voice, 'synthAudio requires voice when elevenlabs is used');
     assert.ok(credentials.api_key, 'synthAudio requires api_key when elevenlabs is used');
     assert.ok(credentials.model_id, 'synthAudio requires model_id when elevenlabs is used');
+  }  else if ('playht' === vendor) {
+    assert.ok(voice, 'synthAudio requires voice when playht is used');
+    assert.ok(credentials.api_key, 'synthAudio requires api_key when playht is used');
+    assert.ok(credentials.user_id, 'synthAudio requires user_id when playht is used');
+    assert.ok(credentials.voice_engine, 'synthAudio requires voice_engine when playht is used');
   } else if ('whisper' === vendor) {
     assert.ok(voice, 'synthAudio requires voice when whisper is used');
     assert.ok(credentials.model_id, 'synthAudio requires model when whisper is used');
@@ -146,7 +151,7 @@ async function synthAudio(client, logger, stats, { account_sid,
     ) ||
     (
       !process.env.JAMBONES_DISABLE_TTS_STREAMING &&
-      vendor === 'elevenlabs'
+      ['elevenlabs', 'deepgram'].includes(vendor)
     )
   ) {
     filePath = `${TMP_FOLDER}/${key.replace('tts:', `tts-${salt || ''}`)}.r8`;
@@ -206,13 +211,21 @@ async function synthAudio(client, logger, stats, { account_sid,
         });
         if (audioBuffer?.filePath) return audioBuffer;
         break;
+      case 'playht':
+        audioBuffer = await synthPlayHT(logger, {
+          credentials, options, stats, language, voice, text, renderForCaching, disableTtsStreaming, filePath
+        });
+        if (audioBuffer?.filePath) return audioBuffer;
+        break;
       case 'whisper':
         audioBuffer = await synthWhisper(logger, {
           credentials, stats, voice, text, renderForCaching, disableTtsStreaming});
         if (audioBuffer?.filePath) return audioBuffer;
         break;
       case 'deepgram':
-        audioBuffer = await synthDeepgram(logger, {credentials, stats, model, text});
+        audioBuffer = await synthDeepgram(logger, {credentials, stats, model, text,
+          renderForCaching, disableTtsStreaming});
+        if (audioBuffer?.filePath) return audioBuffer;
         break;
       case vendor.startsWith('custom') ? vendor : 'cant_match_value':
         ({ audioBuffer, filePath } = await synthCustomVendor(logger,
@@ -680,6 +693,61 @@ const synthElevenlabs = async(logger, {
   }
 };
 
+const synthPlayHT = async(logger, {
+  credentials, options, stats, voice, text, renderForCaching, disableTtsStreaming
+}) => {
+  const {api_key, user_id, voice_engine, options: credOpts} = credentials;
+  const opts = !!options && Object.keys(options).length !== 0 ? options : JSON.parse(credOpts || '{}');
+
+  /* default to using the streaming interface, unless disabled by env var OR we want just a cache file */
+  if (!process.env.JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
+    let params = '';
+    params += `{api_key=${api_key}`;
+    params += `,user_id=${user_id}`;
+    params += ',vendor=playht';
+    params += `,voice=${voice}`;
+    params += `,voice_engine=${voice_engine}`;
+    params += ',write_cache_file=1';
+    if (opts.quality) params += `,quality=${opts.quality}`;
+    if (opts.speed) params += `,speed=${opts.speed}`;
+    if (opts.seed) params += `,style=${opts.seed}`;
+    if (opts.temperature) params += `,temperature=${opts.temperature}`;
+    if (opts.emotion) params += `,emotion=${opts.emotion}`;
+    if (opts.voice_guidance) params += `,voice_guidance=${opts.voice_guidance}`;
+    if (opts.style_guidance) params += `,style_guidance=${opts.style_guidance}`;
+    if (opts.text_guidance) params += `,text_guidance=${opts.text_guidance}`;
+    params += '}';
+
+    return {
+      filePath: `say:${params}${text.replace(/\n/g, ' ').replace(/\r/g, ' ')}`,
+      servedFromCache: false,
+      rtt: 0
+    };
+  }
+
+  try {
+    const post = bent('https://api.play.ht', 'POST', 'buffer', {
+      'AUTHORIZATION': api_key,
+      'X-USER-ID': user_id,
+      'Accept': 'audio/mpeg',
+      'Content-Type': 'application/json'
+    });
+    const mp3 = await post('/api/v2/tts/stream', {
+      text,
+      voice,
+      voice_engine,
+      output_format: 'mp3',
+      sample_rate: 8000,
+      ...opts
+    });
+    return mp3;
+  } catch (err) {
+    logger.info({err}, 'synth PlayHT returned error');
+    stats.increment('tts.count', ['vendor:playht', 'accepted:no']);
+    throw err;
+  }
+};
+
 const synthWhisper = async(logger, {credentials, stats, voice, text, renderForCaching, disableTtsStreaming}) => {
   const {api_key, model_id, baseURL, timeout, speed} = credentials;
   /* if the env is set to stream then bag out, unless we are specifically rendering to generate a cache file */
@@ -720,8 +788,22 @@ const synthWhisper = async(logger, {credentials, stats, voice, text, renderForCa
   }
 };
 
-const synthDeepgram = async(logger, {credentials, stats, model, text}) => {
+const synthDeepgram = async(logger, {credentials, stats, model, text, renderForCaching, disableTtsStreaming}) => {
   const {api_key} = credentials;
+  if (!process.env.JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) {
+    let params = '';
+    params += `{api_key=${api_key}`;
+    params += ',vendor=deepgram';
+    params += `,voice=${model}`;
+    params += ',write_cache_file=1';
+    params += '}';
+
+    return {
+      filePath: `say:${params}${text.replace(/\n/g, ' ')}`,
+      servedFromCache: false,
+      rtt: 0
+    };
+  }
   try {
     const post = bent('https://api.deepgram.com', 'POST', 'buffer', {
       'Authorization': `Token ${api_key}`,

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.0.47",
+  "version": "0.0.50",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "@jambonz/speech-utils",
-      "version": "0.0.47",
+      "version": "0.0.50",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-polly": "^3.496.0",
@@ -19,7 +19,7 @@
         "form-urlencoded": "^6.1.4",
         "google-protobuf": "^3.21.2",
         "ibm-watson": "^8.0.0",
-        "microsoft-cognitiveservices-speech-sdk": "1.34.0",
+        "microsoft-cognitiveservices-speech-sdk": "1.36.0",
         "openai": "^4.25.0",
         "undici": "^6.4.0"
       },
@@ -1124,14 +1124,6 @@
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       }
     },
-    "node_modules/@fastify/busboy": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
-      "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
-      "engines": {
-        "node": ">=14"
-      }
-    },
     "node_modules/@google-cloud/text-to-speech": {
       "version": "5.0.2",
       "resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-5.0.2.tgz",
@@ -3125,13 +3117,14 @@
       }
     },
     "node_modules/es5-ext": {
-      "version": "0.10.62",
-      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz",
-      "integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==",
+      "version": "0.10.64",
+      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.64.tgz",
+      "integrity": "sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==",
       "hasInstallScript": true,
       "dependencies": {
         "es6-iterator": "^2.0.3",
         "es6-symbol": "^3.1.3",
+        "esniff": "^2.0.1",
         "next-tick": "^1.1.0"
       },
       "engines": {
@@ -3278,6 +3271,25 @@
         "url": "https://opencollective.com/eslint"
       }
     },
+    "node_modules/esniff": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/esniff/-/esniff-2.0.1.tgz",
+      "integrity": "sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==",
+      "dependencies": {
+        "d": "^1.0.1",
+        "es5-ext": "^0.10.62",
+        "event-emitter": "^0.3.5",
+        "type": "^2.7.2"
+      },
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
+    "node_modules/esniff/node_modules/type": {
+      "version": "2.7.2",
+      "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz",
+      "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw=="
+    },
     "node_modules/espree": {
       "version": "9.6.1",
       "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz",
@@ -3350,6 +3362,15 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/event-emitter": {
+      "version": "0.3.5",
+      "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz",
+      "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==",
+      "dependencies": {
+        "d": "1",
+        "es5-ext": "~0.10.14"
+      }
+    },
     "node_modules/event-target-shim": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
@@ -3551,9 +3572,9 @@
       "dev": true
     },
     "node_modules/follow-redirects": {
-      "version": "1.15.5",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz",
-      "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==",
+      "version": "1.15.6",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
+      "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==",
       "funding": [
         {
           "type": "individual",
@@ -5084,9 +5105,9 @@
       }
     },
     "node_modules/microsoft-cognitiveservices-speech-sdk": {
-      "version": "1.34.0",
-      "resolved": "https://registry.npmjs.org/microsoft-cognitiveservices-speech-sdk/-/microsoft-cognitiveservices-speech-sdk-1.34.0.tgz",
-      "integrity": "sha512-WAR0YqouRzVux2kI+f5wTPC6NyJgIVC1g65d79dJ9I32WPJs2kK+eb/BMB6mhSdCjackO5FsrW7JLaQ/vB1heQ==",
+      "version": "1.36.0",
+      "resolved": "https://registry.npmjs.org/microsoft-cognitiveservices-speech-sdk/-/microsoft-cognitiveservices-speech-sdk-1.36.0.tgz",
+      "integrity": "sha512-wPxuEXgjLdqMMIrdBtl8jquGahLV19LQE0ie8MI/PcBcNLG5buVzwS2rQEyHMsRGx+C/4OdBo1ROdNIUzCm4Lg==",
       "dependencies": {
         "@types/webrtc": "^0.0.37",
         "agent-base": "^6.0.1",
@@ -6851,12 +6872,9 @@
       }
     },
     "node_modules/undici": {
-      "version": "6.4.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-6.4.0.tgz",
-      "integrity": "sha512-wYaKgftNqf6Je7JQ51YzkEkEevzOgM7at5JytKO7BjaURQpERW8edQSMrr2xb+Yv4U8Yg47J24+lc9+NbeXMFA==",
-      "dependencies": {
-        "@fastify/busboy": "^2.0.0"
-      },
+      "version": "6.11.1",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-6.11.1.tgz",
+      "integrity": "sha512-KyhzaLJnV1qa3BSHdj4AZ2ndqI0QWPxYzaIOio0WzcEJB9gvuysprJSLtpvc2D9mhR9jPDUk7xlJlZbH2KR5iw==",
       "engines": {
         "node": ">=18.0"
       }
@@ -8092,11 +8110,6 @@
       "integrity": "sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A==",
       "dev": true
     },
-    "@fastify/busboy": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
-      "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA=="
-    },
     "@google-cloud/text-to-speech": {
       "version": "5.0.2",
       "resolved": "https://registry.npmjs.org/@google-cloud/text-to-speech/-/text-to-speech-5.0.2.tgz",
@@ -9652,12 +9665,13 @@
       }
     },
     "es5-ext": {
-      "version": "0.10.62",
-      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz",
-      "integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==",
+      "version": "0.10.64",
+      "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.64.tgz",
+      "integrity": "sha512-p2snDhiLaXe6dahss1LddxqEm+SkuDvV8dnIQG0MWjyHpcMNfXKPE+/Cc0y+PhxJX3A4xGNeFCj5oc0BUh6deg==",
       "requires": {
         "es6-iterator": "^2.0.3",
         "es6-symbol": "^3.1.3",
+        "esniff": "^2.0.1",
         "next-tick": "^1.1.0"
       }
     },
@@ -9766,6 +9780,24 @@
       "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==",
       "dev": true
     },
+    "esniff": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/esniff/-/esniff-2.0.1.tgz",
+      "integrity": "sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==",
+      "requires": {
+        "d": "^1.0.1",
+        "es5-ext": "^0.10.62",
+        "event-emitter": "^0.3.5",
+        "type": "^2.7.2"
+      },
+      "dependencies": {
+        "type": {
+          "version": "2.7.2",
+          "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz",
+          "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw=="
+        }
+      }
+    },
     "espree": {
       "version": "9.6.1",
       "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz",
@@ -9813,6 +9845,15 @@
       "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
       "dev": true
     },
+    "event-emitter": {
+      "version": "0.3.5",
+      "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz",
+      "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==",
+      "requires": {
+        "d": "1",
+        "es5-ext": "~0.10.14"
+      }
+    },
     "event-target-shim": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
@@ -9964,9 +10005,9 @@
       "dev": true
     },
     "follow-redirects": {
-      "version": "1.15.5",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz",
-      "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw=="
+      "version": "1.15.6",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
+      "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA=="
     },
     "for-each": {
       "version": "0.3.3",
@@ -11108,9 +11149,9 @@
       }
     },
     "microsoft-cognitiveservices-speech-sdk": {
-      "version": "1.34.0",
-      "resolved": "https://registry.npmjs.org/microsoft-cognitiveservices-speech-sdk/-/microsoft-cognitiveservices-speech-sdk-1.34.0.tgz",
-      "integrity": "sha512-WAR0YqouRzVux2kI+f5wTPC6NyJgIVC1g65d79dJ9I32WPJs2kK+eb/BMB6mhSdCjackO5FsrW7JLaQ/vB1heQ==",
+      "version": "1.36.0",
+      "resolved": "https://registry.npmjs.org/microsoft-cognitiveservices-speech-sdk/-/microsoft-cognitiveservices-speech-sdk-1.36.0.tgz",
+      "integrity": "sha512-wPxuEXgjLdqMMIrdBtl8jquGahLV19LQE0ie8MI/PcBcNLG5buVzwS2rQEyHMsRGx+C/4OdBo1ROdNIUzCm4Lg==",
       "requires": {
         "@types/webrtc": "^0.0.37",
         "agent-base": "^6.0.1",
@@ -12405,12 +12446,9 @@
       }
     },
     "undici": {
-      "version": "6.4.0",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-6.4.0.tgz",
-      "integrity": "sha512-wYaKgftNqf6Je7JQ51YzkEkEevzOgM7at5JytKO7BjaURQpERW8edQSMrr2xb+Yv4U8Yg47J24+lc9+NbeXMFA==",
-      "requires": {
-        "@fastify/busboy": "^2.0.0"
-      }
+      "version": "6.11.1",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-6.11.1.tgz",
+      "integrity": "sha512-KyhzaLJnV1qa3BSHdj4AZ2ndqI0QWPxYzaIOio0WzcEJB9gvuysprJSLtpvc2D9mhR9jPDUk7xlJlZbH2KR5iw=="
     },
     "undici-types": {
       "version": "5.26.5",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.0.47",
+  "version": "0.0.50",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",
@@ -34,7 +34,7 @@
     "form-urlencoded": "^6.1.4",
     "google-protobuf": "^3.21.2",
     "ibm-watson": "^8.0.0",
-    "microsoft-cognitiveservices-speech-sdk": "1.34.0",
+    "microsoft-cognitiveservices-speech-sdk": "1.36.0",
     "openai": "^4.25.0",
     "undici": "^6.4.0"
   },

test/synth.js

@@ -549,6 +549,47 @@ test('Elevenlabs speech synth tests', async(t) => {
   client.quit();
 })
 
+test('PlayHT speech synth tests', async(t) => {
+  const fn = require('..');
+  const {synthAudio, client} = fn(opts, logger);
+
+  if (!process.env.PLAYHT_API_KEY || !process.env.PLAYHT_USER_ID) {
+    t.pass('skipping PlayHT speech synth tests since PLAYHT_API_KEY or PLAYHT_USER_ID is/are not provided');
+    return t.end();
+  }
+  const text = 'Hi there and welcome to jambones!';
+  try {
+    let opts = await synthAudio(stats, {
+      vendor: 'playht',
+      credentials: {
+        api_key: process.env.PLAYHT_API_KEY,
+        user_id: process.env.PLAYHT_USER_ID,
+        voice_engine: 'PlayHT2.0-turbo',
+        options: JSON.stringify({
+          quality: "medium",
+          speed: 1,
+          seed: 1,
+          temperature: 1,
+          emotion: "female_happy",
+          voice_guidance: 3,
+          style_guidance: 20,
+          text_guidance: 1,
+        })
+      },
+      language: 'en-US',
+      voice: 's3://voice-cloning-zero-shot/d9ff78ba-d016-47f6-b0ef-dd630f59414e/female-cs/manifest.json',
+      text,
+      renderForCaching: true
+    });
+    t.ok(!opts.servedFromCache, `successfully synthesized eleven audio to ${opts.filePath}`);
+
+  } catch (err) {
+    console.error(JSON.stringify(err));
+    t.end(err);
+  }
+  client.quit();
+})
+
 test('whisper speech synth tests', async(t) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);
@@ -595,6 +636,7 @@ test('Deepgram speech synth tests', async(t) => {
       },
       model: 'aura-asteria-en',
       text,
+      renderForCaching: true
     });
     t.ok(!opts.servedFromCache, `successfully synthesized deepgram audio to ${opts.filePath}`);