0.0.22

add elevenlabs

lib/synth-audio.js

@@ -14,7 +14,13 @@ const {
   CancellationDetails,
   SpeechSynthesisOutputFormat
 } = sdk;
-const {makeSynthKey, createNuanceClient, createKryptonClient, createRivaClient, noopLogger} = require('./utils');
+const {
+  makeSynthKey,
+  createNuanceClient,
+  createKryptonClient,
+  createRivaClient,
+  noopLogger
+} = require('./utils');
 const getNuanceAccessToken = require('./get-nuance-access-token');
 const {
   SynthesisRequest,
@@ -30,9 +36,27 @@ const {
 const {SynthesizeSpeechRequest} = require('../stubs/riva/proto/riva_tts_pb');
 const {AudioEncoding} = require('../stubs/riva/proto/riva_audio_pb');
 const debug = require('debug')('jambonz:realtimedb-helpers');
-const EXPIRES = (process.env.JAMBONES_TTS_CACHE_DURATION_MINS || 24 * 60) * 60; // cache tts for 24 hours
+const EXPIRES = (process.env.JAMBONES_TTS_CACHE_DURATION_MINS || 4 * 60) * 60; // cache tts for 4 hours
 const TMP_FOLDER = '/tmp';
 
+
+const trimTrailingSilence = (buffer) => {
+  assert.ok(buffer instanceof Buffer, 'trimTrailingSilence - argument is not a Buffer');
+
+  let offset = buffer.length;
+  while (offset > 0) {
+    // Get 16-bit value from the buffer (read in reverse)
+    const value = buffer.readUInt16BE(offset - 2);
+    if (value !== 0) {
+      break;
+    }
+    offset -= 2;
+  }
+
+  // Trim the silence from the end
+  return offset === buffer.length ? buffer : buffer.subarray(0, offset);
+};
+
 /**
  * Synthesize speech to an mp3 file, and also cache the generated speech
  * in redis (base64 format) for 24 hours so as to avoid unnecessarily paying
@@ -58,7 +82,8 @@ async function synthAudio(client, logger, stats, { account_sid,
   let rtt;
   logger = logger || noopLogger;
 
-  assert.ok(['google', 'aws', 'polly', 'microsoft', 'wellsaid', 'nuance', 'nvidia', 'ibm'].includes(vendor) ||
+  assert.ok(['google', 'aws', 'polly', 'microsoft',
+    'wellsaid', 'nuance', 'nvidia', 'ibm', 'elevenlabs'].includes(vendor) ||
   vendor.startsWith('custom'),
   `synthAudio supported vendors are google, aws, microsoft, nuance, nvidia and wellsaid, not ${vendor}`);
   if ('google' === vendor) {
@@ -104,7 +129,12 @@ async function synthAudio(client, logger, stats, { account_sid,
     text
   });
   let filePath;
-  if (['nuance', 'nvidia'].includes(vendor)) {
+  if (['nuance', 'nvidia'].includes(vendor) ||
+    (
+      process.env.JAMBONES_TTS_TRIM_SILENCE &&
+      ['microsoft', 'azure'].includes(vendor)
+    )
+  ) {
     filePath = `${TMP_FOLDER}/${key.replace('tts:', `tts-${salt || ''}`)}.r8`;
   }
   else filePath = `${TMP_FOLDER}/${key.replace('tts:', `tts-${salt || ''}`)}.mp3`;
@@ -154,6 +184,9 @@ async function synthAudio(client, logger, stats, { account_sid,
       case 'wellsaid':
         audioBuffer = await synthWellSaid(logger, {credentials, stats, language, voice, text, filePath});
         break;
+      case 'elevenlabs':
+        audioBuffer = await synthElevenlabs(logger, {credentials, stats, language, voice, text, filePath});
+        break;
       case vendor.startsWith('custom') ? vendor : 'cant_match_value':
         ({ audioBuffer, filePath } = await synthCustomVendor(logger,
           {credentials, stats, language, voice, text, filePath}));
@@ -274,6 +307,48 @@ const synthIbm = async(logger, {credentials, stats, voice, text}) => {
   }
 };
 
+async function _synthOnPremMicrosoft(logger, {
+  credentials,
+  stats,
+  language,
+  voice,
+  text,
+  filePath
+}) {
+  const {use_custom_tts, custom_tts_endpoint_url} = credentials;
+  let content = text;
+
+  if (use_custom_tts && !content.startsWith('<speak')) {
+    /**
+     * Note: it seems that to use custom voice ssml is required with the voice attribute
+     * Otherwise sending plain text we get "Voice does not match"
+     */
+    content = `<speak>${text}</speak>`;
+  }
+
+  if (content.startsWith('<speak>')) {
+    /* microsoft enforces some properties and uses voice xml element so if the user did not supply do it for them */
+    const words = content.slice(7, -8).trim().replace(/(\r\n|\n|\r)/gm, ' ');
+    // eslint-disable-next-line max-len
+    content = `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="https://www.w3.org/2001/mstts" xml:lang="${language}"><voice name="${voice}">${words}</voice></speak>`;
+    logger.info({content}, 'synthMicrosoft');
+  }
+
+  try {
+    const trimSilence = filePath.endsWith('.r8');
+    const post = bent('POST', 'buffer', {
+      'X-Microsoft-OutputFormat': trimSilence ? 'raw-8khz-16bit-mono-pcm' : 'audio-16khz-32kbitrate-mono-mp3',
+      'Content-Type': 'application/ssml+xml',
+      'User-Agent': 'Jambonz'
+    });
+    const mp3 = await post(custom_tts_endpoint_url, content);
+    return mp3;
+  } catch (err) {
+    logger.info({err}, '_synthMicrosoftByHttp returned error');
+    throw err;
+  }
+}
+
 const synthMicrosoft = async(logger, {
   credentials,
   stats,
@@ -283,21 +358,35 @@ const synthMicrosoft = async(logger, {
   filePath
 }) => {
   try {
-    const {api_key: apiKey, region, use_custom_tts, custom_tts_endpoint} = credentials;
+    const {api_key: apiKey, region, use_custom_tts, custom_tts_endpoint, custom_tts_endpoint_url} = credentials;
+    if (use_custom_tts && custom_tts_endpoint_url) {
+      return await _synthOnPremMicrosoft(logger, {
+        credentials,
+        stats,
+        language,
+        voice,
+        text,
+        filePath
+      });
+    }
+    const trimSilence = filePath.endsWith('.r8');
     let content = text;
     const speechConfig = SpeechConfig.fromSubscription(apiKey, region);
     speechConfig.speechSynthesisLanguage = language;
     speechConfig.speechSynthesisVoiceName = voice;
     if (use_custom_tts && custom_tts_endpoint) {
       speechConfig.endpointId = custom_tts_endpoint;
-
+    }
+    if (use_custom_tts && !content.startsWith('<speak')) {
       /**
        * Note: it seems that to use custom voice ssml is required with the voice attribute
        * Otherwise sending plain text we get "Voice does not match"
        */
-      if (!content.startsWith('<speak')) content = `<speak>${text}</speak>`;
+      content = `<speak>${text}</speak>`;
     }
-    speechConfig.speechSynthesisOutputFormat = SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3;
+    speechConfig.speechSynthesisOutputFormat = trimSilence ?
+      SpeechSynthesisOutputFormat.Raw8Khz16BitMonoPcm :
+      SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3;
     const synthesizer = new SpeechSynthesizer(speechConfig);
 
     if (content.startsWith('<speak>')) {
@@ -323,7 +412,9 @@ const synthMicrosoft = async(logger, {
               reject(cancellation.errorDetails);
               break;
             case ResultReason.SynthesizingAudioCompleted:
-              resolve(Buffer.from(result.audioData));
+              let buffer = Buffer.from(result.audioData);
+              if (trimSilence) buffer = trimTrailingSilence(buffer);
+              resolve(buffer);
               synthesizer.close();
               stats.increment('tts.count', ['vendor:microsoft', 'accepted:yes']);
               break;
@@ -481,6 +572,29 @@ const synthCustomVendor = async(logger, {credentials, stats, language, voice, te
   }
 };
 
+const synthElevenlabs = async(logger, {credentials, stats, language, voice, text}) => {
+  const {api_key, model_id} = credentials;
+  try {
+    const post = bent('https://api.elevenlabs.io', 'POST', 'buffer', {
+      'xi-api-key': api_key,
+      'Accept': 'audio/mpeg',
+      'Content-Type': 'application/json'
+    });
+    const mp3 = await post(`/v1/text-to-speech/${voice}`, {
+      text,
+      model_id,
+      voice_settings: {
+        stability: 0.5,
+        similarity_boost: 0.5
+      }
+    });
+    return mp3;
+  } catch (err) {
+    logger.info({err}, 'synthEvenlabs returned error');
+    throw err;
+  }
+};
+
 const getFileExtFromMime = (mime) => {
   switch (mime) {
     case 'audio/wav':

lib/utils.js

@@ -106,7 +106,6 @@ const createRivaClient = async(rivaUri) => {
   return client;
 };
 
-
 module.exports = {
   makeSynthKey,
   makeNuanceKey,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.0.14",
+  "version": "0.0.22",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",
@@ -24,7 +24,7 @@
   },
   "homepage": "https://github.com/jambonz/speech-utils#readme",
   "dependencies": {
-    "@aws-sdk/client-polly": "^3.303.0",
+    "@aws-sdk/client-polly": "^3.359.0",
     "@google-cloud/text-to-speech": "^4.2.1",
     "@grpc/grpc-js": "^1.8.13",
     "bent": "^7.3.12",
@@ -32,7 +32,7 @@
     "form-urlencoded": "^6.1.0",
     "google-protobuf": "^3.21.2",
     "ibm-watson": "^8.0.0",
-    "microsoft-cognitiveservices-speech-sdk": "^1.26.0",
+    "microsoft-cognitiveservices-speech-sdk": "^1.31.0",
     "ioredis": "^5.3.2",
     "undici": "^5.21.0"
   },

test/synth.js

@@ -411,6 +411,35 @@ test('Custom Vendor speech synth tests', async(t) => {
   client.quit();
 });
 
+test('Elevenlabs speech synth tests', async(t) => {
+  const fn = require('..');
+  const {synthAudio, client} = fn(opts, logger);
+
+  if (!process.env.ELEVENLABS_API_KEY || !process.env.ELEVENLABS_VOICE_ID || !process.env.ELEVENLABS_MODEL_ID) {
+    t.pass('skipping IBM Watson speech synth tests since IBM_TTS_API_KEY or IBM_TTS_API_KEY not provided');
+    return t.end();
+  }
+  const text = 'Hi there and welcome to jambones!';
+  try {
+    let opts = await synthAudio(stats, {
+      vendor: 'elevenlabs',
+      credentials: {
+        api_key: process.env.ELEVENLABS_API_KEY,
+        model_id: process.env.ELEVENLABS_MODEL_ID
+      },
+      language: 'en-US',
+      voice: process.env.ELEVENLABS_VOICE_ID,
+      text,
+    });
+    t.ok(!opts.servedFromCache, `successfully synthesized eleven audio to ${opts.filePath}`);
+
+  } catch (err) {
+    console.error(JSON.stringify(err));
+    t.end(err);
+  }
+  client.quit();
+})
+
 test('TTS Cache tests', async(t) => {
   const fn = require('..');
   const {purgeTtsCache, getTtsSize, client} = fn(opts, logger);