0.0.13

fixes for riva tts

lib/get-tts-voices.js

@@ -1,9 +1,11 @@
 const assert = require('assert');
-const {noopLogger, createNuanceClient} = require('./utils');
+const {noopLogger, createNuanceClient, createKryptonClient} = require('./utils');
 const getNuanceAccessToken = require('./get-nuance-access-token');
 const {GetVoicesRequest, Voice} = require('../stubs/nuance/synthesizer_pb');
 const TextToSpeechV1 = require('ibm-watson/text-to-speech/v1');
 const { IamAuthenticator } = require('ibm-watson/auth');
+const ttsGoogle = require('@google-cloud/text-to-speech');
+const { PollyClient, DescribeVoicesCommand } = require('@aws-sdk/client-polly');
 
 const getIbmVoices = async(client, logger, credentials) => {
   const {tts_region, tts_api_key} = credentials;
@@ -21,15 +23,20 @@ const getIbmVoices = async(client, logger, credentials) => {
 };
 
 const getNuanceVoices = async(client, logger, credentials) => {
-  const {client_id: clientId, secret: secret} = credentials;
+  const {client_id: clientId, secret: secret, nuance_tts_uri} = credentials;
 
   return new Promise(async(resolve, reject) => {
     /* get a nuance access token */
     let token, nuanceClient;
     try {
-      const access_token = await getNuanceAccessToken(client, logger, clientId, secret, 'tts');
-      token = access_token.access_token;
-      nuanceClient = await createNuanceClient(token);
+      if (nuance_tts_uri) {
+        nuanceClient = await createKryptonClient(nuance_tts_uri);
+      }
+      else {
+        const access_token = await getNuanceAccessToken(client, logger, clientId, secret, 'tts');
+        token = access_token.access_token;
+        nuanceClient = await createNuanceClient(token);
+      }
     } catch (err) {
       logger.error({err}, 'getTtsVoices: error retrieving access token');
       return reject(err);
@@ -75,6 +82,30 @@ const getNuanceVoices = async(client, logger, credentials) => {
   });
 };
 
+const getGoogleVoices = async(_client, logger, credentials) => {
+  const client = new ttsGoogle.TextToSpeechClient({credentials});
+  return await client.listVoices();
+};
+
+const getAwsVoices = async(_client, logger, credentials) => {
+  try {
+    const {region, accessKeyId, secretAccessKey} = credentials;
+    const client = new PollyClient({
+      region,
+      credentials: {
+        accessKeyId,
+        secretAccessKey
+      }
+    });
+    const command = new DescribeVoicesCommand({LanguageCode: 'en-US'});
+    const response = await client.send(command);
+    return response;
+  } catch (err) {
+    logger.info({err}, 'testMicrosoftTts - failed to list voices for region ${region}');
+    throw err;
+  }
+};
+
 /**
  * Synthesize speech to an mp3 file, and also cache the generated speech
  * in redis (base64 format) for 24 hours so as to avoid unnecessarily paying
@@ -94,7 +125,7 @@ const getNuanceVoices = async(client, logger, credentials) => {
 async function getTtsVoices(client, logger, {vendor, credentials}) {
   logger = logger || noopLogger;
 
-  assert.ok(['nuance', 'ibm'].includes(vendor),
+  assert.ok(['nuance', 'ibm', 'google', 'aws', 'polly'].includes(vendor),
     `getTtsVoices not supported for vendor ${vendor}`);
 
   switch (vendor) {
@@ -102,6 +133,11 @@ async function getTtsVoices(client, logger, {vendor, credentials}) {
       return getNuanceVoices(client, logger, credentials);
     case 'ibm':
       return getIbmVoices(client, logger, credentials);
+    case 'google':
+      return getGoogleVoices(client, logger, credentials);
+    case 'aws':
+    case 'polly':
+      return getAwsVoices(client, logger, credentials);
     default:
       break;
   }

lib/synth-audio.js

@@ -2,21 +2,19 @@ const assert = require('assert');
 const fs = require('fs');
 const bent = require('bent');
 const ttsGoogle = require('@google-cloud/text-to-speech');
-//const Polly = require('aws-sdk/clients/polly');
 const { PollyClient, SynthesizeSpeechCommand } = require('@aws-sdk/client-polly');
 
 const sdk = require('microsoft-cognitiveservices-speech-sdk');
 const TextToSpeechV1 = require('ibm-watson/text-to-speech/v1');
 const { IamAuthenticator } = require('ibm-watson/auth');
 const {
-  AudioConfig,
   ResultReason,
   SpeechConfig,
   SpeechSynthesizer,
   CancellationDetails,
   SpeechSynthesisOutputFormat
 } = sdk;
-const {makeSynthKey, createNuanceClient, noopLogger, createRivaClient} = require('./utils');
+const {makeSynthKey, createNuanceClient, createKryptonClient, createRivaClient, noopLogger} = require('./utils');
 const getNuanceAccessToken = require('./get-nuance-access-token');
 const {
   SynthesisRequest,
@@ -75,13 +73,15 @@ async function synthAudio(client, logger, stats, { account_sid,
   }
   else if ('nuance' === vendor) {
     assert.ok(voice, 'synthAudio requires voice when nuance is used');
-    assert.ok(credentials.client_id, 'synthAudio requires client_id in credentials when nuance is used');
-    assert.ok(credentials.secret, 'synthAudio requires client_id in credentials when nuance is used');
+    if (!credentials.nuance_tts_uri) {
+      assert.ok(credentials.client_id, 'synthAudio requires client_id in credentials when nuance is used');
+      assert.ok(credentials.secret, 'synthAudio requires client_id in credentials when nuance is used');
+    }
   }
   else if ('nvidia' === vendor) {
     assert.ok(voice, 'synthAudio requires voice when nvidia is used');
     assert.ok(language, 'synthAudio requires language when nvidia is used');
-    assert.ok(credentials.riva_uri, 'synthAudio requires riva_uri in credentials when nuance is used');
+    assert.ok(credentials.riva_server_uri, 'synthAudio requires riva_server_uri in credentials when nvidia is used');
   }
   else if ('ibm' === vendor) {
     assert.ok(voice, 'synthAudio requires voice when ibm is used');
@@ -170,8 +170,6 @@ async function synthAudio(client, logger, stats, { account_sid,
 
     client.setexAsync(key, EXPIRES, audioBuffer.toString('base64'))
       .catch((err) => logger.error(err, `error calling setex on key ${key}`));
-
-    if (['microsoft'].includes(vendor)) return {filePath, servedFromCache, rtt};
   }
 
   return new Promise((resolve, reject) => {
@@ -184,7 +182,14 @@ async function synthAudio(client, logger, stats, { account_sid,
 
 const synthPolly = async(logger, {credentials, stats, language, voice, engine, text}) => {
   try {
-    const polly = new PollyClient({credentials});
+    const {region, accessKeyId, secretAccessKey} = credentials;
+    const polly = new PollyClient({
+      region,
+      credentials: {
+        accessKeyId,
+        secretAccessKey
+      }
+    });
     const opts = {
       Engine: engine,
       OutputFormat: 'mp3',
@@ -293,8 +298,7 @@ const synthMicrosoft = async(logger, {
       if (!content.startsWith('<speak')) content = `<speak>${text}</speak>`;
     }
     speechConfig.speechSynthesisOutputFormat = SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3;
-    const config = AudioConfig.fromAudioFileOutput(filePath);
-    const synthesizer = new SpeechSynthesizer(speechConfig, config);
+    const synthesizer = new SpeechSynthesizer(speechConfig);
 
     if (content.startsWith('<speak>')) {
       /* microsoft enforces some properties and uses voice xml element so if the user did not supply do it for them */
@@ -319,12 +323,9 @@ const synthMicrosoft = async(logger, {
               reject(cancellation.errorDetails);
               break;
             case ResultReason.SynthesizingAudioCompleted:
-              stats.increment('tts.count', ['vendor:microsoft', 'accepted:yes']);
+              resolve(Buffer.from(result.audioData));
               synthesizer.close();
-              fs.readFile(filePath, (err, data) => {
-                if (err) return reject(err);
-                resolve(data);
-              });
+              stats.increment('tts.count', ['vendor:microsoft', 'accepted:yes']);
               break;
             default:
               logger.info({result}, 'synthAudio: (Microsoft) unexpected result');
@@ -364,10 +365,16 @@ const synthWellSaid = async(logger, {credentials, stats, language, voice, gender
 };
 
 const synthNuance = async(client, logger, {credentials, stats, voice, model, text}) => {
-  /* get a nuance access token */
-  const {client_id, secret} = credentials;
-  const {access_token} = await getNuanceAccessToken(client, logger, client_id, secret, 'tts');
-  const nuanceClient = await createNuanceClient(access_token);
+  let nuanceClient;
+  const {client_id, secret, nuance_tts_uri} = credentials;
+  if (nuance_tts_uri) {
+    nuanceClient = await createKryptonClient(nuance_tts_uri);
+  }
+  else {
+    /* get a nuance access token */
+    const {access_token} = await getNuanceAccessToken(client, logger, client_id, secret, 'tts');
+    nuanceClient = await createNuanceClient(access_token);
+  }
 
   const v = new Voice();
   const p = new AudioParameters();
@@ -418,21 +425,25 @@ const synthNuance = async(client, logger, {credentials, stats, voice, model, tex
 };
 
 const synthNvidia = async(client, logger, {credentials, stats, language,  voice, model, text}) => {
-  const {riva_uri} = credentials;
-  const rivaClient = await createRivaClient(riva_uri);
-
-  const request = new SynthesizeSpeechRequest();
-  request.setVoiceName(voice);
-  request.setLanguageCode(language);
-  request.setSampleRateHz(8000);
-  request.setEncoding(AudioEncoding.LINEAR_PCM);
-  request.setText(text);
+  const {riva_server_uri} = credentials;
+  let rivaClient, request;
+  try {
+    rivaClient = await createRivaClient(riva_server_uri);
+    request = new SynthesizeSpeechRequest();
+    request.setVoiceName(voice);
+    request.setLanguageCode(language);
+    request.setSampleRateHz(8000);
+    request.setEncoding(AudioEncoding.LINEAR_PCM);
+    request.setText(text);
+  } catch (err) {
+    logger.info({err}, 'error creating riva client');
+    return Promise.reject(err);
+  }
 
   return new Promise((resolve, reject) => {
-    console.log(`language ${language} voice ${voice} model ${model} text ${text}`);
     rivaClient.synthesize(request, (err, response) => {
       if (err) {
-        console.error(err);
+        logger.info({err, voice, language}, 'error synthesizing speech using Nvidia');
         return reject(err);
       }
       resolve(Buffer.from(response.getAudio()));

lib/utils.js

@@ -77,6 +77,11 @@ const getNuanceAccessToken = async(clientId, secret, scope = 'asr tts') => {
   return json.access_token;
 };
 
+const createKryptonClient = async(uri) => {
+  const client = new SynthesizerClient(uri, grpc.credentials.createInsecure());
+  return client;
+};
+
 const createNuanceClient = async(access_token) => {
 
   //if (nuanceClientMap.has(access_token)) return nuanceClientMap.get(access_token);
@@ -108,6 +113,7 @@ module.exports = {
   makeIbmKey,
   getNuanceAccessToken,
   createNuanceClient,
+  createKryptonClient,
   createRivaClient,
   makeBasicAuthHeader,
   NUANCE_AUTH_ENDPOINT,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jambonz/speech-utils",
-  "version": "0.0.4",
+  "version": "0.0.13",
   "description": "TTS-related speech utilities for jambonz",
   "main": "index.js",
   "author": "Dave Horton",
@@ -24,18 +24,18 @@
   },
   "homepage": "https://github.com/jambonz/speech-utils#readme",
   "dependencies": {
-    "@aws-sdk/client-polly": "^3.276.0",
-    "@google-cloud/text-to-speech": "^4.2.0",
-    "@grpc/grpc-js": "^1.8.7",
-    "@jambonz/realtimedb-helpers": "^0.6.3",
-    "aws-sdk": "^2.1310.0",
+    "@aws-sdk/client-polly": "^3.303.0",
+    "@google-cloud/text-to-speech": "^4.2.1",
+    "@grpc/grpc-js": "^1.8.13",
+    "@jambonz/promisify-redis": "^0.0.6",
     "bent": "^7.3.12",
     "debug": "^4.3.4",
-    "google-protobuf": "^3.21.2",
-    "ibm-watson": "^7.1.2",
     "form-urlencoded": "^6.1.0",
-    "microsoft-cognitiveservices-speech-sdk": "^1.25.0",
-    "undici": "^5.19.1"
+    "google-protobuf": "^3.21.2",
+    "ibm-watson": "^8.0.0",
+    "microsoft-cognitiveservices-speech-sdk": "^1.26.0",
+    "redis": "^3.1.2",
+    "undici": "^5.21.0"
   },
   "devDependencies": {
     "config": "^3.3.9",

test/index.js

@@ -1,5 +1,4 @@
 require('./docker_start');
 require('./synth');
-require('./nuance');
-require('./ibm');
+require('./list-voices');
 require('./docker_stop');

test/list-voices.js

@@ -0,0 +1,205 @@
+const test = require('tape').test ;
+const config = require('config');
+const opts = config.get('redis');
+const fs = require('fs');
+const logger = require('pino')({level: 'error'});
+process.on('unhandledRejection', (reason, p) => {
+  console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+});
+
+const stats = {
+  increment: () => {},
+  histogram: () => {}
+};
+
+test('IBM - create access key', async(t) => {
+  const fn = require('..');
+  const {client, getIbmAccessToken} = fn(opts, logger);
+
+  if (!process.env.IBM_API_KEY ) {
+      t.pass('skipping IBM test since no IBM api_key provided');
+      t.end();
+      client.quit();
+      return;
+  }
+  try {
+    let obj = await getIbmAccessToken(process.env.IBM_API_KEY);
+    //console.log({obj}, 'received access token from IBM');
+    t.ok(obj.access_token && !obj.servedFromCache, 'successfull received access token from IBM');
+
+    obj = await getIbmAccessToken(process.env.IBM_API_KEY);
+    //console.log({obj}, 'received access token from IBM - second request');
+    t.ok(obj.access_token && obj.servedFromCache, 'successfully received access token from cache');
+ 
+    await client.flushallAsync();
+    t.end();
+  }
+  catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+  client.quit();
+});
+
+test('IBM - retrieve tts voices test', async(t) => {
+  const fn = require('..');
+  const {client, getTtsVoices} = fn(opts, logger);
+
+  if (!process.env.IBM_TTS_API_KEY || !process.env.IBM_TTS_REGION) {
+      t.pass('skipping IBM test since no IBM api_key and/or region provided');
+      t.end();
+      client.quit();
+      return;
+  }
+  try {
+    const opts = {
+      vendor: 'ibm',
+      credentials: {
+        tts_api_key: process.env.IBM_TTS_API_KEY,
+        tts_region: process.env.IBM_TTS_REGION
+      }
+    };
+    const obj = await getTtsVoices(opts);
+    const {voices} = obj.result;
+    //console.log(JSON.stringify(voices));
+    t.ok(voices.length > 0 && voices[0].language, 
+      `GetVoices: successfully retrieved ${voices.length} voices from IBM`);
+ 
+    await client.flushallAsync();
+
+    t.end();
+
+  }
+  catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+  client.quit();
+});
+
+test('Nuance hosted tests', async(t) => {
+  const fn = require('..');
+  const {client, getTtsVoices} = fn(opts, logger);
+
+  if (!process.env.NUANCE_CLIENT_ID || !process.env.NUANCE_SECRET ) {
+      t.pass('skipping Nuance hosted test since no Nuance client_id and secret provided');
+      t.end();
+      client.quit();
+      return;
+  }
+  try {
+    const opts = {
+      vendor: 'nuance',
+      credentials: {
+        client_id: process.env.NUANCE_CLIENT_ID,
+        secret: process.env.NUANCE_SECRET
+      }
+    };
+    let voices = await getTtsVoices(opts);
+    t.ok(voices.length > 0 && voices[0].language, 
+      `GetVoices: successfully retrieved ${voices.length} voices from Nuance`);
+
+    await client.flushallAsync();
+
+    t.end();
+
+  }
+  catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+  client.quit();
+});
+
+test('Nuance on-prem tests', async(t) => {
+  const fn = require('..');
+  const {client, getTtsVoices} = fn(opts, logger);
+
+  if (!process.env.NUANCE_TTS_URI ) {
+      t.pass('skipping Nuance on-prem test since no Nuance uri provided');
+      t.end();
+      client.quit();
+      return;
+  }
+  try {
+    const opts = {
+      vendor: 'nuance',
+      credentials: {
+        nuance_tts_uri: process.env.NUANCE_TTS_URI
+      }
+    };
+    let voices = await getTtsVoices(opts);
+    t.ok(voices.length > 0 && voices[0].language, 
+      `GetVoices: successfully retrieved ${voices.length} voices from Nuance`);
+
+    await client.flushallAsync();
+
+    t.end();
+
+  }
+  catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+  client.quit();
+});
+
+test('Google tests', async(t) => {
+  const fn = require('..');
+  const {client, getTtsVoices} = fn(opts, logger);
+
+  if (!process.env.GCP_FILE && !process.env.GCP_JSON_KEY) {
+    t.pass('skipping google speech synth tests since neither GCP_FILE nor GCP_JSON_KEY provided');
+    return t.end();
+  }
+  try {
+    const str = process.env.GCP_JSON_KEY || fs.readFileSync(process.env.GCP_FILE);
+    const credentials = JSON.parse(str);
+    const opts = {
+      vendor: 'google',
+      credentials
+    };
+    let result = await getTtsVoices(opts);
+    t.ok(result[0].voices.length > 0, `GetVoices: successfully retrieved ${result[0].voices.length} voices from Google`);
+
+    await client.flushallAsync();
+
+    t.end();
+  }
+  catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+  client.quit();
+});
+
+test('AWS tests', async(t) => {
+  const fn = require('..');
+  const {client, getTtsVoices} = fn(opts, logger);
+
+  if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY || !process.env.AWS_REGION) {
+    t.pass('skipping AWS speech synth tests since AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, or AWS_REGION not provided');
+    return t.end();
+  }
+  try {
+    const opts = {
+      vendor: 'aws',
+      credentials: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+        region: process.env.AWS_REGION,
+      }
+    };
+    let result = await getTtsVoices(opts);
+    t.ok(result?.Voices?.length > 0, `GetVoices: successfully retrieved ${result.Voices.length} voices from AWS`);
+
+    await client.flushallAsync();
+
+    t.end();
+  }
+  catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+  client.quit();
+});

test/nuance.js

@@ -12,12 +12,12 @@ const stats = {
   histogram: () => {}
 };
 
-test('Nuance tests', async(t) => {
+test('Nuance hosted tests', async(t) => {
   const fn = require('..');
   const {client, getTtsVoices} = fn(opts, logger);
 
   if (!process.env.NUANCE_CLIENT_ID || !process.env.NUANCE_SECRET ) {
-      t.pass('skipping Nuance test since no Nuance client_id and secret provided');
+      t.pass('skipping Nuance hosted test since no Nuance client_id and secret provided');
       t.end();
       client.quit();
       return;
@@ -31,8 +31,39 @@ test('Nuance tests', async(t) => {
       }
     };
     let voices = await getTtsVoices(opts);
-    //console.log(`received ${voices.length} voices from Nuance`);
-    //console.log(JSON.stringify(voices));
+    t.ok(voices.length > 0 && voices[0].language, 
+      `GetVoices: successfully retrieved ${voices.length} voices from Nuance`);
+
+    await client.flushallAsync();
+
+    t.end();
+
+  }
+  catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+  client.quit();
+});
+
+test('Nuance on-prem tests', async(t) => {
+  const fn = require('..');
+  const {client, getTtsVoices} = fn(opts, logger);
+
+  if (!process.env.NUANCE_TTS_URI ) {
+      t.pass('skipping Nuance on-prem test since no Nuance uri provided');
+      t.end();
+      client.quit();
+      return;
+  }
+  try {
+    const opts = {
+      vendor: 'nuance',
+      credentials: {
+        nuance_tts_uri: process.env.NUANCE_TTS_URI
+      }
+    };
+    let voices = await getTtsVoices(opts);
     t.ok(voices.length > 0 && voices[0].language, 
       `GetVoices: successfully retrieved ${voices.length} voices from Nuance`);
 

test/synth.js

@@ -212,7 +212,7 @@ test('Azure custom voice speech synth tests', async(t) => {
   client.quit();
 });
 
-test('Nuance speech synth tests', async(t) => {
+test('Nuance hosted speech synth tests', async(t) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);
 
@@ -251,6 +251,43 @@ test('Nuance speech synth tests', async(t) => {
   client.quit();
 });
 
+test('Nuance on-prem speech synth tests', async(t) => {
+  const fn = require('..');
+  const {synthAudio, client} = fn(opts, logger);
+
+  if (!process.env.NUANCE_TTS_URI) {
+    t.pass('skipping Nuance on prem speech synth tests since NUANCE_TTS_URI not provided');
+    return t.end();
+  }
+  try {
+    let opts = await synthAudio(stats, {
+      vendor: 'nuance',
+      credentials: {
+        nuance_tts_uri: process.env.NUANCE_TTS_URI
+      },
+      language: 'en-US',
+      voice: 'Evan',
+      text: 'This is a test of on-prem.  This is only a test',
+    });
+    t.ok(!opts.servedFromCache, `successfully synthesized nuance audio to ${opts.filePath}`);
+
+    opts = await synthAudio(stats, {
+      vendor: 'nuance',
+      credentials: {
+        nuance_tts_uri: process.env.NUANCE_TTS_URI
+      },
+      language: 'en-US',
+      voice: 'Evan',
+      text: 'This is a test of on-prem.  This is only a test',
+    });
+    t.ok(opts.servedFromCache, `successfully retrieved nuance audio from cache ${opts.filePath}`);
+  } catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+  client.quit();
+});
+
 test('Nvidia speech synth tests', async(t) => {
   const fn = require('..');
   const {synthAudio, client} = fn(opts, logger);
@@ -263,7 +300,7 @@ test('Nvidia speech synth tests', async(t) => {
     let opts = await synthAudio(stats, {
       vendor: 'nvidia',
       credentials: {
-        riva_uri: process.env.RIVA_URI,
+        riva_server_uri: process.env.RIVA_URI,
       },
       language: 'en-US',
       voice: 'English-US.Female-1',
@@ -274,7 +311,7 @@ test('Nvidia speech synth tests', async(t) => {
     opts = await synthAudio(stats, {
       vendor: 'nvidia',
       credentials: {
-        riva_uri: process.env.RIVA_URI,
+        riva_server_uri: process.env.RIVA_URI,
       },
       language: 'en-US',
       voice: 'English-US.Female-1',