From e848d0276e505410a067caa64f8387bddc4a7bb9 Mon Sep 17 00:00:00 2001 From: Dave Horton Date: Tue, 4 Aug 2026 09:02:07 -0400 Subject: [PATCH] feat(tts): add gradium as a TTS vendor (#151) Streaming arm returns a say: url for the mediajam dialect; the cache-render arm posts to /api/post/speech/tts with only_audio and pcm_8000, which is bare r8 samples and avoids gradium's streaming wav header (0xffffffff RIFF size). Co-authored-by: Claude Opus 5 --- CLAUDE.md | 84 ++++++++++++++++++++++++++++++++++++++++++++++ lib/synth-audio.js | 67 +++++++++++++++++++++++++++++++++++- test/synth.js | 29 ++++++++++++++++ 3 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ec91593 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,84 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +`@jambonz/speech-utils` is a Node.js library providing TTS (Text-to-Speech) utilities for the jambonz CPaaS platform. It handles speech synthesis with caching through Redis and supports multiple TTS vendors. + +## Commands + +```bash +# Run tests (requires Docker for Redis) +npm test + +# Run linter +npm run jslint + +# Auto-fix lint issues +npm run jslint:fix + +# Generate coverage report +npm run coverage +``` + +## Testing + +Tests use `tape` and require Redis. The test harness automatically starts/stops Redis via Docker Compose (`test/docker-compose-testbed.yaml`). + +Most tests are conditional based on environment variables for vendor credentials: +- `GCP_FILE` or `GCP_JSON_KEY` - Google TTS +- `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION` - AWS Polly +- `MICROSOFT_API_KEY`, `MICROSOFT_REGION` - Azure TTS +- `ELEVENLABS_API_KEY`, `ELEVENLABS_VOICE_ID`, `ELEVENLABS_MODEL_ID` - ElevenLabs +- `OPENAI_API_KEY` - OpenAI Whisper TTS +- And others per vendor + +Redis config is in `config/test.json` (port 3379). + +## Architecture + +### Entry Point + +`index.js` exports a factory function that takes Redis options and a logger, returning an object with these methods: +- `synthAudio` - Main synthesis function +- `getTtsVoices` - List available voices for a vendor +- `purgeTtsCache` / `getTtsSize` / `addFileToCache` - Cache management +- `getAwsAuthToken` - Token management + +### Core Module: `lib/synth-audio.js` + +The `synthAudio` function handles synthesis for all vendors. Key behaviors: +1. **Cache check**: Generates SHA1 hash key from (vendor, language, voice, engine, model, text, instructions) +2. **Streaming vs non-streaming**: When `JAMBONES_DISABLE_TTS_STREAMING` is not set and `renderForCaching=false`, returns `say:{params}text` format for FreeSWITCH streaming playback instead of generating files +3. **Vendor dispatch**: Switch statement routes to vendor-specific synth functions (`synthGoogle`, `synthPolly`, `synthMicrosoft`, etc.) +4. **Caching**: Stores audio as base64 JSON in Redis with configurable TTL (default 4 hours) + +### Supported Vendors + +google, aws/polly, microsoft/azure, nvidia (Riva), wellsaid, elevenlabs, cartesia, inworld, rimelabs, whisper (OpenAI), deepgram, resemble, custom:* + +### gRPC Stubs + +`stubs/riva/` contains generated protobuf/gRPC code for NVIDIA Riva. + +## Environment Variables + +Key configuration via env vars (see `lib/config.js`): +- `JAMBONES_DISABLE_TTS_STREAMING` - Force non-streaming mode +- `JAMBONES_DISABLE_AZURE_TTS_STREAMING` - Azure-specific streaming disable +- `JAMBONES_TTS_CACHE_DURATION_MINS` - Cache TTL in minutes (default: 240) +- `JAMBONES_TTS_TRIM_SILENCE` - Trim trailing silence from audio +- `JAMBONES_TMP_FOLDER` - Temp folder for audio files (default: /tmp) +- `JAMBONES_HTTP_PROXY_IP`, `JAMBONES_HTTP_PROXY_PORT` - HTTP proxy for Azure +- `JAMBONES_AZURE_ENABLE_SSML` - Force SSML wrapper for Azure plain text + +## Key Dependencies + +- `@jambonz/realtimedb-helpers` - Redis client and hash utilities +- `@google-cloud/text-to-speech` - Google TTS +- `@aws-sdk/client-polly` - AWS Polly +- `microsoft-cognitiveservices-speech-sdk` - Azure TTS +- `@grpc/grpc-js` - gRPC for Riva +- `openai` - OpenAI Whisper TTS +- `bent` - HTTP client for REST-based vendors diff --git a/lib/synth-audio.js b/lib/synth-audio.js index bd8c38a..6601812 100644 --- a/lib/synth-audio.js +++ b/lib/synth-audio.js @@ -80,7 +80,7 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc logger = logger || noopLogger; assert.ok(['google', 'aws', 'polly', 'microsoft', 'wellsaid', 'nvidia', 'elevenlabs', - 'whisper', 'deepgram', 'deepgramflux', 'rimelabs', 'cartesia', 'nineninesix', 'inworld', 'resemble', + 'whisper', 'deepgram', 'deepgramflux', 'rimelabs', 'cartesia', 'gradium', 'nineninesix', 'inworld', 'resemble', 'murf', 'xai'] .includes(vendor) || vendor.startsWith('custom'), @@ -136,6 +136,9 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc } else if ('cartesia' === vendor) { assert.ok(credentials.api_key, 'synthAudio requires api_key when cartesia is used'); assert.ok(credentials.model_id, 'synthAudio requires model_id when cartesia is used'); + } else if ('gradium' === vendor) { + assert.ok(voice, 'synthAudio requires voice when gradium is used'); + assert.ok(credentials.api_key, 'synthAudio requires api_key when gradium is used'); } else if ('nineninesix' === vendor) { assert.ok(voice, 'synthAudio requires voice when nineninesix is used'); assert.ok(credentials.api_key, 'synthAudio requires api_key when nineninesix is used'); @@ -217,6 +220,11 @@ async function synthAudio(client, createHash, retrieveHash, logger, stats, { acc credentials, options, stats, language, voice, key, text, renderForCaching, disableTtsStreaming, disableTtsCache}); break; + case 'gradium': + audioData = await synthGradium(logger, { + credentials, options, stats, voice, key, text, renderForCaching, disableTtsStreaming, + disableTtsCache}); + break; case 'nineninesix': audioData = await synthNineninesix(logger, { credentials, stats, language, voice, key, text, renderForCaching, disableTtsStreaming, @@ -1433,6 +1441,63 @@ const synthCartesia = async(logger, { }; +/* gradium.ai — json websocket for streaming, and a POST endpoint for the cache + render. only_audio:true + pcm_8000 returns bare little-endian 16-bit samples, + which is exactly the r8 container, so we avoid gradium's streaming wav header + (it carries 0xffffffff as the RIFF size, since length is unknown up front). +*/ +const synthGradium = async(logger, { + credentials, options, stats, voice, key, text, renderForCaching, disableTtsStreaming, disableTtsCache +}) => { + const {api_key, model_id} = credentials; + const {json_config, pronunciation_id} = options || {}; + + /* default to using the streaming interface, unless disabled by env var OR we want just a cache file */ + if (!JAMBONES_DISABLE_TTS_STREAMING && !renderForCaching && !disableTtsStreaming) { + let params = '{'; + params += `api_key=${api_key}`; + params += `,playback_id=${key}`; + params += ',vendor=gradium'; + params += `,voice=${voice}`; + params += `,write_cache_file=${disableTtsCache ? 0 : 1}`; + if (model_id) params += `,model_id=${model_id}`; + if (pronunciation_id) params += `,pronunciation_id=${pronunciation_id}`; + params += '}'; + + return { + filePath: `say:${params}${text.replace(/\n/g, ' ').replace(/\r/g, ' ')}`, + servedFromCache: false, + rtt: 0 + }; + } + + try { + const sampleRate = 8000; + const post = bent('https://api.gradium.ai', 'POST', 'buffer', { + 'x-api-key': api_key, + 'Content-Type': 'application/json' + }); + const audioContent = await post('/api/post/speech/tts', { + text, + voice_id: voice, + model_name: model_id || 'default', + output_format: `pcm_${sampleRate}`, + only_audio: true, + ...(json_config && {json_config}), + ...(pronunciation_id && {pronunciation_id}) + }); + return { + audioContent, + extension: 'r8', + sampleRate + }; + } catch (err) { + logger.info({err}, 'synth gradium returned error'); + stats.increment('tts.count', ['vendor:gradium', 'accepted:no']); + throw err; + } +}; + /* nineninesix.ai — a Cartesia-compatible API, but only raw/wav come back (mp3 is rejected), so the cache render asks for wav rather than mp3. */ const synthNineninesix = async(logger, { diff --git a/test/synth.js b/test/synth.js index ecab513..cc9aca3 100644 --- a/test/synth.js +++ b/test/synth.js @@ -1058,6 +1058,35 @@ test('Cartesia speech synth tests', async(t) => { client.quit(); }); +test('gradium speech synth tests', async(t) => { + const fn = require('..'); + const {synthAudio, client} = fn(opts, logger); + + if (!process.env.GRADIUM_API_KEY) { + t.pass('skipping gradium speech synth tests since GRADIUM_API_KEY is not provided'); + return t.end(); + } + const text = 'Hi there and welcome to jambones! ' + Date.now(); + try { + const opts = await synthAudio(stats, { + vendor: 'gradium', + credentials: { + api_key: process.env.GRADIUM_API_KEY, + model_id: 'default' + }, + voice: 'YTpq7expH9539ERJ', + text, + renderForCaching: true + }); + t.ok(!opts.servedFromCache, `successfully synthed gradium audio to ${opts.filePath}`); + + } catch (err) { + console.error(JSON.stringify(err)); + t.end(err); + } + client.quit(); +}); + test('nineninesix speech synth tests', async(t) => { const fn = require('..'); const {synthAudio, client} = fn(opts, logger);