mirror of
https://github.com/jambonz/jambonz-feature-server.git
synced 2026-02-14 18:30:59 +00:00
Fix/0.8.5 cherries (#724)
* kill play task if bot responds verbs while actionHook delay is enabled (#712) * kill play task if bot responds verbs while actionHook delay is enabled * fix actionHook delay continues even the bot already responded verbs * wip * wip * wip * gather is hang if listenDuringPrompt = false and say/play task throw exception (#717) * merge fix for Support ASR TTS fallback (#713) --------- Co-authored-by: Hoan Luu Huu <110280845+xquanluu@users.noreply.github.com>
This commit is contained in:
Dave Horton
167
lib/tasks/say.js
167
lib/tasks/say.js
@@ -52,6 +52,151 @@ class TaskSay extends TtsTask {
|
||||
return `${this.name}{${this.text[0]}}`;
|
||||
}
|
||||
|
||||
_validateURL(urlString) {
|
||||
try {
|
||||
new URL(urlString);
|
||||
return true;
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label, preCache = false}) {
|
||||
const {srf, accountSid:account_sid} = cs;
|
||||
const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, srf);
|
||||
const {writeAlerts, AlertType, stats} = srf.locals;
|
||||
const {synthAudio} = srf.locals.dbHelpers;
|
||||
const engine = this.synthesizer.engine || 'standard';
|
||||
const salt = cs.callSid;
|
||||
|
||||
let credentials = cs.getSpeechCredentials(vendor, 'tts', label);
|
||||
/* parse Nuance voices into name and model */
|
||||
let model;
|
||||
if (vendor === 'nuance' && voice) {
|
||||
const arr = /([A-Za-z-]*)\s+-\s+(enhanced|standard)/.exec(voice);
|
||||
if (arr) {
|
||||
voice = arr[1];
|
||||
model = arr[2];
|
||||
}
|
||||
} else if (vendor === 'deepgram') {
|
||||
model = voice;
|
||||
}
|
||||
|
||||
/* allow for microsoft custom region voice and api_key to be specified as an override */
|
||||
if (vendor === 'microsoft' && this.options.deploymentId) {
|
||||
credentials = credentials || {};
|
||||
credentials.use_custom_tts = true;
|
||||
credentials.custom_tts_endpoint = this.options.deploymentId;
|
||||
credentials.api_key = this.options.apiKey || credentials.apiKey;
|
||||
credentials.region = this.options.region || credentials.region;
|
||||
voice = this.options.voice || voice;
|
||||
} else if (vendor === 'elevenlabs') {
|
||||
credentials = credentials || {};
|
||||
credentials.model_id = this.options.model_id || credentials.model_id;
|
||||
credentials.voice_settings = this.options.voice_settings || {};
|
||||
credentials.optimize_streaming_latency = this.options.optimize_streaming_latency
|
||||
|| credentials.optimize_streaming_latency;
|
||||
voice = this.options.voice_id || voice;
|
||||
}
|
||||
|
||||
ep.set({
|
||||
tts_engine: vendor,
|
||||
tts_voice: voice,
|
||||
cache_speech_handles: 1,
|
||||
}).catch((err) => this.logger.info({err}, 'Error setting tts_engine on endpoint'));
|
||||
|
||||
if (!preCache) this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
|
||||
try {
|
||||
if (!credentials) {
|
||||
writeAlerts({
|
||||
account_sid,
|
||||
alert_type: AlertType.TTS_NOT_PROVISIONED,
|
||||
vendor
|
||||
}).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
|
||||
throw new Error('no provisioned speech credentials for TTS');
|
||||
}
|
||||
// synthesize all of the text elements
|
||||
let lastUpdated = false;
|
||||
|
||||
/* produce an audio segment from the provided text */
|
||||
const generateAudio = async(text) => {
|
||||
if (this.killed) return;
|
||||
if (text.startsWith('silence_stream://')) return text;
|
||||
|
||||
/* otel: trace time for tts */
|
||||
if (!preCache) {
|
||||
const {span} = this.startChildSpan('tts-generation', {
|
||||
'tts.vendor': vendor,
|
||||
'tts.language': language,
|
||||
'tts.voice': voice
|
||||
});
|
||||
this.otelSpan = span;
|
||||
}
|
||||
try {
|
||||
const {filePath, servedFromCache, rtt} = await synthAudio(stats, {
|
||||
account_sid,
|
||||
text,
|
||||
vendor,
|
||||
language,
|
||||
voice,
|
||||
engine,
|
||||
model,
|
||||
salt,
|
||||
credentials,
|
||||
options: this.options,
|
||||
disableTtsCache : this.disableTtsCache,
|
||||
preCache
|
||||
});
|
||||
if (!filePath.startsWith('say:')) {
|
||||
this.logger.debug(`file ${filePath}, served from cache ${servedFromCache}`);
|
||||
if (filePath) cs.trackTmpFile(filePath);
|
||||
if (this.otelSpan) {
|
||||
this.otelSpan.setAttributes({'tts.cached': servedFromCache});
|
||||
this.otelSpan.end();
|
||||
this.otelSpan = null;
|
||||
}
|
||||
if (!servedFromCache && !lastUpdated) {
|
||||
lastUpdated = true;
|
||||
updateSpeechCredentialLastUsed(credentials.speech_credential_sid).catch(() => {/* logged error */});
|
||||
}
|
||||
if (!servedFromCache && rtt && !preCache) {
|
||||
this.notifyStatus({
|
||||
event: 'synthesized-audio',
|
||||
vendor,
|
||||
language,
|
||||
characters: text.length,
|
||||
elapsedTime: rtt
|
||||
});
|
||||
}
|
||||
}
|
||||
else {
|
||||
this.logger.debug('a streaming tts api will be used');
|
||||
const modifiedPath = filePath.replace('say:{', `say:{session-uuid=${ep.uuid},`);
|
||||
return modifiedPath;
|
||||
}
|
||||
return filePath;
|
||||
} catch (err) {
|
||||
this.logger.info({err}, 'Error synthesizing tts');
|
||||
if (this.otelSpan) this.otelSpan.end();
|
||||
writeAlerts({
|
||||
account_sid: cs.accountSid,
|
||||
alert_type: AlertType.TTS_FAILURE,
|
||||
vendor,
|
||||
detail: err.message
|
||||
}).catch((err) => this.logger.info({err}, 'Error generating alert for tts failure'));
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
const arr = this.text.map((t) => (this._validateURL(t) ? t : generateAudio(t)));
|
||||
return (await Promise.all(arr)).filter((fp) => fp && fp.length);
|
||||
} catch (err) {
|
||||
this.logger.info(err, 'TaskSay:exec error');
|
||||
throw err;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
async exec(cs, {ep}) {
|
||||
const {srf, accountSid:account_sid} = cs;
|
||||
const {writeAlerts, AlertType} = srf.locals;
|
||||
@@ -61,16 +206,16 @@ class TaskSay extends TtsTask {
|
||||
await super.exec(cs);
|
||||
this.ep = ep;
|
||||
|
||||
const vendor = this.synthesizer.vendor && this.synthesizer.vendor !== 'default' ?
|
||||
let vendor = this.synthesizer.vendor && this.synthesizer.vendor !== 'default' ?
|
||||
this.synthesizer.vendor :
|
||||
cs.speechSynthesisVendor;
|
||||
const language = this.synthesizer.language && this.synthesizer.language !== 'default' ?
|
||||
let language = this.synthesizer.language && this.synthesizer.language !== 'default' ?
|
||||
this.synthesizer.language :
|
||||
cs.speechSynthesisLanguage ;
|
||||
const voice = this.synthesizer.voice && this.synthesizer.voice !== 'default' ?
|
||||
let voice = this.synthesizer.voice && this.synthesizer.voice !== 'default' ?
|
||||
this.synthesizer.voice :
|
||||
cs.speechSynthesisVoice;
|
||||
const label = this.synthesizer.label && this.synthesizer.label !== 'default' ?
|
||||
let label = this.synthesizer.label && this.synthesizer.label !== 'default' ?
|
||||
this.synthesizer.label :
|
||||
cs.speechSynthesisLabel;
|
||||
|
||||
@@ -87,12 +232,22 @@ class TaskSay extends TtsTask {
|
||||
this.synthesizer.fallbackLabel :
|
||||
cs.fallbackSpeechSynthesisLabel;
|
||||
|
||||
if (cs.hasFallbackTts) {
|
||||
vendor = fallbackVendor;
|
||||
language = fallbackLanguage;
|
||||
voice = fallbackVoice;
|
||||
label = fallbackLabel;
|
||||
}
|
||||
|
||||
let filepath;
|
||||
try {
|
||||
filepath = await this._synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label});
|
||||
} catch (error) {
|
||||
if (fallbackVendor && this.isHandledByPrimaryProvider) {
|
||||
if (fallbackVendor && this.isHandledByPrimaryProvider && !cs.hasFallbackTts) {
|
||||
this.notifyError(
|
||||
{ msg: 'TTS error', details:`TTS vendor ${vendor} error: ${error}`, failover: 'in progress'});
|
||||
this.isHandledByPrimaryProvider = false;
|
||||
cs.hasFallbackTts = true;
|
||||
this.logger.info(`Synthesize error, fallback to ${fallbackVendor}`);
|
||||
filepath = await this._synthesizeWithSpecificVendor(cs, ep,
|
||||
{
|
||||
@@ -102,6 +257,8 @@ class TaskSay extends TtsTask {
|
||||
label: fallbackLabel
|
||||
});
|
||||
} else {
|
||||
this.notifyError(
|
||||
{ msg: 'TTS error', details:`TTS vendor ${vendor} error: ${error}`, failover: 'not available'});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user