diff --git a/lib/session/call-session.js b/lib/session/call-session.js index 72232449..4ec979dd 100644 --- a/lib/session/call-session.js +++ b/lib/session/call-session.js @@ -188,6 +188,24 @@ class CallSession extends Emitter { this._synthesizer = synth; } + /** + * ASR TTS fallback + */ + get hasFallbackAsr() { + return this._hasFallbackAsr || false; + } + + set hasFallbackAsr(i) { + this._hasFallbackAsr = i; + } + + get hasFallbackTts() { + return this._hasFallbackTts || false; + } + + set hasFallbackTts(i) { + this._hasFallbackTts = i; + } /** * default vendor to use for speech synthesis if not provided in the app */ @@ -1832,6 +1850,7 @@ Duration=${duration} ` /** * called when the caller has hung up. Provided for subclasses to override * in order to apply logic at this point if needed. + * return true if success fallback, return false if not */ _callerHungup() { assert(false, 'subclass responsibility to override this method'); diff --git a/lib/tasks/gather.js b/lib/tasks/gather.js index e9adb65c..644dfb48 100644 --- a/lib/tasks/gather.js +++ b/lib/tasks/gather.js @@ -174,12 +174,7 @@ class TaskGather extends SttTask { this._startTranscribing(ep); return updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid); } catch (e) { - if (this.fallbackVendor && this.isHandledByPrimaryProvider) { - await this._fallback(); - startListening(cs, ep); - } else { - this.logger.error({error: e}, 'error in initSpeech'); - } + await this._startFallback(cs, ep, {error: e}); } } }; @@ -906,9 +901,9 @@ class TaskGather extends SttTask { _onTranscriptionComplete(cs, ep) { this.logger.debug('TaskGather:_onTranscriptionComplete'); } - async _onJambonzError(cs, ep, evt) { - this.logger.info({evt}, 'TaskGather:_onJambonzError'); - if (this.isHandledByPrimaryProvider && this.fallbackVendor) { + + async _startFallback(cs, ep, evt) { + if (this.canFallback) { ep.stopTranscription({ vendor: this.vendor, bugname: this.bugname @@ -916,17 +911,31 @@ class TaskGather extends SttTask { .catch((err) => this.logger.error({err}, `Error stopping transcription for primary vendor ${this.vendor}`)); const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf); try { - await this._fallback(); - await this._initSpeech(cs, ep); + this.logger.debug('gather:_startFallback'); + this.notifyError({ msg: 'ASR error', + details:`STT Vendor ${this.vendor} error: ${evt.error || evt.reason}`, failover: 'in progress'}); + await this._initFallback(); + this._speechHandlersSet = false; + await this._setSpeechHandlers(cs, ep); this._startTranscribing(ep); updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid); - return; + return true; } catch (error) { this.logger.info({error}, `There is error while falling back to ${this.fallbackVendor}`); + this.notifyError({ msg: 'ASR error', + details:`STT Vendor ${this.vendor} error: ${evt.error || evt.reason}`, failover: 'not available'}); } + } else { + this.logger.debug('gather:_startFallback no condition for falling back'); + this.notifyError({ msg: 'ASR error', + details:`STT Vendor ${this.vendor} error: ${evt.error || evt.reason}`, failover: 'not available'}); } - const {writeAlerts, AlertType} = cs.srf.locals; + return false; + } + async _onJambonzError(cs, ep, evt) { + this.logger.info({evt}, 'TaskGather:_onJambonzError'); + const {writeAlerts, AlertType} = cs.srf.locals; if (this.vendor === 'nuance') { const {code, error} = evt; if (code === 404 && error === 'No speech') return this._resolve('timeout'); @@ -939,17 +948,23 @@ class TaskGather extends SttTask { message: `Custom speech vendor ${this.vendor} error: ${evt.error}`, vendor: this.vendor, }).catch((err) => this.logger.info({err}, 'Error generating alert for jambonz custom connection failure')); - this.notifyError({msg: 'ASR error', details:`Custom speech vendor ${this.vendor} error: ${evt.error}`}); + if (!(await this._startFallback(cs, ep, evt))) { + this.notifyTaskDone(); + } } - _onVendorConnectFailure(cs, _ep, evt) { + async _onVendorConnectFailure(cs, _ep, evt) { super._onVendorConnectFailure(cs, _ep, evt); - this.notifyTaskDone(); + if (!(await this._startFallback(cs, _ep, evt))) { + this.notifyTaskDone(); + } } - _onVendorError(cs, _ep, evt) { + async _onVendorError(cs, _ep, evt) { super._onVendorError(cs, _ep, evt); - this._resolve('stt-error', evt); + if (!(await this._startFallback(cs, _ep, evt))) { + this._resolve('stt-error', evt); + } } _onVadDetected(cs, ep) { diff --git a/lib/tasks/say.js b/lib/tasks/say.js index 88c5b216..96b8b6d6 100644 --- a/lib/tasks/say.js +++ b/lib/tasks/say.js @@ -117,10 +117,6 @@ class TaskSay extends Task { alert_type: AlertType.TTS_NOT_PROVISIONED, vendor }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts')); - this.notifyError({ - msg: 'TTS error', - details:`No speech credentials provisioned for selected vendor ${vendor}` - }); throw new Error('no provisioned speech credentials for TTS'); } // synthesize all of the text elements @@ -192,7 +188,6 @@ class TaskSay extends Task { vendor, detail: err.message }).catch((err) => this.logger.info({err}, 'Error generating alert for tts failure')); - this.notifyError({msg: 'TTS error', details: err.message || err}); throw err; } }; @@ -215,16 +210,16 @@ class TaskSay extends Task { await super.exec(cs); this.ep = ep; - const vendor = this.synthesizer.vendor && this.synthesizer.vendor !== 'default' ? + let vendor = this.synthesizer.vendor && this.synthesizer.vendor !== 'default' ? this.synthesizer.vendor : cs.speechSynthesisVendor; - const language = this.synthesizer.language && this.synthesizer.language !== 'default' ? + let language = this.synthesizer.language && this.synthesizer.language !== 'default' ? this.synthesizer.language : cs.speechSynthesisLanguage ; - const voice = this.synthesizer.voice && this.synthesizer.voice !== 'default' ? + let voice = this.synthesizer.voice && this.synthesizer.voice !== 'default' ? this.synthesizer.voice : cs.speechSynthesisVoice; - const label = this.synthesizer.label && this.synthesizer.label !== 'default' ? + let label = this.synthesizer.label && this.synthesizer.label !== 'default' ? this.synthesizer.label : cs.speechSynthesisLabel; @@ -241,12 +236,22 @@ class TaskSay extends Task { this.synthesizer.fallbackLabel : cs.fallbackSpeechSynthesisLabel; + if (cs.hasFallbackTts) { + vendor = fallbackVendor; + language = fallbackLanguage; + voice = fallbackVoice; + label = fallbackLabel; + } + let filepath; try { filepath = await this._synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label}); } catch (error) { - if (fallbackVendor && this.isHandledByPrimaryProvider) { + if (fallbackVendor && this.isHandledByPrimaryProvider && !cs.hasFallbackTts) { + this.notifyError( + { msg: 'TTS error', details:`TTS vendor ${vendor} error: ${error}`, failover: 'in progress'}); this.isHandledByPrimaryProvider = false; + cs.hasFallbackTts = true; this.logger.info(`Synthesize error, fallback to ${fallbackVendor}`); filepath = await this._synthesizeWithSpecificVendor(cs, ep, { @@ -256,6 +261,8 @@ class TaskSay extends Task { label: fallbackLabel }); } else { + this.notifyError( + { msg: 'TTS error', details:`TTS vendor ${vendor} error: ${error}`, failover: 'not available'}); throw error; } } diff --git a/lib/tasks/stt-task.js b/lib/tasks/stt-task.js index 2af27664..4b344236 100644 --- a/lib/tasks/stt-task.js +++ b/lib/tasks/stt-task.js @@ -102,6 +102,13 @@ class SttTask extends Task { this.fallbackLabel = cs.fallbackSpeechRecognizerLabel; if (this.data.recognizer) this.data.recognizer.fallbackLabel = this.fallbackLabel; } + // If call is already fallback to 2nd ASR vendor + // use that. + if (cs.hasFallbackAsr) { + this.vendor = this.fallbackVendor; + this.language = this.fallbackLanguage; + this.label = this.fallbackLabel; + } if (!this.data.recognizer.vendor) { this.data.recognizer.vendor = this.vendor; } @@ -119,9 +126,11 @@ class SttTask extends Task { try { this.sttCredentials = await this._initSpeechCredentials(this.cs, this.vendor, this.label); } catch (error) { - if (this.fallbackVendor && this.isHandledByPrimaryProvider) { - await this._fallback(); + if (this.canFallback) { + await this._initFallback(); + this.notifyError({ msg: 'ASR error', details:`Invalid vendor ${this.vendor}`, failover: 'in progress'}); } else { + this.notifyError({ msg: 'ASR error', details:`Invalid vendor ${this.vendor}`, failover: 'not available'}); throw error; } } @@ -190,9 +199,14 @@ class SttTask extends Task { return credentials; } - async _fallback() { + get canFallback() { + return this.fallbackVendor && this.isHandledByPrimaryProvider && !this.cs.hasFallbackAsr; + } + + async _initFallback() { assert(this.fallbackVendor, 'fallback failed without fallbackVendor configuration'); this.isHandledByPrimaryProvider = false; + this.cs.hasFallbackAsr = true; this.logger.info(`Failed to use primary STT provider, fallback to ${this.fallbackVendor}`); this.vendor = this.fallbackVendor; this.language = this.fallbackLanguage; @@ -201,6 +215,8 @@ class SttTask extends Task { this.data.recognizer.language = this.language; this.data.recognizer.label = this.label; this.sttCredentials = await this._initSpeechCredentials(this.cs, this.vendor, this.label); + // cleanup previous listener from previous vendor + this.removeCustomEventListeners(); } async compileHintsForCobalt(ep, hostport, model, token, hints) { @@ -263,7 +279,6 @@ class SttTask extends Task { detail: evt.error, vendor: this.vendor, }).catch((err) => this.logger.info({err}, `Error generating alert for ${this.vendor} connection failure`)); - this.notifyError({msg: 'ASR error', details:`Failed connecting to speech vendor ${this.vendor}: ${evt.error}`}); } _onVendorConnectFailure(cs, _ep, evt) { @@ -276,7 +291,6 @@ class SttTask extends Task { message: `Failed connecting to ${this.vendor} speech recognizer: ${reason}`, vendor: this.vendor, }).catch((err) => this.logger.info({err}, `Error generating alert for ${this.vendor} connection failure`)); - this.notifyError({msg: 'ASR error', details:`Failed connecting to speech vendor ${this.vendor}: ${reason}`}); } } diff --git a/lib/tasks/transcribe.js b/lib/tasks/transcribe.js index 4a72a781..21097a33 100644 --- a/lib/tasks/transcribe.js +++ b/lib/tasks/transcribe.js @@ -80,12 +80,15 @@ class TaskTranscribe extends SttTask { updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid) .catch(() => {/*already logged error */}); - - await this.awaitTaskDone(); } catch (err) { - this.logger.info(err, 'TaskTranscribe:exec - error'); - this.parentTask && this.parentTask.emit('error', err); + if (!(await this._startFallback(cs, ep, {error: err}))) { + this.logger.info(err, 'TaskTranscribe:exec - error'); + this.parentTask && this.parentTask.emit('error', err); + this.removeCustomEventListeners(); + return; + } } + await this.awaitTaskDone(); this.removeCustomEventListeners(); } @@ -512,10 +515,8 @@ class TaskTranscribe extends SttTask { } } - async _onJambonzError(cs, _ep, evt) { - this.logger.info({evt}, 'TaskTranscribe:_onJambonzError'); - if (this.paused) return; - if (this.isHandledByPrimaryProvider && this.fallbackVendor) { + async _startFallback(cs, _ep, evt) { + if (this.canFallback) { _ep.stopTranscription({ vendor: this.vendor, bugname: this.bugname @@ -523,37 +524,53 @@ class TaskTranscribe extends SttTask { .catch((err) => this.logger.error({err}, `Error stopping transcription for primary vendor ${this.vendor}`)); const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf); try { - await this._fallback(); + this.notifyError({ msg: 'ASR error', + details:`STT Vendor ${this.vendor} error: ${evt.error || evt.reason}`, failover: 'in progress'}); + await this._initFallback(); let channel = 1; if (this.ep !== _ep) { channel = 2; } + this[`_speechHandlersSet_${channel}`] = false; this._startTranscribing(cs, _ep, channel); updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid); - return; + return true; } catch (error) { + this.notifyError({ msg: 'ASR error', + details:`STT Vendor ${this.vendor} error: ${evt.error || evt.reason}`, failover: 'not available'}); this.logger.info({error}, `There is error while falling back to ${this.fallbackVendor}`); } } else { - const {writeAlerts, AlertType} = cs.srf.locals; + this.logger.debug('transcribe:_startFallback no condition for falling back'); + this.notifyError({ msg: 'ASR error', + details:`STT Vendor ${this.vendor} error: ${evt.error || evt.reason}`, failover: 'not available'}); + } + return false; + } - if (this.vendor === 'nuance') { - const {code, error} = evt; - if (code === 404 && error === 'No speech') return this._resolve('timeout'); - if (code === 413 && error === 'Too much speech') return this._resolve('timeout'); - } - this.logger.info({evt}, 'TaskTranscribe:_onJambonzError'); - writeAlerts({ - account_sid: cs.accountSid, - alert_type: AlertType.STT_FAILURE, - message: `Custom speech vendor ${this.vendor} error: ${evt.error}`, - vendor: this.vendor, - }).catch((err) => this.logger.info({err}, 'Error generating alert for jambonz custom connection failure')); - this.notifyError({msg: 'ASR error', details:`Custom speech vendor ${this.vendor} error: ${evt.error}`}); + async _onJambonzError(cs, _ep, evt) { + this.logger.info({evt}, 'TaskTranscribe:_onJambonzError'); + if (this.paused) return; + const {writeAlerts, AlertType} = cs.srf.locals; + + if (this.vendor === 'nuance') { + const {code, error} = evt; + if (code === 404 && error === 'No speech') return this._resolve('timeout'); + if (code === 413 && error === 'Too much speech') return this._resolve('timeout'); + } + this.logger.info({evt}, 'TaskTranscribe:_onJambonzError'); + writeAlerts({ + account_sid: cs.accountSid, + alert_type: AlertType.STT_FAILURE, + message: `Custom speech vendor ${this.vendor} error: ${evt.error}`, + vendor: this.vendor, + }).catch((err) => this.logger.info({err}, 'Error generating alert for jambonz custom connection failure')); + if (!(await this._startFallback(cs, _ep, evt))) { + this.notifyTaskDone(); } } - _onVendorConnectFailure(cs, _ep, channel, evt) { + async _onVendorConnectFailure(cs, _ep, channel, evt) { super._onVendorConnectFailure(cs, _ep, evt); if (this.childSpan[channel - 1] && this.childSpan[channel - 1].span) { this.childSpan[channel - 1].span.setAttributes({ @@ -562,7 +579,9 @@ class TaskTranscribe extends SttTask { }); this.childSpan[channel - 1].span.end(); } - this.notifyTaskDone(); + if (!(await this._startFallback(cs, _ep, evt))) { + this.notifyTaskDone(); + } } _startAsrTimer(channel) {