diff --git a/lib/tasks/gather.js b/lib/tasks/gather.js index 8fc42149..72d0339c 100644 --- a/lib/tasks/gather.js +++ b/lib/tasks/gather.js @@ -65,6 +65,11 @@ class TaskGather extends Task { const recognizer = this.data.recognizer; this.vendor = recognizer.vendor; this.language = recognizer.language; + this.label = recognizer.label; + + this.fallbackVendor = recognizer.fallbackVendor || 'default'; + this.fallbackLanguage = recognizer.fallbackLanguage || 'default'; + this.fallbackLabel = recognizer.fallbackLabel || 'default'; /* let credentials be supplied in the recognizer object at runtime */ this.sttCredentials = setSpeechCredentialsAtRuntime(recognizer); @@ -133,11 +138,48 @@ class TaskGather extends Task { return s; } + async _initSpeechCredentials(cs, vendor, label) { + const {getNuanceAccessToken, getIbmAccessToken} = this.cs.srf.locals.dbHelpers; + let credentials = cs.getSpeechCredentials(vendor, 'stt', label); + + if (!credentials) { + const {writeAlerts, AlertType} = cs.srf.locals; + this.logger.info(`TaskGather:exec - ERROR stt using ${vendor} requested but creds not supplied`); + writeAlerts({ + account_sid: cs.accountSid, + alert_type: AlertType.STT_NOT_PROVISIONED, + vendor + }).catch((err) => this.logger.info({err}, 'Error generating alert for no stt')); + // Notify application that STT vender is wrong. + this.notifyError({ + msg: 'ASR error', + details: `No speech-to-text service credentials for ${vendor} have been configured` + }); + this.notifyTaskDone(); + throw new Error(`No speech-to-text service credentials for ${vendor} have been configured`); + } + + if (vendor === 'nuance' && credentials.client_id) { + /* get nuance access token */ + const {client_id, secret} = credentials; + const {access_token, servedFromCache} = await getNuanceAccessToken(client_id, secret, 'asr tts'); + this.logger.debug({client_id}, `Gather:exec - got nuance access token ${servedFromCache ? 'from cache' : ''}`); + credentials = {...credentials, access_token}; + } + else if (vendor == 'ibm' && credentials.stt_api_key) { + /* get ibm access token */ + const {stt_api_key, stt_region} = credentials; + const {access_token, servedFromCache} = await getIbmAccessToken(stt_api_key); + this.logger.debug({stt_api_key}, `Gather:exec - got ibm access token ${servedFromCache ? 'from cache' : ''}`); + credentials = {...credentials, access_token, stt_region}; + } + return credentials; + } + async exec(cs, {ep}) { this.logger.debug({options: this.data}, 'Gather:exec'); await super.exec(cs); const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf); - const {getNuanceAccessToken, getIbmAccessToken} = cs.srf.locals.dbHelpers; if (cs.hasGlobalSttHints && !this.maskGlobalSttHints) { const {hints, hintsBoost} = cs.globalSttHints; @@ -184,58 +226,61 @@ class TaskGather extends Task { this.language = cs.speechRecognizerLanguage; if (this.data.recognizer) this.data.recognizer.language = this.language; } + this.fallbackVendor = this.data.recognizer && this.data.recognizer.fallbackVendor !== 'default' ? + this.data.recognizer.fallbackVendor : + cs.fallbackSpeechRecognizerVendor; + + this.fallbackLanguage = this.data.recognizer && this.data.recognizer.fallbackLanguage !== 'default' ? + this.data.recognizer.fallbackLanguage : + cs.fallbackSpeechRecognizerLanguage; + + this.fallbackLabel = this.data.recognizer && this.data.recognizer.fallbackLabel !== 'default' ? + this.data.recognizer.fallbackLabel : + cs.fallbackSpeechRecognizerLabel; + if (!this.data.recognizer.vendor) { this.data.recognizer.vendor = this.vendor; } - if (this.needsStt && !this.sttCredentials) this.sttCredentials = - cs.getSpeechCredentials(this.vendor, 'stt', this.data.recognizer?.label || cs.speechRecognizerLabel); + if (this.needsStt && !this.sttCredentials) { - const {writeAlerts, AlertType} = cs.srf.locals; - this.logger.info(`TaskGather:exec - ERROR stt using ${this.vendor} requested but creds not supplied`); - writeAlerts({ - account_sid: cs.accountSid, - alert_type: AlertType.STT_NOT_PROVISIONED, - vendor: this.vendor - }).catch((err) => this.logger.info({err}, 'Error generating alert for no stt')); - // Notify application that STT vender is wrong. - this.notifyError({ - msg: 'ASR error', - details: `No speech-to-text service credentials for ${this.vendor} have been configured` - }); - this.notifyTaskDone(); - throw new Error(`No speech-to-text service credentials for ${this.vendor} have been configured`); + this.sttCredentials = await this._initSpeechCredentials(cs, this.vendor, this.label); } - if (this.vendor === 'nuance' && this.sttCredentials.client_id) { - /* get nuance access token */ - const {client_id, secret} = this.sttCredentials; - const {access_token, servedFromCache} = await getNuanceAccessToken(client_id, secret, 'asr tts'); - this.logger.debug({client_id}, `Gather:exec - got nuance access token ${servedFromCache ? 'from cache' : ''}`); - this.sttCredentials = {...this.sttCredentials, access_token}; + // Fetch credential for fallback recognizer + if (this.needsStt && !this.fallbackSttCredentials && this.fallbackVendor) { + this.fallbackSttCredentials = await this._initSpeechCredentials( + cs, this.fallbackSttCredentials, this.fallbackLabel); } - else if (this.vendor == 'ibm' && this.sttCredentials.stt_api_key) { - /* get ibm access token */ - const {stt_api_key, stt_region} = this.sttCredentials; - const {access_token, servedFromCache} = await getIbmAccessToken(stt_api_key); - this.logger.debug({stt_api_key}, `Gather:exec - got ibm access token ${servedFromCache ? 'from cache' : ''}`); - this.sttCredentials = {...this.sttCredentials, access_token, stt_region}; - } - const startListening = (cs, ep) => { + + const startListening = async(cs, ep) => { this._startTimer(); if (this.isContinuousAsr && 0 === this.timeout) this._startAsrTimer(); if (this.input.includes('speech') && !this.listenDuringPrompt) { - this._initSpeech(cs, ep) - .then(() => { - if (this.killed) { - this.logger.info('Gather:exec - task was quickly killed so do not transcribe'); - return; + try { + await this._initSpeech(cs, ep, this.sttCredentials); + if (this.killed) { + this.logger.info('Gather:exec - task was quickly killed so do not transcribe'); + return; + } + this._startTranscribing(ep, this.vendor, this.language); + return updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid); + } catch (error) { + this.logger.error({error}, 'error in initSpeech'); + if (this.fallbackSttCredentials) { + try { + this.logger.info(`fallback to STT provider: ${this.fallbackSttCredentials.vendor}`); + await this._initSpeech(cs, ep, this.fallbackSttCredentials); + if (this.killed) { + this.logger.info('Gather:exec - task was quickly killed so do not transcribe'); + return; + } + this._startTranscribing(ep, this.fallbackVendor, this.fallbackLanguage); + return updateSpeechCredentialLastUsed(this.fallbackSttCredentials.speech_credential_sid); + } catch (err) { + this.logger.error({err}, `error in initSpeech for fallback STT provider ${this.fallbackVendor}`); } - this._startTranscribing(ep); - return updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid); - }) - .catch((err) => { - this.logger.error({err}, 'error in initSpeech'); - }); + } + } } }; @@ -363,9 +408,9 @@ class TaskGather extends Task { } } - async _initSpeech(cs, ep) { - const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.data.recognizer); - switch (this.vendor) { + async _initSpeech(cs, ep, credentials) { + const opts = this.setChannelVarsForStt(this, credentials, this.data.recognizer); + switch (credentials.vendor) { case 'google': this.bugname = 'google_transcribe'; ep.addCustomEventListener(GoogleTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep)); @@ -451,9 +496,9 @@ class TaskGather extends Task { break; } else { - this.notifyError({ msg: 'ASR error', details:`Invalid vendor ${this.vendor}`}); + this.notifyError({ msg: 'ASR error', details:`Invalid vendor ${credentials.vendor}`}); this.notifyTaskDone(); - throw new Error(`Invalid vendor ${this.vendor}`); + throw new Error(`Invalid vendor ${credentials.vendor}`); } } @@ -463,16 +508,16 @@ class TaskGather extends Task { .catch((err) => this.logger.info(err, 'Error setting channel variables')); } - _startTranscribing(ep) { + _startTranscribing(ep, vendor = this.vendor, language = this.language) { this.logger.debug({ - vendor: this.vendor, - locale: this.language, + vendor, + locale: language, interim: this.interim, bugname: this.bugname }, 'Gather:_startTranscribing'); ep.startTranscription({ - vendor: this.vendor, - locale: this.language, + vendor, + locale: language, interim: this.interim, bugname: this.bugname, }).catch((err) => { @@ -481,7 +526,7 @@ class TaskGather extends Task { writeAlerts({ account_sid: this.cs.accountSid, alert_type: AlertType.STT_FAILURE, - vendor: this.vendor, + vendor, detail: err.message }); }).catch((err) => this.logger.info({err}, 'Error generating alert for tts failure')); diff --git a/lib/tasks/say.js b/lib/tasks/say.js index b49734c2..cc294fac 100644 --- a/lib/tasks/say.js +++ b/lib/tasks/say.js @@ -74,13 +74,15 @@ class TaskSay extends Task { const fallbackVoice = this.synthesizer.fallbackVoice && this.synthesizer.fallbackVoice !== 'default' ? this.synthesizer.fallbackVoice : cs.fallbackSpeechSynthesisVoice; + const label = this.synthesizer.label && this.synthesizer.label !== 'default' ? + this.synthesizer.label : + cs.speechSynthesisLabel; const fallbackLabel = this.synthesizer.fallbackLabel && this.synthesizer.fallbackLabel !== 'default' ? this.synthesizer.fallbackLabel : cs.fallbackSpeechSynthesisLabel; const engine = this.synthesizer.engine || 'standard'; const salt = cs.callSid; - let credentials = cs.getSpeechCredentials(vendor, 'tts', this.data.synthesizer ? - this.data.synthesizer?.label : cs.speechSynthesisLabel); + let credentials = cs.getSpeechCredentials(vendor, 'tts', label); /* parse Nuance voices into name and model */ let model; diff --git a/package-lock.json b/package-lock.json index 0ede0450..0a93c445 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@jambonz/speech-utils": "^0.0.19", "@jambonz/stats-collector": "^0.1.9", "@jambonz/time-series": "^0.2.8", - "@jambonz/verb-specifications": "^0.0.27", + "@jambonz/verb-specifications": "^0.0.29", "@opentelemetry/api": "^1.4.0", "@opentelemetry/exporter-jaeger": "^1.9.0", "@opentelemetry/exporter-trace-otlp-http": "^0.35.0", @@ -3019,9 +3019,9 @@ } }, "node_modules/@jambonz/verb-specifications": { - "version": "0.0.27", - "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.27.tgz", - "integrity": "sha512-DIcxhCNrgr2RTE3YrGNP15RqKyV+P8f97SPBlKd2zTM5aN2oV5xv+pRDx5gLzmrUZ5TIEaBXQN3vTmM2Zx5Q6g==", + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.29.tgz", + "integrity": "sha512-jeYI+GN7Y5nXhdFG3SXvXaBlhCjIC+l5AcBywDDGxxyuuKRTukPS0MSvCtWPZP6H3wYYGqfJ4DR/vgtBF3pvyQ==", "dependencies": { "debug": "^4.3.4", "pino": "^8.8.0" @@ -12985,9 +12985,9 @@ } }, "@jambonz/verb-specifications": { - "version": "0.0.27", - "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.27.tgz", - "integrity": "sha512-DIcxhCNrgr2RTE3YrGNP15RqKyV+P8f97SPBlKd2zTM5aN2oV5xv+pRDx5gLzmrUZ5TIEaBXQN3vTmM2Zx5Q6g==", + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.29.tgz", + "integrity": "sha512-jeYI+GN7Y5nXhdFG3SXvXaBlhCjIC+l5AcBywDDGxxyuuKRTukPS0MSvCtWPZP6H3wYYGqfJ4DR/vgtBF3pvyQ==", "requires": { "debug": "^4.3.4", "pino": "^8.8.0" diff --git a/package.json b/package.json index 5d142350..0dfab117 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "@jambonz/speech-utils": "^0.0.19", "@jambonz/stats-collector": "^0.1.9", "@jambonz/time-series": "^0.2.8", - "@jambonz/verb-specifications": "^0.0.27", + "@jambonz/verb-specifications": "^0.0.29", "@opentelemetry/api": "^1.4.0", "@opentelemetry/exporter-jaeger": "^1.9.0", "@opentelemetry/exporter-trace-otlp-http": "^0.35.0",