From 887c6243e2ab7fd4fa9763a99b0f49efdcefaea0 Mon Sep 17 00:00:00 2001 From: Dave Horton Date: Thu, 25 Aug 2022 22:43:38 +0200 Subject: [PATCH] handle altLanguages set at the session level via config verb; fix azure stt race condition with final transcripts from stopped recognition --- lib/session/call-session.js | 13 ++++++++++++- lib/tasks/config.js | 4 ++++ lib/tasks/gather.js | 35 +++++++++++++++++++++++++++++------ lib/tasks/transcribe.js | 11 +++++++++-- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/lib/session/call-session.js b/lib/session/call-session.js index 12341972..4d3bdbff 100644 --- a/lib/session/call-session.js +++ b/lib/session/call-session.js @@ -274,6 +274,18 @@ class CallSession extends Emitter { return this._globalSttHints; } + set altLanguages(langs) { + this._globalAltLanguages = langs; + } + + get hasAltLanguages() { + return Array.isArray(this._globalAltLanguages); + } + + get altLanguages() { + return this._globalAltLanguages; + } + async notifyRecordOptions(opts) { const {action} = opts; this.logger.debug({opts}, 'CallSession:notifyRecordOptions'); @@ -1112,7 +1124,6 @@ class CallSession extends Emitter { * @param {Task} task - task to be executed */ async _evalEndpointPrecondition(task) { - this.logger.debug('CallSession:_evalEndpointPrecondition'); if (this.callGone) new Error(`${BADPRECONDITIONS}: call gone`); if (this.ep) { diff --git a/lib/tasks/config.js b/lib/tasks/config.js index 230db0d3..20702c04 100644 --- a/lib/tasks/config.js +++ b/lib/tasks/config.js @@ -94,6 +94,10 @@ class TaskConfig extends Task { } cs.globalSttHints = obj; } + if (Array.isArray(this.recognizer.altLanguages)) { + this.logger.info({altLanguages: this.recognizer.altLanguages}, 'Config: updated altLanguages'); + cs.altLanguages = this.recognizer.altLanguages; + } this.logger.info({ recognizer: this.recognizer, isContinuousAsr: cs.isContinuousAsr diff --git a/lib/tasks/gather.js b/lib/tasks/gather.js index 4a531831..d901518e 100644 --- a/lib/tasks/gather.js +++ b/lib/tasks/gather.js @@ -141,6 +141,11 @@ class TaskGather extends Task { this.logger.debug({hints: this.hints, hintsBoost: this.hintsBoost}, 'Gather:exec - applying global sttHints'); } + if (cs.hasAltLanguages) { + this.altLanguages = this.altLanguages.concat(cs.altLanguages); + this.logger.debug({altLanguages: this.altLanguages}, + 'Gather:exec - applying altLanguages'); + } if (!this.isContinuousAsr && cs.isContinuousAsr) { this.isContinuousAsr = true; this.asrTimeout = cs.asrTimeout * 1000; @@ -312,7 +317,8 @@ class TaskGather extends Task { opts.GOOGLE_SPEECH_HINTS_BOOST = this.hintsBoost; } } - if (this.altLanguages.length > 1) opts.GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES = this.altLanguages.join(','); + if (this.altLanguages.length > 0) opts.GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES = this.altLanguages.join(','); + else opts.GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES = ''; if ('unspecified' !== this.interactionType) { opts.GOOGLE_SPEECH_METADATA_INTERACTION_TYPE = this.interactionType; } @@ -359,6 +365,9 @@ class TaskGather extends Task { if (this.altLanguages && this.altLanguages.length > 0) { opts.AZURE_SPEECH_ALTERNATIVE_LANGUAGE_CODES = this.altLanguages.join(','); } + else { + opts.AZURE_SPEECH_ALTERNATIVE_LANGUAGE_CODES = ''; + } if (this.requestSnr) opts.AZURE_REQUEST_SNR = 1; if (this.profanityOption && this.profanityOption !== 'raw') opts.AZURE_PROFANITY_OPTION = this.profanityOption; if (this.azureServiceEndpoint) opts.AZURE_SERVICE_ENDPOINT = this.azureServiceEndpoint; @@ -468,6 +477,7 @@ class TaskGather extends Task { _onTranscription(cs, ep, evt, fsEvent) { // make sure this is not a transcript from answering machine detection const bugname = fsEvent.getHeader('media-bugname'); + const finished = fsEvent.getHeader('transcription-session-finished'); if (bugname && this.bugname !== bugname) return; if ('aws' === this.vendor && Array.isArray(evt) && evt.length > 0) evt = evt[0]; @@ -509,8 +519,14 @@ class TaskGather extends Task { if (evt.is_final) { if (evt.alternatives[0].transcript === '' && !this.callSession.callGone && !this.killed) { - this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, listen again'); - return this._startTranscribing(ep); + if ('microsoft' === this.vendor && finished === 'true') { + this.logger.debug({evt}, 'TaskGather:_onTranscription - got empty transcript from old gather, disregarding'); + } + else { + this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, listen again'); + this._startTranscribing(ep); + } + return; } if (this.isContinuousAsr) { /* append the transcript and start listening again for asrTimeout */ @@ -586,10 +602,17 @@ class TaskGather extends Task { } } - _onNoSpeechDetected(cs, ep) { + _onNoSpeechDetected(cs, ep, evt, fsEvent) { if (!this.callSession.callGone && !this.killed) { - this.logger.debug('TaskGather:_onNoSpeechDetected - listen again'); - return this._startTranscribing(ep); + const finished = fsEvent.getHeader('transcription-session-finished'); + if (this.vendor === 'microsoft' && finished === 'true') { + this.logger.debug('TaskGather:_onNoSpeechDetected for old gather, ignoring'); + } + else { + this.logger.debug('TaskGather:_onNoSpeechDetected - listen again'); + this._startTranscribing(ep); + } + return; } } diff --git a/lib/tasks/transcribe.js b/lib/tasks/transcribe.js index c06e209b..f35158a5 100644 --- a/lib/tasks/transcribe.js +++ b/lib/tasks/transcribe.js @@ -69,6 +69,11 @@ class TaskTranscribe extends Task { this.logger.debug({hints: this.hints, hintsBoost: this.hintsBoost}, 'Transcribe:exec - applying global `sttHints'); } + if (cs.hasAltLanguages) { + this.altLanguages = this.altLanguages.concat(cs.altLanguages); + this.logger.debug({altLanguages: this.altLanguages}, + 'Gather:exec - applying altLanguages'); + } this.ep = ep; this.ep2 = ep2; @@ -172,7 +177,8 @@ class TaskTranscribe extends Task { opts.GOOGLE_SPEECH_HINTS_BOOST = this.hintsBoost; } } - if (this.altLanguages.length > 1) opts.GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES = this.altLanguages.join(','); + if (this.altLanguages.length > 0) opts.GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES = this.altLanguages.join(','); + else opts.GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES = ''; if ('unspecified' !== this.interactionType) { opts.GOOGLE_SPEECH_METADATA_INTERACTION_TYPE = this.interactionType; } @@ -229,7 +235,8 @@ class TaskTranscribe extends Task { if (this.hints && this.hints.length > 0) { opts.AZURE_SPEECH_HINTS = this.hints.map((h) => h.trim()).join(','); } - if (this.altLanguages.length > 1) opts.AZURE_SPEECH_ALTERNATIVE_LANGUAGE_CODES = this.altLanguages.join(','); + if (this.altLanguages.length > 0) opts.AZURE_SPEECH_ALTERNATIVE_LANGUAGE_CODES = this.altLanguages.join(','); + else opts.AZURE_SPEECH_ALTERNATIVE_LANGUAGE_CODES = ''; if (this.requestSnr) opts.AZURE_REQUEST_SNR = 1; if (this.profanityOption !== 'raw') opts.AZURE_PROFANITY_OPTION = this.profanityOption; if (this.initialSpeechTimeoutMs > 0) opts.AZURE_INITIAL_SPEECH_TIMEOUT_MS = this.initialSpeechTimeoutMs;