fix gather

2025-12-21 09:08:02 +00:00 · 2023-08-18 09:57:02 +07:00
parent a97d99650c
commit e9f2837370
4 changed files with 110 additions and 63 deletions
--- a/lib/tasks/gather.js
+++ b/lib/tasks/gather.js
@@ -65,6 +65,11 @@ class TaskGather extends Task {
      const recognizer = this.data.recognizer;
      this.vendor = recognizer.vendor;
      this.language = recognizer.language;
+      this.label = recognizer.label;
+
+      this.fallbackVendor = recognizer.fallbackVendor || 'default';
+      this.fallbackLanguage = recognizer.fallbackLanguage || 'default';
+      this.fallbackLabel = recognizer.fallbackLabel || 'default';

      /* let credentials be supplied in the recognizer object at runtime */
      this.sttCredentials = setSpeechCredentialsAtRuntime(recognizer);
@@ -133,11 +138,48 @@ class TaskGather extends Task {
    return s;
  }

+  async _initSpeechCredentials(cs, vendor, label) {
+    const {getNuanceAccessToken, getIbmAccessToken} = this.cs.srf.locals.dbHelpers;
+    let credentials = cs.getSpeechCredentials(vendor, 'stt', label);
+
+    if (!credentials) {
+      const {writeAlerts, AlertType} = cs.srf.locals;
+      this.logger.info(`TaskGather:exec - ERROR stt using ${vendor} requested but creds not supplied`);
+      writeAlerts({
+        account_sid: cs.accountSid,
+        alert_type: AlertType.STT_NOT_PROVISIONED,
+        vendor
+      }).catch((err) => this.logger.info({err}, 'Error generating alert for no stt'));
+      // Notify application that STT vender is wrong.
+      this.notifyError({
+        msg: 'ASR error',
+        details: `No speech-to-text service credentials for ${vendor} have been configured`
+      });
+      this.notifyTaskDone();
+      throw new Error(`No speech-to-text service credentials for ${vendor} have been configured`);
+    }
+
+    if (vendor === 'nuance' && credentials.client_id) {
+      /* get nuance access token */
+      const {client_id, secret} = credentials;
+      const {access_token, servedFromCache} = await getNuanceAccessToken(client_id, secret, 'asr tts');
+      this.logger.debug({client_id}, `Gather:exec - got nuance access token ${servedFromCache ? 'from cache' : ''}`);
+      credentials = {...credentials, access_token};
+    }
+    else if (vendor == 'ibm' && credentials.stt_api_key) {
+      /* get ibm access token */
+      const {stt_api_key, stt_region} = credentials;
+      const {access_token, servedFromCache} = await getIbmAccessToken(stt_api_key);
+      this.logger.debug({stt_api_key}, `Gather:exec - got ibm access token ${servedFromCache ? 'from cache' : ''}`);
+      credentials = {...credentials, access_token, stt_region};
+    }
+    return credentials;
+  }
+
  async exec(cs, {ep}) {
    this.logger.debug({options: this.data}, 'Gather:exec');
    await super.exec(cs);
    const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
-    const {getNuanceAccessToken, getIbmAccessToken} = cs.srf.locals.dbHelpers;

    if (cs.hasGlobalSttHints && !this.maskGlobalSttHints) {
      const {hints, hintsBoost} = cs.globalSttHints;
@@ -184,58 +226,61 @@ class TaskGather extends Task {
      this.language = cs.speechRecognizerLanguage;
      if (this.data.recognizer) this.data.recognizer.language = this.language;
    }
+    this.fallbackVendor = this.data.recognizer && this.data.recognizer.fallbackVendor !== 'default' ?
+      this.data.recognizer.fallbackVendor :
+      cs.fallbackSpeechRecognizerVendor;
+
+    this.fallbackLanguage = this.data.recognizer && this.data.recognizer.fallbackLanguage !== 'default' ?
+      this.data.recognizer.fallbackLanguage :
+      cs.fallbackSpeechRecognizerLanguage;
+
+    this.fallbackLabel = this.data.recognizer && this.data.recognizer.fallbackLabel !== 'default' ?
+      this.data.recognizer.fallbackLabel :
+      cs.fallbackSpeechRecognizerLabel;
+
    if (!this.data.recognizer.vendor) {
      this.data.recognizer.vendor = this.vendor;
    }
-    if (this.needsStt && !this.sttCredentials) this.sttCredentials =
-      cs.getSpeechCredentials(this.vendor, 'stt', this.data.recognizer?.label || cs.speechRecognizerLabel);
+
    if (this.needsStt && !this.sttCredentials) {
-      const {writeAlerts, AlertType} = cs.srf.locals;
-      this.logger.info(`TaskGather:exec - ERROR stt using ${this.vendor} requested but creds not supplied`);
-      writeAlerts({
-        account_sid: cs.accountSid,
-        alert_type: AlertType.STT_NOT_PROVISIONED,
-        vendor: this.vendor
-      }).catch((err) => this.logger.info({err}, 'Error generating alert for no stt'));
-      // Notify application that STT vender is wrong.
-      this.notifyError({
-        msg: 'ASR error',
-        details: `No speech-to-text service credentials for ${this.vendor} have been configured`
-      });
-      this.notifyTaskDone();
-      throw new Error(`No speech-to-text service credentials for ${this.vendor} have been configured`);
+      this.sttCredentials = await this._initSpeechCredentials(cs, this.vendor, this.label);
    }

-    if (this.vendor === 'nuance' && this.sttCredentials.client_id) {
-      /* get nuance access token */
-      const {client_id, secret} = this.sttCredentials;
-      const {access_token, servedFromCache} = await getNuanceAccessToken(client_id, secret, 'asr tts');
-      this.logger.debug({client_id}, `Gather:exec - got nuance access token ${servedFromCache ? 'from cache' : ''}`);
-      this.sttCredentials = {...this.sttCredentials, access_token};
+    // Fetch credential for fallback recognizer
+    if (this.needsStt && !this.fallbackSttCredentials && this.fallbackVendor) {
+      this.fallbackSttCredentials = await this._initSpeechCredentials(
+        cs, this.fallbackSttCredentials, this.fallbackLabel);
    }
-    else if (this.vendor == 'ibm' && this.sttCredentials.stt_api_key) {
-      /* get ibm access token */
-      const {stt_api_key, stt_region} = this.sttCredentials;
-      const {access_token, servedFromCache} = await getIbmAccessToken(stt_api_key);
-      this.logger.debug({stt_api_key}, `Gather:exec - got ibm access token ${servedFromCache ? 'from cache' : ''}`);
-      this.sttCredentials = {...this.sttCredentials, access_token, stt_region};
-    }
-    const startListening = (cs, ep) => {
+
+    const startListening = async(cs, ep) => {
      this._startTimer();
      if (this.isContinuousAsr && 0 === this.timeout) this._startAsrTimer();
      if (this.input.includes('speech') && !this.listenDuringPrompt) {
-        this._initSpeech(cs, ep)
-          .then(() => {
+        try {
+          await this._initSpeech(cs, ep, this.sttCredentials);
          if (this.killed) {
            this.logger.info('Gather:exec - task was quickly killed so do not transcribe');
            return;
          }
-            this._startTranscribing(ep);
+          this._startTranscribing(ep, this.vendor, this.language);
          return updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid);
-          })
-          .catch((err) => {
-            this.logger.error({err}, 'error in initSpeech');
-          });
+        } catch (error) {
+          this.logger.error({error}, 'error in initSpeech');
+          if (this.fallbackSttCredentials) {
+            try {
+              this.logger.info(`fallback to STT provider: ${this.fallbackSttCredentials.vendor}`);
+              await this._initSpeech(cs, ep, this.fallbackSttCredentials);
+              if (this.killed) {
+                this.logger.info('Gather:exec - task was quickly killed so do not transcribe');
+                return;
+              }
+              this._startTranscribing(ep, this.fallbackVendor, this.fallbackLanguage);
+              return updateSpeechCredentialLastUsed(this.fallbackSttCredentials.speech_credential_sid);
+            } catch (err) {
+              this.logger.error({err}, `error in initSpeech for fallback STT provider ${this.fallbackVendor}`);
+            }
+          }
+        }
      }
    };

@@ -363,9 +408,9 @@ class TaskGather extends Task {
    }
  }

-  async _initSpeech(cs, ep) {
-    const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.data.recognizer);
-    switch (this.vendor) {
+  async _initSpeech(cs, ep, credentials) {
+    const opts = this.setChannelVarsForStt(this, credentials, this.data.recognizer);
+    switch (credentials.vendor) {
      case 'google':
        this.bugname = 'google_transcribe';
        ep.addCustomEventListener(GoogleTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
@@ -451,9 +496,9 @@ class TaskGather extends Task {
          break;
        }
        else {
-          this.notifyError({ msg: 'ASR error', details:`Invalid vendor ${this.vendor}`});
+          this.notifyError({ msg: 'ASR error', details:`Invalid vendor ${credentials.vendor}`});
          this.notifyTaskDone();
-          throw new Error(`Invalid vendor ${this.vendor}`);
+          throw new Error(`Invalid vendor ${credentials.vendor}`);
        }
    }

@@ -463,16 +508,16 @@ class TaskGather extends Task {
      .catch((err) => this.logger.info(err, 'Error setting channel variables'));
  }

-  _startTranscribing(ep) {
+  _startTranscribing(ep, vendor = this.vendor, language = this.language) {
    this.logger.debug({
-      vendor: this.vendor,
-      locale: this.language,
+      vendor,
+      locale: language,
      interim: this.interim,
      bugname: this.bugname
    }, 'Gather:_startTranscribing');
    ep.startTranscription({
-      vendor: this.vendor,
-      locale: this.language,
+      vendor,
+      locale: language,
      interim: this.interim,
      bugname: this.bugname,
    }).catch((err) => {
@@ -481,7 +526,7 @@ class TaskGather extends Task {
      writeAlerts({
        account_sid: this.cs.accountSid,
        alert_type: AlertType.STT_FAILURE,
-        vendor: this.vendor,
+        vendor,
        detail: err.message
      });
    }).catch((err) => this.logger.info({err}, 'Error generating alert for tts failure'));
--- a/lib/tasks/say.js
+++ b/lib/tasks/say.js
@@ -74,13 +74,15 @@ class TaskSay extends Task {
    const fallbackVoice = this.synthesizer.fallbackVoice && this.synthesizer.fallbackVoice !== 'default' ?
      this.synthesizer.fallbackVoice :
      cs.fallbackSpeechSynthesisVoice;
+    const label = this.synthesizer.label && this.synthesizer.label !== 'default' ?
+      this.synthesizer.label :
+      cs.speechSynthesisLabel;
    const fallbackLabel = this.synthesizer.fallbackLabel && this.synthesizer.fallbackLabel !== 'default' ?
      this.synthesizer.fallbackLabel :
      cs.fallbackSpeechSynthesisLabel;
    const engine = this.synthesizer.engine || 'standard';
    const salt = cs.callSid;
-    let credentials = cs.getSpeechCredentials(vendor, 'tts', this.data.synthesizer ?
-      this.data.synthesizer?.label : cs.speechSynthesisLabel);
+    let credentials = cs.getSpeechCredentials(vendor, 'tts', label);

    /* parse Nuance voices into name and model */
    let model;
--- a/package-lock.json
+++ b/package-lock.json
@@ -17,7 +17,7 @@
        "@jambonz/speech-utils": "^0.0.19",
        "@jambonz/stats-collector": "^0.1.9",
        "@jambonz/time-series": "^0.2.8",
-        "@jambonz/verb-specifications": "^0.0.27",
+        "@jambonz/verb-specifications": "^0.0.29",
        "@opentelemetry/api": "^1.4.0",
        "@opentelemetry/exporter-jaeger": "^1.9.0",
        "@opentelemetry/exporter-trace-otlp-http": "^0.35.0",
@@ -3019,9 +3019,9 @@
      }
    },
    "node_modules/@jambonz/verb-specifications": {
-      "version": "0.0.27",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.27.tgz",
-      "integrity": "sha512-DIcxhCNrgr2RTE3YrGNP15RqKyV+P8f97SPBlKd2zTM5aN2oV5xv+pRDx5gLzmrUZ5TIEaBXQN3vTmM2Zx5Q6g==",
+      "version": "0.0.29",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.29.tgz",
+      "integrity": "sha512-jeYI+GN7Y5nXhdFG3SXvXaBlhCjIC+l5AcBywDDGxxyuuKRTukPS0MSvCtWPZP6H3wYYGqfJ4DR/vgtBF3pvyQ==",
      "dependencies": {
        "debug": "^4.3.4",
        "pino": "^8.8.0"
@@ -12985,9 +12985,9 @@
      }
    },
    "@jambonz/verb-specifications": {
-      "version": "0.0.27",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.27.tgz",
-      "integrity": "sha512-DIcxhCNrgr2RTE3YrGNP15RqKyV+P8f97SPBlKd2zTM5aN2oV5xv+pRDx5gLzmrUZ5TIEaBXQN3vTmM2Zx5Q6g==",
+      "version": "0.0.29",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.29.tgz",
+      "integrity": "sha512-jeYI+GN7Y5nXhdFG3SXvXaBlhCjIC+l5AcBywDDGxxyuuKRTukPS0MSvCtWPZP6H3wYYGqfJ4DR/vgtBF3pvyQ==",
      "requires": {
        "debug": "^4.3.4",
        "pino": "^8.8.0"
--- a/package.json
+++ b/package.json
@@ -31,7 +31,7 @@
    "@jambonz/speech-utils": "^0.0.19",
    "@jambonz/stats-collector": "^0.1.9",
    "@jambonz/time-series": "^0.2.8",
-    "@jambonz/verb-specifications": "^0.0.27",
+    "@jambonz/verb-specifications": "^0.0.29",
    "@opentelemetry/api": "^1.4.0",
    "@opentelemetry/exporter-jaeger": "^1.9.0",
    "@opentelemetry/exporter-trace-otlp-http": "^0.35.0",