wip

* feat/868: Use the properties from global config in verb for TTS

* feat/868: setting this.options to combination of cs.synthesizer.options and this.options

* feat/868: Move the logic of copying cs properties to parent class tts-task.js

* feat/868: add empty line that was removed, say.js restored to original version

* feat/868: moved _synthesizeWithSpecificVendor to tts-task.js

---------

Co-authored-by: Rammohan Yadavalli <rammohan.yadavalli@kore.com>

lib/http-routes/api/create-call.js

@@ -218,7 +218,7 @@ router.post('/',
       }
       if (!app.notifier && app.call_status_hook) {
         app.notifier = new HttpRequestor(logger, account.account_sid, app.call_status_hook, account.webhook_secret);
-        logger.debug({call_hook: app.call_hook}, 'creating http client for call status hook');
+        logger.debug({call_status_hook: app.call_status_hook}, 'creating http client for call status hook');
       }
       else if (!app.notifier) {
         logger.debug('creating null call status hook');

lib/middleware.js

@@ -354,11 +354,13 @@ module.exports = function(srf, logger) {
       });
       // if transferred call contains callInfo, let update original data to newly created callInfo in this instance.
       if (app.transferredCall && app.callInfo) {
-        req.locals.callInfo.callerName = app.callInfo.callerName;
-        req.locals.callInfo.from = app.callInfo.from;
-        req.locals.callInfo.to = app.callInfo.to;
-        req.locals.callInfo.originatingSipIp = app.callInfo.originatingSipIp;
-        req.locals.callInfo.originatingSipTrunkName = app.callInfo.originatingSipTrunkName;
+        const {direction, callerName, from, to, originatingSipIp, originatingSipTrunkName} = app.callInfo;
+        req.locals.callInfo.direction = direction;
+        req.locals.callInfo.callerName = callerName;
+        req.locals.callInfo.from = from;
+        req.locals.callInfo.to = to;
+        req.locals.callInfo.originatingSipIp = originatingSipIp;
+        req.locals.callInfo.originatingSipTrunkName = originatingSipTrunkName;
         delete app.callInfo;
       }
       next();

lib/session/rest-call-session.js

@@ -46,6 +46,7 @@ class RestCallSession extends CallSession {
     this.dlg = dlg;
     dlg.on('destroy', this._callerHungup.bind(this));
     dlg.on('refer', this._onRefer.bind(this));
+    dlg.on('modify', this._onReinvite.bind(this));
     this.wrapDialog(dlg);
   }
 

lib/tasks/conference.js

@@ -544,6 +544,13 @@ class Conference extends Task {
     } while (!this.killed && this.conf_hold_status === 'hold');
   }
 
+  /**
+   * mute or unmute side of the call
+   */
+  mute(callSid, doMute) {
+    this.doConferenceMute(this.callSession, {conf_mute_status: doMute});
+  }
+
   /**
    * Add ourselves to the waitlist of sessions to be notified once
    * the conference starts

lib/tasks/dialogflow/index.js

@@ -1,3 +1,4 @@
+const assert = require('assert');
 const Task = require('../task');
 const {TaskName, TaskPreconditions} = require('../../utils/constants');
 const Intent = require('./intent');
@@ -10,19 +11,29 @@ class Dialogflow extends Task {
     super(logger, opts);
     this.preconditions = TaskPreconditions.Endpoint;
     this.credentials = this.data.credentials;
+    this.project = this.data.project;
+    this.agent = this.data.agent;
+    this.region = this.data.region || 'default';
+    this.model = this.data.model || 'es';
+    this.queryInput = this.data.queryInput || {};
 
-    /* set project id with environment and region (optionally) */
-    if (this.data.environment && this.data.region) {
-      this.project = `${this.data.project}:${this.data.environment}:${this.data.region}`;
-    }
-    else if (this.data.environment) {
-      this.project = `${this.data.project}:${this.data.environment}`;
-    }
-    else if (this.data.region) {
-      this.project = `${this.data.project}::${this.data.region}`;
+    assert(this.agent || !this.isCX, 'agent is required for dialogflow cx');
+    assert(this.credentials, 'dialogflow credentials are required');
+
+    if (this.isCX) {
+      this.environment = this.data.environment || 'default';
     }
     else {
-      this.project = this.data.project;
+      /* ES: set project id with environment and region (optionally) */
+      if (this.data.environment && this.data.region) {
+        this.project = `${this.data.project}:${this.data.environment}:${this.data.region}`;
+      }
+      else if (this.data.environment) {
+        this.project = `${this.data.project}:${this.data.environment}`;
+      }
+      else if (this.data.region) {
+        this.project = `${this.data.project}::${this.data.region}`;
+      }
     }
 
     this.lang = this.data.lang || 'en-US';
@@ -67,31 +78,23 @@ class Dialogflow extends Task {
       this.fallbackLabel = this.data.tts.fallbackLabel;
     }
     this.bargein = this.data.bargein;
+
+    this.cmd = this.model === 'cx' ? 'dialogflow_cx_start' : 'dialogflow_start';
+    this.cmdStop = this.model === 'cx' ? 'dialogflow_cx_stop' : 'dialogflow_stop';
   }
 
   get name() { return TaskName.Dialogflow; }
 
+  get isCX() { return this.model === 'cx'; }
+
+  get isES() { return !this.isCX; }
+
   async exec(cs, {ep}) {
     await super.exec(cs);
 
     try {
       await this.init(cs, ep);
-
-      this.logger.debug(`starting dialogflow bot ${this.project}`);
-
-      // kick it off
-      const baseArgs = `${this.ep.uuid} ${this.project} ${this.lang} ${this.welcomeEvent}`;
-      if (this.welcomeEventParams) {
-        this.ep.api('dialogflow_start', `${baseArgs} '${JSON.stringify(this.welcomeEventParams)}'`);
-      }
-      else if (this.welcomeEvent.length) {
-        this.ep.api('dialogflow_start', baseArgs);
-      }
-      else {
-        this.ep.api('dialogflow_start', `${this.ep.uuid} ${this.project} ${this.lang}`);
-      }
-      this.logger.debug(`started dialogflow bot ${this.project}`);
-
+      await this.startBot('default');
       await this.awaitTaskDone();
     } catch (err) {
       this.logger.error({err}, 'Dialogflow:exec error');
@@ -108,6 +111,12 @@ class Dialogflow extends Task {
       this.ep.removeCustomEventListener('dialogflow::end_of_utterance');
       this.ep.removeCustomEventListener('dialogflow::error');
 
+      this.ep.removeCustomEventListener('dialogflow_cx::intent');
+      this.ep.removeCustomEventListener('dialogflow_cx::transcription');
+      this.ep.removeCustomEventListener('dialogflow_cx::audio_provided');
+      this.ep.removeCustomEventListener('dialogflow_cx::end_of_utterance');
+      this.ep.removeCustomEventListener('dialogflow_cx::error');
+
       this._clearNoinputTimer();
 
       if (!this.reportedFinalAction) this.performAction({dialogflowResult: 'caller hungup'})
@@ -141,6 +150,12 @@ class Dialogflow extends Task {
       this.ep.addCustomEventListener('dialogflow::end_of_utterance', this._onEndOfUtterance.bind(this, ep, cs));
       this.ep.addCustomEventListener('dialogflow::error', this._onError.bind(this, ep, cs));
 
+      this.ep.addCustomEventListener('dialogflow_cx::intent', this._onIntent.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow_cx::transcription', this._onTranscription.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow_cx::audio_provided', this._onAudioProvided.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow_cx::end_of_utterance', this._onEndOfUtterance.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow_cx::error', this._onError.bind(this, ep, cs));
+
       const obj = typeof this.credentials === 'string' ? JSON.parse(this.credentials) : this.credentials;
       const creds = JSON.stringify(obj);
       await this.ep.set('GOOGLE_APPLICATION_CREDENTIALS', creds);
@@ -151,6 +166,51 @@ class Dialogflow extends Task {
     }
   }
 
+  async startBot(intent) {
+    if (this.isCX) {
+      await this.startBotCX(intent);
+    }
+    else {
+      await this.startBotES(intent);
+    }
+  }
+
+  async startBotES() {
+    this.logger.info('starting dialogflow ES bot');
+    const baseArgs = `${this.ep.uuid} ${this.project} ${this.lang} ${this.welcomeEvent}`;
+    if (this.welcomeEventParams) {
+      await this.ep.api(this.cmd, `${baseArgs} '${JSON.stringify(this.welcomeEventParams)}'`);
+    }
+    else if (this.welcomeEvent.length) {
+      await this.ep.api(this.cmd, baseArgs);
+    }
+    else {
+      await this.ep.api(this.cmd, `${this.ep.uuid} ${this.project} ${this.lang}`);
+    }
+  }
+
+  async startBotCX(intent) {
+    const baseArgs = [
+      this.ep.uuid,
+      this.region,
+      this.project,
+      this.agent,
+      this.environment,
+      this.lang,
+    ];
+    if (intent) {
+      baseArgs.push(intent);
+    }
+    /*
+    if (Object.keys(this.queryInput).length > 0) {
+      baseArgs.push(`'${JSON.stringify(this.queryInput)}'`);
+    }
+      */
+    this.logger.info({args: baseArgs}, 'starting dialogflow CX bot');
+
+    await this.ep.api(this.cmd, `${baseArgs.join(' ')}`);
+  }
+
   /**
    * An intent has been returned.  Since we are using SINGLE_UTTERANCE on the dialogflow side,
    * we may get an empty intent, signified by the lack of a 'response_id' attribute.
@@ -171,20 +231,20 @@ class Dialogflow extends Task {
       if (this.noinput && this.greetingPlayed) {
         this.logger.info('no input timer fired, reprompting..');
         this.noinput = false;
-        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang} ${this.noInputEvent}`);
+        ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang} ${this.noInputEvent}`);
       }
       else if (this.dtmfEntry && this.greetingPlayed) {
         this.logger.info('dtmf detected, reprompting..');
-        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang} none \'${this.dtmfEntry}\'`);
+        ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang} none \'${this.dtmfEntry}\'`);
         this.dtmfEntry = null;
       }
       else if (this.greetingPlayed) {
         this.logger.info('starting another intent');
-        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
+        ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang}`);
       }
       else {
         this.logger.info('got empty intent');
-        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
+        ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang}`);
       }
       return;
     }
@@ -231,7 +291,7 @@ class Dialogflow extends Task {
       // start a new intent, (we want to continue to listen during the audio playback)
       // _unless_ we are transferring or ending the session
       if (!this.hangupAfterPlayDone) {
-        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
+        ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang}`);
       }
 
       try {
@@ -386,10 +446,12 @@ class Dialogflow extends Task {
     // kill filler audio
     await ep.api('uuid_break', ep.uuid);
 
-    // start a new intent, (we want to continue to listen during the audio playback)
+    // if ES start a new intent (for CX we do not set single_utterance on),
+    // (we want to continue to listen during the audio playback)
     // _unless_ we are transferring or ending the session
-    if (/*this.greetingPlayed &&*/ !this.hangupAfterPlayDone) {
-      ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
+    if (!this.hangupAfterPlayDone) {
+      this.startBot();
+      //ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang}`);
     }
 
     this.playInProgress = true;
@@ -414,12 +476,7 @@ class Dialogflow extends Task {
         return;
       }
     }
-    /*
-    if (!this.inbound && !this.greetingPlayed) {
-      this.logger.info('finished greeting on outbound call, starting new intent');
-      this.ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
-    }
-    */
+
     this.greetingPlayed = true;
 
     if (this.hangupAfterPlayDone) {
@@ -454,7 +511,7 @@ class Dialogflow extends Task {
     }
 
     // kill the current dialogflow, which will result in us getting an immediate intent
-    ep.api('dialogflow_stop', `${ep.uuid}`)
+    ep.api(this.cmdStop, `${ep.uuid}`)
       .catch((err) => this.logger.info(`dialogflow_stop failed: ${err.message}`));
   }
 
@@ -472,7 +529,7 @@ class Dialogflow extends Task {
     }
 
     // kill the current dialogflow, which will result in us getting an immediate intent
-    ep.api('dialogflow_stop', `${ep.uuid}`)
+    ep.api(this.cmdStop, `${ep.uuid}`)
       .catch((err) => this.logger.info(`dialogflow_stop failed: ${err.message}`));
   }
 

lib/tasks/dialogflow/intent.js

@@ -4,19 +4,31 @@ class Intent {
     this.evt = evt;
 
     this.logger.debug({evt}, 'intent');
-    this.dtmfRequest = checkIntentForDtmfEntry(logger, evt);
+    this.qr = this.isCX ? evt.detect_intent_response.query_result : evt.query_result;
+    this.dtmfRequest = this._checkIntentForDtmfEntry(logger, evt);
+  }
+
+  get response_id() {
+    return this.isCX ? this.evt.detect_intent_response.response_id : this.evt.response_id;
   }
 
   get isEmpty() {
-    return this.evt.response_id.length === 0;
+    return !(this.response_id?.length > 0);
   }
 
   get fulfillmentText() {
-    return this.evt.query_result.fulfillment_text;
+    return this.qr.fulfillment_text;
   }
 
   get saysEndInteraction() {
-    return this.evt.query_result.intent.end_interaction ;
+    if (this.isCX) {
+      const end_interaction = this.qr.response_messages
+        .find((m) => typeof m === 'object' && 'end_interaction' in m)?.end_interaction;
+
+      //TODO: need to do more checking on the actual contents
+      return end_interaction && Object.keys(end_interaction).length > 0;
+    }
+    else return this.qr.intent.end_interaction ;
   }
 
   get saysCollectDtmf() {
@@ -28,7 +40,22 @@ class Intent {
   }
 
   get name() {
-    if (!this.isEmpty) return this.evt.query_result.intent.display_name;
+    if (!this.isEmpty) {
+      if (this.isCX) {
+        return this.qr.match?.intent?.display_name;
+      }
+      else {
+        return this.qr.intent.display_name;
+      }
+    }
+  }
+
+  get isCX() {
+    return typeof this.evt.detect_intent_response === 'object';
+  }
+
+  get isES() {
+    return !this.isCX;
   }
 
   toJSON() {
@@ -38,11 +65,7 @@ class Intent {
     };
   }
 
-}
-
-module.exports = Intent;
-
-/**
+  /**
  * Parse a returned intent for DTMF entry information
  * i.e.
  * allow-dtmf-x-y-z
@@ -55,35 +78,39 @@ module.exports = Intent;
  * allow-dtmf-1-4-# : collect 1-4 digits, terminating if '#' is entered
  * @param {*} intent - dialogflow intent
  */
-const checkIntentForDtmfEntry = (logger, intent) => {
-  const qr = intent.query_result;
-  if (!qr || !qr.fulfillment_messages || !qr.output_contexts) {
-    logger.info({f: qr.fulfillment_messages, o: qr.output_contexts}, 'no dtmfs');
-    return;
-  }
+  _checkIntentForDtmfEntry(logger, intent) {
+    const qr = this.isCX ? intent.detect_intent_response.query_result : intent.query_result;
 
-  // check for custom payloads with a gather verb
-  const custom = qr.fulfillment_messages.find((f) => f.payload && f.payload.verb === 'gather');
-  if (custom && custom.payload && custom.payload.verb === 'gather') {
-    logger.info({custom}, 'found dtmf custom payload');
-    return {
-      max: custom.payload.numDigits,
-      term: custom.payload.finishOnKey,
-      template: custom.payload.responseTemplate
-    };
-  }
+    if (!qr || !qr.fulfillment_messages || !qr.output_contexts) {
+      logger.info({f: qr.fulfillment_messages, o: qr.output_contexts}, 'no dtmfs');
+      return;
+    }
 
-  // check for an output context with a specific naming convention
-  const context = qr.output_contexts.find((oc) => oc.name.includes('/contexts/allow-dtmf-'));
-  if (context) {
-    const arr = /allow-dtmf-(\d+)(?:-(\d+))?(?:-(.*))?/.exec(context.name);
-    if (arr) {
-      logger.info({custom}, 'found dtmf output context');
+    // check for custom payloads with a gather verb
+    const custom = qr.fulfillment_messages.find((f) => f.payload && f.payload.verb === 'gather');
+    if (custom && custom.payload && custom.payload.verb === 'gather') {
+      logger.info({custom}, 'found dtmf custom payload');
       return {
-        min: parseInt(arr[1]),
-        max: arr.length > 2 ? parseInt(arr[2]) : null,
-        term: arr.length > 3 ? arr[3] : null
+        max: custom.payload.numDigits,
+        term: custom.payload.finishOnKey,
+        template: custom.payload.responseTemplate
       };
     }
+
+    // check for an output context with a specific naming convention
+    const context = qr.output_contexts.find((oc) => oc.name.includes('/contexts/allow-dtmf-'));
+    if (context) {
+      const arr = /allow-dtmf-(\d+)(?:-(\d+))?(?:-(.*))?/.exec(context.name);
+      if (arr) {
+        logger.info({custom}, 'found dtmf output context');
+        return {
+          min: parseInt(arr[1]),
+          max: arr.length > 2 ? parseInt(arr[2]) : null,
+          term: arr.length > 3 ? arr[3] : null
+        };
+      }
+    }
   }
-};
+}
+
+module.exports = Intent;

lib/tasks/gather.js

@@ -891,8 +891,9 @@ class TaskGather extends SttTask {
       if (this._timeoutTimer) {
         this._startTimer();
       }
-      /* restart asr timer if we get a partial transcript */
-      if (this.isContinuousAsr) this._startAsrTimer();
+      /* restart asr timer if we get a partial transcript (only if the asr timer is already running) */
+      /* note: https://github.com/jambonz/jambonz-feature-server/issues/866 */
+      if (this.isContinuousAsr && this._asrTimer) this._startAsrTimer();
     }
   }
   _onEndOfUtterance(cs, ep) {

lib/tasks/say.js

@@ -61,147 +61,6 @@ class TaskSay extends TtsTask {
     }
   }
 
-  async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label, preCache = false}) {
-    const {srf, accountSid:account_sid} = cs;
-    const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, srf);
-    const {writeAlerts, AlertType, stats} = srf.locals;
-    const {synthAudio} = srf.locals.dbHelpers;
-    const engine = this.synthesizer.engine || cs.synthesizer?.engine || 'neural';
-    const salt = cs.callSid;
-
-    let credentials = cs.getSpeechCredentials(vendor, 'tts', label);
-    /* parse Nuance voices into name and model */
-    let model;
-    if (vendor === 'nuance' && voice) {
-      const arr = /([A-Za-z-]*)\s+-\s+(enhanced|standard)/.exec(voice);
-      if (arr) {
-        voice = arr[1];
-        model = arr[2];
-      }
-    } else if (vendor === 'deepgram') {
-      model = voice;
-    }
-
-    /* allow for microsoft custom region voice and api_key to be specified as an override */
-    if (vendor === 'microsoft' && this.options.deploymentId) {
-      credentials = credentials || {};
-      credentials.use_custom_tts = true;
-      credentials.custom_tts_endpoint = this.options.deploymentId;
-      credentials.api_key = this.options.apiKey || credentials.apiKey;
-      credentials.region = this.options.region || credentials.region;
-      voice = this.options.voice || voice;
-    } else if (vendor === 'elevenlabs') {
-      credentials = credentials || {};
-      credentials.model_id = this.options.model_id || credentials.model_id;
-      credentials.voice_settings = this.options.voice_settings || {};
-      credentials.optimize_streaming_latency = this.options.optimize_streaming_latency
-      || credentials.optimize_streaming_latency;
-      voice = this.options.voice_id || voice;
-    }
-
-    ep.set({
-      tts_engine: vendor.startsWith('custom:') ? 'custom' : vendor,
-      tts_voice: voice,
-      cache_speech_handles: !cs.currentTtsVendor || cs.currentTtsVendor === vendor ? 1 : 0,
-    }).catch((err) => this.logger.info({err}, 'Error setting tts_engine on endpoint'));
-    // set the current vendor on the call session
-    // If vendor is changed from the previous one, then reset the cache_speech_handles flag
-    cs.currentTtsVendor = vendor;
-
-    if (!preCache && !this._disableTracing) this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
-    try {
-      if (!credentials) {
-        writeAlerts({
-          account_sid,
-          alert_type: AlertType.TTS_NOT_PROVISIONED,
-          vendor,
-          target_sid: cs.callSid
-        }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
-        throw new Error('no provisioned speech credentials for TTS');
-      }
-      // synthesize all of the text elements
-      let lastUpdated = false;
-
-      /* produce an audio segment from the provided text */
-      const generateAudio = async(text) => {
-        if (this.killed) return;
-        if (text.startsWith('silence_stream://')) return text;
-
-        /* otel: trace time for tts */
-        if (!preCache && !this._disableTracing)  {
-          const {span} = this.startChildSpan('tts-generation', {
-            'tts.vendor': vendor,
-            'tts.language': language,
-            'tts.voice': voice
-          });
-          this.otelSpan = span;
-        }
-        try {
-          const {filePath, servedFromCache, rtt} = await synthAudio(stats, {
-            account_sid,
-            text,
-            vendor,
-            language,
-            voice,
-            engine,
-            model,
-            salt,
-            credentials,
-            options: this.options,
-            disableTtsCache : this.disableTtsCache,
-            preCache
-          });
-          if (!filePath.startsWith('say:')) {
-            this.logger.debug(`Say: file ${filePath}, served from cache ${servedFromCache}`);
-            if (filePath) cs.trackTmpFile(filePath);
-            if (this.otelSpan) {
-              this.otelSpan.setAttributes({'tts.cached': servedFromCache});
-              this.otelSpan.end();
-              this.otelSpan = null;
-            }
-            if (!servedFromCache && !lastUpdated) {
-              lastUpdated = true;
-              updateSpeechCredentialLastUsed(credentials.speech_credential_sid).catch(() => {/* logged error */});
-            }
-            if (!servedFromCache && rtt && !preCache  && !this._disableTracing) {
-              this.notifyStatus({
-                event: 'synthesized-audio',
-                vendor,
-                language,
-                characters: text.length,
-                elapsedTime: rtt
-              });
-            }
-          }
-          else {
-            this.logger.debug('Say: a streaming tts api will be used');
-            const modifiedPath = filePath.replace('say:{', `say:{session-uuid=${ep.uuid},`);
-            return modifiedPath;
-          }
-          return filePath;
-        } catch (err) {
-          this.logger.info({err}, 'Error synthesizing tts');
-          if (this.otelSpan) this.otelSpan.end();
-          writeAlerts({
-            account_sid: cs.accountSid,
-            alert_type: AlertType.TTS_FAILURE,
-            vendor,
-            detail: err.message,
-            target_sid: cs.callSid
-          }).catch((err) => this.logger.info({err}, 'Error generating alert for tts failure'));
-          throw err;
-        }
-      };
-
-      const arr = this.text.map((t) => (this._validateURL(t) ? t : generateAudio(t)));
-      return (await Promise.all(arr)).filter((fp) => fp && fp.length);
-    } catch (err) {
-      this.logger.info(err, 'TaskSay:exec error');
-      throw err;
-    }
-
-  }
-
   async exec(cs, {ep}) {
     const {srf, accountSid:account_sid, callSid:target_sid} = cs;
     const {writeAlerts, AlertType} = srf.locals;

lib/tasks/tts-task.js

@@ -17,16 +17,26 @@ class TtsTask extends Task {
 
   async exec(cs) {
     super.exec(cs);
+    if (cs.synthesizer) {
+      this.options = {...cs.synthesizer.options, ...this.options};
+      this.data.synthesizer = this.data.synthesizer || {};
+      for (const k in cs.synthesizer) {
+        const newValue = this.data.synthesizer && this.data.synthesizer[k] !== undefined ?
+          this.data.synthesizer[k] :
+          cs.synthesizer[k];
+
+        if (Array.isArray(newValue)) {
+          this.data.synthesizer[k] = [...(this.data.synthesizer[k] || []), ...cs.synthesizer[k]];
+        } else if (typeof newValue === 'object' && newValue !== null) {
+          this.data.synthesizer[k] = { ...(this.data.synthesizer[k] || {}), ...cs.synthesizer[k] };
+        } else {
+          this.data.synthesizer[k] = newValue;
+        }
+      }
+    }
   }
 
-  async _synthesizeWithSpecificVendor(cs, ep, {
-    vendor,
-    language,
-    voice,
-    label,
-    disableTtsStreaming,
-    preCache
-  }) {
+  async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label, preCache = false}) {
     const {srf, accountSid:account_sid} = cs;
     const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, srf);
     const {writeAlerts, AlertType, stats} = srf.locals;
@@ -65,23 +75,23 @@ class TtsTask extends Task {
     }
 
     ep.set({
-      tts_engine: vendor,
+      tts_engine: vendor.startsWith('custom:') ? 'custom' : vendor,
       tts_voice: voice,
-      cache_speech_handles: 1,
-    }).catch((err) => this.logger.info({err}, `${this.name}: Error setting tts_engine on endpoint`));
+      cache_speech_handles: !cs.currentTtsVendor || cs.currentTtsVendor === vendor ? 1 : 0,
+    }).catch((err) => this.logger.info({err}, 'Error setting tts_engine on endpoint'));
+    // set the current vendor on the call session
+    // If vendor is changed from the previous one, then reset the cache_speech_handles flag
+    cs.currentTtsVendor = vendor;
 
-    if (!preCache) this.logger.info({vendor, language, voice, model}, `${this.name}:exec`);
+    if (!preCache && !this._disableTracing) this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
     try {
       if (!credentials) {
         writeAlerts({
           account_sid,
           alert_type: AlertType.TTS_NOT_PROVISIONED,
-          vendor
+          vendor,
+          target_sid: cs.callSid
         }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
-        this.notifyError({
-          msg: 'TTS error',
-          details:`No speech credentials provisioned for selected vendor ${vendor}`
-        });
         throw new Error('no provisioned speech credentials for TTS');
       }
       // synthesize all of the text elements
@@ -93,7 +103,7 @@ class TtsTask extends Task {
         if (text.startsWith('silence_stream://')) return text;
 
         /* otel: trace time for tts */
-        if (!preCache && !this.parentTask)  {
+        if (!preCache && !this._disableTracing)  {
           const {span} = this.startChildSpan('tts-generation', {
             'tts.vendor': vendor,
             'tts.language': language,
@@ -114,11 +124,10 @@ class TtsTask extends Task {
             credentials,
             options: this.options,
             disableTtsCache : this.disableTtsCache,
-            disableTtsStreaming,
-            preCache
+            renderForCaching: preCache
           });
           if (!filePath.startsWith('say:')) {
-            this.logger.debug(`file ${filePath}, served from cache ${servedFromCache}`);
+            this.logger.debug(`Say: file ${filePath}, served from cache ${servedFromCache}`);
             if (filePath) cs.trackTmpFile(filePath);
             if (this.otelSpan) {
               this.otelSpan.setAttributes({'tts.cached': servedFromCache});
@@ -129,7 +138,7 @@ class TtsTask extends Task {
               lastUpdated = true;
               updateSpeechCredentialLastUsed(credentials.speech_credential_sid).catch(() => {/* logged error */});
             }
-            if (!servedFromCache && rtt && !preCache) {
+            if (!servedFromCache && rtt && !preCache  && !this._disableTracing) {
               this.notifyStatus({
                 event: 'synthesized-audio',
                 vendor,
@@ -140,7 +149,7 @@ class TtsTask extends Task {
             }
           }
           else {
-            this.logger.debug('a streaming tts api will be used');
+            this.logger.debug('Say: a streaming tts api will be used');
             const modifiedPath = filePath.replace('say:{', `say:{session-uuid=${ep.uuid},`);
             return modifiedPath;
           }
@@ -155,7 +164,6 @@ class TtsTask extends Task {
             detail: err.message,
             target_sid: cs.callSid
           }).catch((err) => this.logger.info({err}, 'Error generating alert for tts failure'));
-          this.notifyError({msg: 'TTS error', details: err.message || err});
           throw err;
         }
       };
@@ -166,6 +174,7 @@ class TtsTask extends Task {
       this.logger.info(err, 'TaskSay:exec error');
       throw err;
     }
+
   }
 
   _validateURL(urlString) {

lib/utils/amd-utils.js

@@ -246,7 +246,10 @@ module.exports = (logger) => {
     const amd = ep.amd = new Amd(logger, cs, opts);
     const {vendor, language} = amd;
     let sttCredentials = amd.sttCredentials;
-    const hints = voicemailHints[language] || [];
+    // hints from configuration might be too long for specific language and vendor that make transcribe freeswitch
+    // modules cannot connect to the vendor. hints is used in next step to validate if the transcription
+    // matchs voice mail hints.
+    const hints = [];
 
     if (vendor === 'nuance' && sttCredentials.client_id) {
       /* get nuance access token */

package-lock.json

@@ -15,10 +15,10 @@
         "@jambonz/http-health-check": "^0.0.1",
         "@jambonz/mw-registrar": "^0.2.7",
         "@jambonz/realtimedb-helpers": "^0.8.8",
-        "@jambonz/speech-utils": "^0.1.13",
+        "@jambonz/speech-utils": "^0.1.15",
         "@jambonz/stats-collector": "^0.1.10",
         "@jambonz/time-series": "^0.2.9",
-        "@jambonz/verb-specifications": "^0.0.76",
+        "@jambonz/verb-specifications": "^0.0.79",
         "@opentelemetry/api": "^1.8.0",
         "@opentelemetry/exporter-jaeger": "^1.23.0",
         "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
@@ -1536,9 +1536,9 @@
       }
     },
     "node_modules/@jambonz/speech-utils": {
-      "version": "0.1.13",
-      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.1.13.tgz",
-      "integrity": "sha512-QeVmNFLtJGPGQfmp7jXpy742AyJIv2EteelDmNTqWGFEwTBj88q8GLP51hUsIR2ZbE5n/ZmZb/ytT6Y6LIQSDg==",
+      "version": "0.1.15",
+      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.1.15.tgz",
+      "integrity": "sha512-TICSKVqFqjc1Ty08Wr+byMQv1g/Piku0x/4cuw25JB3nrOolW/sLABRxoPOUUpbY5EGjXhaVT52FpazCopiIxg==",
       "license": "MIT",
       "dependencies": {
         "@aws-sdk/client-polly": "^3.496.0",
@@ -1575,9 +1575,9 @@
       }
     },
     "node_modules/@jambonz/verb-specifications": {
-      "version": "0.0.76",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.76.tgz",
-      "integrity": "sha512-7s61qAsG07xLLaEAHW236rSYzEoh9Qg0aRWHPbTfxCsuTKDNeq+5EwGAShDU5R5ZpjgweZJLhArQm8Ym+4xJ2A==",
+      "version": "0.0.79",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.79.tgz",
+      "integrity": "sha512-SJpUfRivPaBBF16sUhkKPuXC4KFf2vE03LuSNYGhtjzZ03PnIGXbsuz16cK+XeQow5tkof+ptmxwFgfv6TM5RQ==",
       "dependencies": {
         "debug": "^4.3.4",
         "pino": "^8.8.0"
@@ -10501,9 +10501,9 @@
       }
     },
     "@jambonz/speech-utils": {
-      "version": "0.1.13",
-      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.1.13.tgz",
-      "integrity": "sha512-QeVmNFLtJGPGQfmp7jXpy742AyJIv2EteelDmNTqWGFEwTBj88q8GLP51hUsIR2ZbE5n/ZmZb/ytT6Y6LIQSDg==",
+      "version": "0.1.15",
+      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.1.15.tgz",
+      "integrity": "sha512-TICSKVqFqjc1Ty08Wr+byMQv1g/Piku0x/4cuw25JB3nrOolW/sLABRxoPOUUpbY5EGjXhaVT52FpazCopiIxg==",
       "requires": {
         "@aws-sdk/client-polly": "^3.496.0",
         "@aws-sdk/client-sts": "^3.496.0",
@@ -10539,9 +10539,9 @@
       }
     },
     "@jambonz/verb-specifications": {
-      "version": "0.0.76",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.76.tgz",
-      "integrity": "sha512-7s61qAsG07xLLaEAHW236rSYzEoh9Qg0aRWHPbTfxCsuTKDNeq+5EwGAShDU5R5ZpjgweZJLhArQm8Ym+4xJ2A==",
+      "version": "0.0.79",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.79.tgz",
+      "integrity": "sha512-SJpUfRivPaBBF16sUhkKPuXC4KFf2vE03LuSNYGhtjzZ03PnIGXbsuz16cK+XeQow5tkof+ptmxwFgfv6TM5RQ==",
       "requires": {
         "debug": "^4.3.4",
         "pino": "^8.8.0"

package.json

@@ -31,10 +31,10 @@
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/mw-registrar": "^0.2.7",
     "@jambonz/realtimedb-helpers": "^0.8.8",
-    "@jambonz/speech-utils": "^0.1.13",
+    "@jambonz/speech-utils": "^0.1.15",
     "@jambonz/stats-collector": "^0.1.10",
     "@jambonz/time-series": "^0.2.9",
-    "@jambonz/verb-specifications": "^0.0.76",
+    "@jambonz/verb-specifications": "^0.0.79",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/exporter-jaeger": "^1.23.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",

test/docker-compose-testbed.yaml

@@ -57,7 +57,7 @@ services:
         condition: service_healthy
 
   freeswitch:
-    image: drachtio/drachtio-freeswitch-mrf:0.7.3
+    image: drachtio/drachtio-freeswitch-mrf:latest
     restart: always
     command: freeswitch --rtp-range-start 20000 --rtp-range-end 20100
     environment: