initial revamp of cognigy verb to use gather, accept session and turn-level config from bot

lib/tasks/cognigy/index.js

@@ -1,7 +1,8 @@
-const Task = require('./task');
+const Task = require('../task');
-const {TaskName, TaskPreconditions} = require('../utils/constants');
+const {TaskName, TaskPreconditions} = require('../../utils/constants');
-const makeTask = require('./make_task');
+const makeTask = require('../make_task');
 const { SocketClient } = require('@cognigy/socket-client');
+const SpeechConfig = require('./speech-config');
 
 const parseGallery = (obj = {}) => {
   const {_default} = obj;
@@ -56,27 +57,32 @@ class Cognigy extends Task {
   async exec(cs, ep) {
     await super.exec(cs);
 
+    const opts = {
+      session: {
+        synthesizer: this.data.synthesizer || {
+          vendor: 'default',
+          language: 'default',
+          voice: 'default'
+        },
+        recognizer: this.data.recognizer || {
+          vendor: 'default',
+          language: 'default'
+        },
+        bargein: this.data.bargein || {},
+        bot: this.data.bot || {},
+        user: this.data.user || {},
+        dtmf: this.data.dtmf || {}
+      }
+    };
+    this.config = new SpeechConfig({logger: this.logger, ep, opts});
     this.ep = ep;
     try {
+
       /* set event handlers and start transcribing */
       this.on('transcription', this._onTranscription.bind(this, cs, ep));
+      this.on('timeout', this._onTimeout.bind(this, cs, ep));
       this.on('error', this._onError.bind(this, cs, ep));
 
-      this.transcribeTask = this._makeTranscribeTask();
-      this.transcribeTask.exec(cs, ep, this)
-        .catch((err) => {
-          this.logger.info({err}, 'Cognigy transcribe task returned error');
-          this.notifyTaskDone();
-        });
-      if (this.prompt) {
-        this.sayTask = this._makeSayTask(this.prompt);
-        this.sayTask.exec(cs, ep, this)
-          .catch((err) => {
-            this.logger.info({err}, 'Cognigy say task returned error');
-            this.notifyTaskDone();
-          });
-      }
-
       /* connect to the bot and send initial data */
       this.client = new SocketClient(
         this.url,
@@ -92,7 +98,6 @@ class Cognigy extends Task {
         }
       );
       this.client.on('output', this._onBotUtterance.bind(this, cs, ep));
-      this.client.on('typingStatus', this._onBotTypingStatus.bind(this, cs, ep));
       this.client.on('error', this._onBotError.bind(this, cs, ep));
       this.client.on('finalPing', this._onBotFinalPing.bind(this, cs, ep));
       await this.client.connect();
@@ -127,17 +132,10 @@ class Cognigy extends Task {
     this.notifyTaskDone();
   }
 
-  _makeTranscribeTask() {
+  _makeGatherTask(prompt) {
-    const opts = {
+    const config = this.config.makeGatherTaskConfig(prompt);
-      recognizer: this.data.recognizer || {
+    const gather = makeTask(this.logger, {gather: config}, this);
-        vendor: 'default',
+    return gather;
-        language: 'default',
-        outputFormat: 'detailed'
-      }
-    };
-    this.logger.debug({opts}, 'constructing a nested transcribe object');
-    const transcribe = makeTask(this.logger, {transcribe: opts}, this);
-    return transcribe;
   }
 
   _makeSayTask(text) {
@@ -162,23 +160,17 @@ class Cognigy extends Task {
     this.notifyTaskDone();
   }
 
-  async _onBotTypingStatus(cs, ep, evt) {
-    this.logger.info({evt}, 'Cognigy:_onBotTypingStatus');
-  }
   async _onBotFinalPing(cs, ep) {
-    this.logger.info('Cognigy:_onBotFinalPing');
+    this.logger.info({prompts: this.prompts}, 'Cognigy:_onBotFinalPing');
     if (this.prompts.length) {
       const text = this.prompts.join('.');
-      this.prompts = [];
       if (text && !this.killed) {
-        this.sayTask = this._makeSayTask(text);
+        this.gatherTask = this._makeGatherTask(text);
-        this.sayTask.exec(cs, ep, this)
+        this.gatherTask.exec(cs, ep, this)
-          .catch((err) => {
+          .catch((err) => this.logger.info({err}, 'Cognigy gather task returned error'));
-            this.logger.info({err}, 'Cognigy say task returned error');
-            this.notifyTaskDone();
-          });
       }
     }
+    this.prompts = [];
   }
 
   async _onBotUtterance(cs, ep, evt) {
@@ -199,7 +191,8 @@ class Cognigy extends Task {
         });
     }
     const text = parseBotText(evt);
-    this.prompts.push(text);
+    if (evt.data) this.config.update(evt.data);
+    if (text) this.prompts.push(text);
   }
 
   async _onTranscription(cs, ep, evt) {
@@ -243,6 +236,13 @@ class Cognigy extends Task {
     this.reportedFinalAction = true;
     this.notifyTaskDone();
   }
+
+  _onTimeout(cs, ep, evt) {
+    this.logger.debug({evt}, 'Rasa: got timeout');
+    if (!this.hasReportedFinalAction) this.performAction({rasaResult: 'timeout'});
+    this.reportedFinalAction = true;
+    this.notifyTaskDone();
+  }
 }
 
 module.exports = Cognigy;

lib/tasks/cognigy/speech-config.js

@@ -0,0 +1,68 @@
+const Emitter = require('events');
+
+const hasKeys = (obj) => typeof obj === 'object' && Object.keys(obj) > 0;
+
+class SpeechConfig extends Emitter {
+  constructor({logger, ep, opts = {}}) {
+    super();
+    this.logger = logger;
+    this.ep = ep;
+    this.sessionConfig = opts.session || {};
+    this.turnConfig = opts.nextTurn || {};
+    this.update(opts);
+  }
+
+  update(opts = {}) {
+    const {session, nextTurn = {}} = opts;
+    if (session) this.sessionConfig = {...this.sessionConfig, ...session};
+    this.turnConfig = nextTurn;
+    this.logger.debug({opts, sessionLevel: this.sessionConfig, turnLevel: this.turnConfig}, 'SpeechConfig updated');
+  }
+
+  makeGatherTaskConfig(prompt) {
+    const opts = JSON.parse(JSON.stringify(this.sessionConfig || {}));
+    const nextTurnKeys = Object.keys(this.turnConfig || {});
+    const newKeys = nextTurnKeys.filter((k) => !(k in opts));
+    const bothKeys = nextTurnKeys.filter((k) => k in opts);
+
+    for (const key of newKeys) opts[key] = this.turnConfig[key];
+    for (const key of bothKeys) opts[key] = {...opts[key], ...this.turnConfig[key]};
+
+    this.logger.debug({
+      opts,
+      sessionConfig: this.sessionConfig,
+      turnConfig: this.turnConfig,
+    }, 'Congigy SpeechConfig:_makeGatherTask current options');
+
+    /* input type: speech and/or dtmf entry */
+    const input = [];
+    if (opts.recognizer) input.push('speech');
+    if (hasKeys(opts.dtmf)) input.push('digits');
+
+    /* bargein settings */
+    const bargein = opts.bargein || {};
+    const speechBargein = Array.isArray(bargein.enable) && bargein.enable.includes('speech');
+    const minBargeinWordCount = speechBargein ? (bargein.minWordCount || 1) : 0;
+    const sayConfig = {
+      text: prompt,
+      synthesizer: opts.synthesizer
+    };
+    const config = {
+      input,
+      listenDuringPrompt: speechBargein,
+      bargein: speechBargein,
+      minBargeinWordCount,
+      recognizer: opts?.recognizer,
+      timeout: opts?.user?.noInputTimeout || 0,
+      say: sayConfig
+    };
+
+    this.logger.debug({config}, 'Congigy SpeechConfig:_makeGatherTask config');
+
+    /* turn config can now be emptied for next turn of conversation */
+    this.turnConfig = {};
+    return config;
+  }
+}
+
+module.exports = SpeechConfig;

lib/tasks/gather.js

@@ -17,10 +17,11 @@ class TaskGather extends Task {
 
     [
       'finishOnKey', 'hints', 'input', 'numDigits',
-      'partialResultHook',
+      'partialResultHook', 'bargein',
       'speechTimeout', 'timeout', 'say', 'play'
     ].forEach((k) => this[k] = this.data[k]);
-
+    this.listenDuringPrompt = this.data.listenDuringPrompt === false ? false : true;
+    this.minBargeinWordCount = this.data.minBargeinWordCount || 1;
     this.timeout = (this.timeout || 5) * 1000;
     this.interim = this.partialResultCallback;
     if (this.data.recognizer) {
@@ -80,22 +81,38 @@ class TaskGather extends Task {
       throw new Error(`no speech-to-text service credentials for ${this.vendor} have been configured`);
     }
 
+    const startListening = (cs, ep) => {
+      this._startTimer();
+      if (this.input.includes('speech') && !this.listenDuringPrompt) {
+        this._initSpeech(cs, ep)
+          .then(() => {
+            this._startTranscribing(ep);
+            return updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid);
+          })
+          .catch(() => {});
+      }
+    };
+
     try {
       if (this.sayTask) {
+        this.logger.debug('Gather: kicking off say task');
         this.sayTask.exec(cs, ep);  // kicked off, _not_ waiting for it to complete
-        this.sayTask.on('playDone', (err) => {
+        this.sayTask.on('playDone', async(err) => {
-          if (!this.killed) this._startTimer();
+          if (err) return this.logger.error({err}, 'Gather:exec Error playing tts');
+          this.logger.debug('Gather: say task completed');
+          if (!this.killed) startListening(cs, ep);
         });
       }
       else if (this.playTask) {
         this.playTask.exec(cs, ep);  // kicked off, _not_ waiting for it to complete
-        this.playTask.on('playDone', (err) => {
+        this.playTask.on('playDone', async(err) => {
-          if (!this.killed) this._startTimer();
+          if (err) return this.logger.error({err}, 'Gather:exec Error playing url');
+          if (!this.killed) startListening(cs, ep);
         });
       }
       else this._startTimer();
 
-      if (this.input.includes('speech')) {
+      if (this.input.includes('speech') && this.listenDuringPrompt) {
         await this._initSpeech(cs, ep);
         this._startTranscribing(ep);
         updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid)
@@ -107,6 +124,7 @@ class TaskGather extends Task {
       }
 
       await this.awaitTaskDone();
+      this.logger.debug('Gather:exec task has completed');
     } catch (err) {
       this.logger.error(err, 'TaskGather:exec error');
     }
@@ -118,6 +136,7 @@ class TaskGather extends Task {
   }
 
   kill(cs) {
+    this.logger.debug('Gather:kill');
     super.kill(cs);
     this._killAudio(cs);
     this.ep.removeAllListeners('dtmf');
@@ -197,7 +216,7 @@ class TaskGather extends Task {
     ep.startTranscription({
       vendor: this.vendor,
       locale: this.language,
-      interim: this.partialResultCallback ? true : false,
+      interim: this.partialResultCallback || this.bargein,
     }).catch((err) => {
       const {writeAlerts, AlertType} = this.cs.srf.locals;
       this.logger.error(err, 'TaskGather:_startTranscribing error');
@@ -253,9 +272,15 @@ class TaskGather extends Task {
     }
     this.logger.debug(evt, 'TaskGather:_onTranscription');
     if (evt.is_final) this._resolve('speech', evt);
-    else if (this.partialResultHook) {
+    else {
-      this.cs.requestor.request(this.partialResultHook,  Object.assign({speech: evt}, this.cs.callInfo))
+      if (this.bargein && evt.alternatives[0].transcript.split(' ').length >= this.minBargeinWordCount) {
-        .catch((err) => this.logger.info(err, 'GatherTask:_onTranscription error'));
+        this.logger.debug('Gather:_onTranscription - killing audio due to bargein');
+        this._killAudio(cs);
+      }
+      if (this.partialResultHook) {
+        this.cs.requestor.request(this.partialResultHook,  Object.assign({speech: evt}, this.cs.callInfo))
+          .catch((err) => this.logger.info(err, 'GatherTask:_onTranscription error'));
+      }
     }
   }
   _onEndOfUtterance(cs, ep) {

lib/tasks/say.js

@@ -79,7 +79,11 @@ class TaskSay extends Task {
             const {memberId, confName, confUuid} = cs;
             await this.playToConfMember(this.ep, memberId, confName, confUuid, filepath[segment]);
           }
-          else await ep.play(filepath[segment]);
+          else {
+            this.logger.debug(`Say:exec sending command to play file ${filepath[segment]}`);
+            await ep.play(filepath[segment]);
+            this.logger.debug(`Say:exec completed play file ${filepath[segment]}`);
+          }
         } while (!this.killed && ++segment < filepath.length);
       }
     } catch (err) {

lib/tasks/specs.json

@@ -100,6 +100,9 @@
       "numDigits": "number",
       "partialResultHook": "object|string",
       "speechTimeout": "number",
+      "listenDuringPrompt": "boolean",
+      "bargein": "boolean",
+      "minBargeinWordCount": "number",
       "timeout": "number",
       "recognizer": "#recognizer",
       "play": "#play",