diff --git a/lib/tasks/cognigy.js b/lib/tasks/cognigy/index.js similarity index 80% rename from lib/tasks/cognigy.js rename to lib/tasks/cognigy/index.js index d3fb5c41..c7613397 100644 --- a/lib/tasks/cognigy.js +++ b/lib/tasks/cognigy/index.js @@ -1,7 +1,8 @@ -const Task = require('./task'); -const {TaskName, TaskPreconditions} = require('../utils/constants'); -const makeTask = require('./make_task'); +const Task = require('../task'); +const {TaskName, TaskPreconditions} = require('../../utils/constants'); +const makeTask = require('../make_task'); const { SocketClient } = require('@cognigy/socket-client'); +const SpeechConfig = require('./speech-config'); const parseGallery = (obj = {}) => { const {_default} = obj; @@ -56,27 +57,32 @@ class Cognigy extends Task { async exec(cs, ep) { await super.exec(cs); + const opts = { + session: { + synthesizer: this.data.synthesizer || { + vendor: 'default', + language: 'default', + voice: 'default' + }, + recognizer: this.data.recognizer || { + vendor: 'default', + language: 'default' + }, + bargein: this.data.bargein || {}, + bot: this.data.bot || {}, + user: this.data.user || {}, + dtmf: this.data.dtmf || {} + } + }; + this.config = new SpeechConfig({logger: this.logger, ep, opts}); this.ep = ep; try { + /* set event handlers and start transcribing */ this.on('transcription', this._onTranscription.bind(this, cs, ep)); + this.on('timeout', this._onTimeout.bind(this, cs, ep)); this.on('error', this._onError.bind(this, cs, ep)); - this.transcribeTask = this._makeTranscribeTask(); - this.transcribeTask.exec(cs, ep, this) - .catch((err) => { - this.logger.info({err}, 'Cognigy transcribe task returned error'); - this.notifyTaskDone(); - }); - if (this.prompt) { - this.sayTask = this._makeSayTask(this.prompt); - this.sayTask.exec(cs, ep, this) - .catch((err) => { - this.logger.info({err}, 'Cognigy say task returned error'); - this.notifyTaskDone(); - }); - } - /* connect to the bot and send initial data */ this.client = new SocketClient( this.url, @@ -92,7 +98,6 @@ class Cognigy extends Task { } ); this.client.on('output', this._onBotUtterance.bind(this, cs, ep)); - this.client.on('typingStatus', this._onBotTypingStatus.bind(this, cs, ep)); this.client.on('error', this._onBotError.bind(this, cs, ep)); this.client.on('finalPing', this._onBotFinalPing.bind(this, cs, ep)); await this.client.connect(); @@ -127,17 +132,10 @@ class Cognigy extends Task { this.notifyTaskDone(); } - _makeTranscribeTask() { - const opts = { - recognizer: this.data.recognizer || { - vendor: 'default', - language: 'default', - outputFormat: 'detailed' - } - }; - this.logger.debug({opts}, 'constructing a nested transcribe object'); - const transcribe = makeTask(this.logger, {transcribe: opts}, this); - return transcribe; + _makeGatherTask(prompt) { + const config = this.config.makeGatherTaskConfig(prompt); + const gather = makeTask(this.logger, {gather: config}, this); + return gather; } _makeSayTask(text) { @@ -162,23 +160,17 @@ class Cognigy extends Task { this.notifyTaskDone(); } - async _onBotTypingStatus(cs, ep, evt) { - this.logger.info({evt}, 'Cognigy:_onBotTypingStatus'); - } async _onBotFinalPing(cs, ep) { - this.logger.info('Cognigy:_onBotFinalPing'); + this.logger.info({prompts: this.prompts}, 'Cognigy:_onBotFinalPing'); if (this.prompts.length) { const text = this.prompts.join('.'); - this.prompts = []; if (text && !this.killed) { - this.sayTask = this._makeSayTask(text); - this.sayTask.exec(cs, ep, this) - .catch((err) => { - this.logger.info({err}, 'Cognigy say task returned error'); - this.notifyTaskDone(); - }); + this.gatherTask = this._makeGatherTask(text); + this.gatherTask.exec(cs, ep, this) + .catch((err) => this.logger.info({err}, 'Cognigy gather task returned error')); } } + this.prompts = []; } async _onBotUtterance(cs, ep, evt) { @@ -199,7 +191,8 @@ class Cognigy extends Task { }); } const text = parseBotText(evt); - this.prompts.push(text); + if (evt.data) this.config.update(evt.data); + if (text) this.prompts.push(text); } async _onTranscription(cs, ep, evt) { @@ -243,6 +236,13 @@ class Cognigy extends Task { this.reportedFinalAction = true; this.notifyTaskDone(); } + + _onTimeout(cs, ep, evt) { + this.logger.debug({evt}, 'Rasa: got timeout'); + if (!this.hasReportedFinalAction) this.performAction({rasaResult: 'timeout'}); + this.reportedFinalAction = true; + this.notifyTaskDone(); + } } module.exports = Cognigy; diff --git a/lib/tasks/cognigy/speech-config.js b/lib/tasks/cognigy/speech-config.js new file mode 100644 index 00000000..b50da9f7 --- /dev/null +++ b/lib/tasks/cognigy/speech-config.js @@ -0,0 +1,68 @@ +const Emitter = require('events'); + +const hasKeys = (obj) => typeof obj === 'object' && Object.keys(obj) > 0; + +class SpeechConfig extends Emitter { + constructor({logger, ep, opts = {}}) { + super(); + this.logger = logger; + this.ep = ep; + this.sessionConfig = opts.session || {}; + this.turnConfig = opts.nextTurn || {}; + this.update(opts); + } + + update(opts = {}) { + const {session, nextTurn = {}} = opts; + if (session) this.sessionConfig = {...this.sessionConfig, ...session}; + this.turnConfig = nextTurn; + this.logger.debug({opts, sessionLevel: this.sessionConfig, turnLevel: this.turnConfig}, 'SpeechConfig updated'); + } + + makeGatherTaskConfig(prompt) { + const opts = JSON.parse(JSON.stringify(this.sessionConfig || {})); + const nextTurnKeys = Object.keys(this.turnConfig || {}); + const newKeys = nextTurnKeys.filter((k) => !(k in opts)); + const bothKeys = nextTurnKeys.filter((k) => k in opts); + + for (const key of newKeys) opts[key] = this.turnConfig[key]; + for (const key of bothKeys) opts[key] = {...opts[key], ...this.turnConfig[key]}; + + this.logger.debug({ + opts, + sessionConfig: this.sessionConfig, + turnConfig: this.turnConfig, + }, 'Congigy SpeechConfig:_makeGatherTask current options'); + + /* input type: speech and/or dtmf entry */ + const input = []; + if (opts.recognizer) input.push('speech'); + if (hasKeys(opts.dtmf)) input.push('digits'); + + /* bargein settings */ + const bargein = opts.bargein || {}; + const speechBargein = Array.isArray(bargein.enable) && bargein.enable.includes('speech'); + const minBargeinWordCount = speechBargein ? (bargein.minWordCount || 1) : 0; + const sayConfig = { + text: prompt, + synthesizer: opts.synthesizer + }; + const config = { + input, + listenDuringPrompt: speechBargein, + bargein: speechBargein, + minBargeinWordCount, + recognizer: opts?.recognizer, + timeout: opts?.user?.noInputTimeout || 0, + say: sayConfig + }; + + this.logger.debug({config}, 'Congigy SpeechConfig:_makeGatherTask config'); + + /* turn config can now be emptied for next turn of conversation */ + this.turnConfig = {}; + return config; + } +} + +module.exports = SpeechConfig; diff --git a/lib/tasks/gather.js b/lib/tasks/gather.js index 77eb2fa9..34406bab 100644 --- a/lib/tasks/gather.js +++ b/lib/tasks/gather.js @@ -17,10 +17,11 @@ class TaskGather extends Task { [ 'finishOnKey', 'hints', 'input', 'numDigits', - 'partialResultHook', + 'partialResultHook', 'bargein', 'speechTimeout', 'timeout', 'say', 'play' ].forEach((k) => this[k] = this.data[k]); - + this.listenDuringPrompt = this.data.listenDuringPrompt === false ? false : true; + this.minBargeinWordCount = this.data.minBargeinWordCount || 1; this.timeout = (this.timeout || 5) * 1000; this.interim = this.partialResultCallback; if (this.data.recognizer) { @@ -80,22 +81,38 @@ class TaskGather extends Task { throw new Error(`no speech-to-text service credentials for ${this.vendor} have been configured`); } + const startListening = (cs, ep) => { + this._startTimer(); + if (this.input.includes('speech') && !this.listenDuringPrompt) { + this._initSpeech(cs, ep) + .then(() => { + this._startTranscribing(ep); + return updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid); + }) + .catch(() => {}); + } + }; + try { if (this.sayTask) { + this.logger.debug('Gather: kicking off say task'); this.sayTask.exec(cs, ep); // kicked off, _not_ waiting for it to complete - this.sayTask.on('playDone', (err) => { - if (!this.killed) this._startTimer(); + this.sayTask.on('playDone', async(err) => { + if (err) return this.logger.error({err}, 'Gather:exec Error playing tts'); + this.logger.debug('Gather: say task completed'); + if (!this.killed) startListening(cs, ep); }); } else if (this.playTask) { this.playTask.exec(cs, ep); // kicked off, _not_ waiting for it to complete - this.playTask.on('playDone', (err) => { - if (!this.killed) this._startTimer(); + this.playTask.on('playDone', async(err) => { + if (err) return this.logger.error({err}, 'Gather:exec Error playing url'); + if (!this.killed) startListening(cs, ep); }); } else this._startTimer(); - if (this.input.includes('speech')) { + if (this.input.includes('speech') && this.listenDuringPrompt) { await this._initSpeech(cs, ep); this._startTranscribing(ep); updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid) @@ -107,6 +124,7 @@ class TaskGather extends Task { } await this.awaitTaskDone(); + this.logger.debug('Gather:exec task has completed'); } catch (err) { this.logger.error(err, 'TaskGather:exec error'); } @@ -118,6 +136,7 @@ class TaskGather extends Task { } kill(cs) { + this.logger.debug('Gather:kill'); super.kill(cs); this._killAudio(cs); this.ep.removeAllListeners('dtmf'); @@ -197,7 +216,7 @@ class TaskGather extends Task { ep.startTranscription({ vendor: this.vendor, locale: this.language, - interim: this.partialResultCallback ? true : false, + interim: this.partialResultCallback || this.bargein, }).catch((err) => { const {writeAlerts, AlertType} = this.cs.srf.locals; this.logger.error(err, 'TaskGather:_startTranscribing error'); @@ -253,9 +272,15 @@ class TaskGather extends Task { } this.logger.debug(evt, 'TaskGather:_onTranscription'); if (evt.is_final) this._resolve('speech', evt); - else if (this.partialResultHook) { - this.cs.requestor.request(this.partialResultHook, Object.assign({speech: evt}, this.cs.callInfo)) - .catch((err) => this.logger.info(err, 'GatherTask:_onTranscription error')); + else { + if (this.bargein && evt.alternatives[0].transcript.split(' ').length >= this.minBargeinWordCount) { + this.logger.debug('Gather:_onTranscription - killing audio due to bargein'); + this._killAudio(cs); + } + if (this.partialResultHook) { + this.cs.requestor.request(this.partialResultHook, Object.assign({speech: evt}, this.cs.callInfo)) + .catch((err) => this.logger.info(err, 'GatherTask:_onTranscription error')); + } } } _onEndOfUtterance(cs, ep) { diff --git a/lib/tasks/say.js b/lib/tasks/say.js index 3d100f1d..dd92bd38 100644 --- a/lib/tasks/say.js +++ b/lib/tasks/say.js @@ -79,7 +79,11 @@ class TaskSay extends Task { const {memberId, confName, confUuid} = cs; await this.playToConfMember(this.ep, memberId, confName, confUuid, filepath[segment]); } - else await ep.play(filepath[segment]); + else { + this.logger.debug(`Say:exec sending command to play file ${filepath[segment]}`); + await ep.play(filepath[segment]); + this.logger.debug(`Say:exec completed play file ${filepath[segment]}`); + } } while (!this.killed && ++segment < filepath.length); } } catch (err) { diff --git a/lib/tasks/specs.json b/lib/tasks/specs.json index d6e3d27d..d454a6e5 100644 --- a/lib/tasks/specs.json +++ b/lib/tasks/specs.json @@ -100,6 +100,9 @@ "numDigits": "number", "partialResultHook": "object|string", "speechTimeout": "number", + "listenDuringPrompt": "boolean", + "bargein": "boolean", + "minBargeinWordCount": "number", "timeout": "number", "recognizer": "#recognizer", "play": "#play",