ben.carrasco/jambonz-feature-server
1
0
Fork 0
mirror of https://github.com/jambonz/jambonz-feature-server.git synced 2025-12-21 17:17:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

initial revamp of cognigy verb to use gather, accept session and turn-level config from bot

Browse Source
This commit is contained in:
Dave Horton
2022-01-23 23:07:00 -05:00
parent 27d6d32359
commit b94605127e
5 changed files with 154 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
lib/tasks/cognigy.js → lib/tasks/cognigy/index.js
View File

@@ -1,7 +1,8 @@
const Task = require('./task');
const {TaskName, TaskPreconditions} = require('../utils/constants');
const makeTask = require('./make_task');
const Task = require('../task');
const {TaskName, TaskPreconditions} = require('../../utils/constants');
const makeTask = require('../make_task');
const { SocketClient } = require('@cognigy/socket-client');
const SpeechConfig = require('./speech-config');
const parseGallery = (obj = {}) => {
const {_default} = obj;
@@ -56,27 +57,32 @@ class Cognigy extends Task {
async exec(cs, ep) {
await super.exec(cs);
const opts = {
session: {
synthesizer: this.data.synthesizer || {
vendor: 'default',
language: 'default',
voice: 'default'
},
recognizer: this.data.recognizer || {
vendor: 'default',
language: 'default'
},
bargein: this.data.bargein || {},
bot: this.data.bot || {},
user: this.data.user || {},
dtmf: this.data.dtmf || {}
}
};
this.config = new SpeechConfig({logger: this.logger, ep, opts});
this.ep = ep;
try {
/* set event handlers and start transcribing */
this.on('transcription', this._onTranscription.bind(this, cs, ep));
this.on('timeout', this._onTimeout.bind(this, cs, ep));
this.on('error', this._onError.bind(this, cs, ep));
this.transcribeTask = this._makeTranscribeTask();
this.transcribeTask.exec(cs, ep, this)
.catch((err) => {
this.logger.info({err}, 'Cognigy transcribe task returned error');
this.notifyTaskDone();
});
if (this.prompt) {
this.sayTask = this._makeSayTask(this.prompt);
this.sayTask.exec(cs, ep, this)
.catch((err) => {
this.logger.info({err}, 'Cognigy say task returned error');
this.notifyTaskDone();
});
}
/* connect to the bot and send initial data */
this.client = new SocketClient(
this.url,
@@ -92,7 +98,6 @@ class Cognigy extends Task {
}
);
this.client.on('output', this._onBotUtterance.bind(this, cs, ep));
this.client.on('typingStatus', this._onBotTypingStatus.bind(this, cs, ep));
this.client.on('error', this._onBotError.bind(this, cs, ep));
this.client.on('finalPing', this._onBotFinalPing.bind(this, cs, ep));
await this.client.connect();
@@ -127,17 +132,10 @@ class Cognigy extends Task {
this.notifyTaskDone();
}
_makeTranscribeTask() {
const opts = {
recognizer: this.data.recognizer || {
vendor: 'default',
language: 'default',
outputFormat: 'detailed'
}
};
this.logger.debug({opts}, 'constructing a nested transcribe object');
const transcribe = makeTask(this.logger, {transcribe: opts}, this);
return transcribe;
_makeGatherTask(prompt) {
const config = this.config.makeGatherTaskConfig(prompt);
const gather = makeTask(this.logger, {gather: config}, this);
return gather;
}
_makeSayTask(text) {
@@ -162,23 +160,17 @@ class Cognigy extends Task {
this.notifyTaskDone();
}
async _onBotTypingStatus(cs, ep, evt) {
this.logger.info({evt}, 'Cognigy:_onBotTypingStatus');
}
async _onBotFinalPing(cs, ep) {
this.logger.info('Cognigy:_onBotFinalPing');
this.logger.info({prompts: this.prompts}, 'Cognigy:_onBotFinalPing');
if (this.prompts.length) {
const text = this.prompts.join('.');
this.prompts = [];
if (text && !this.killed) {
this.sayTask = this._makeSayTask(text);
this.sayTask.exec(cs, ep, this)
.catch((err) => {
this.logger.info({err}, 'Cognigy say task returned error');
this.notifyTaskDone();
});
this.gatherTask = this._makeGatherTask(text);
this.gatherTask.exec(cs, ep, this)
.catch((err) => this.logger.info({err}, 'Cognigy gather task returned error'));
}
}
this.prompts = [];
}
async _onBotUtterance(cs, ep, evt) {
@@ -199,7 +191,8 @@ class Cognigy extends Task {
});
}
const text = parseBotText(evt);
this.prompts.push(text);
if (evt.data) this.config.update(evt.data);
if (text) this.prompts.push(text);
}
async _onTranscription(cs, ep, evt) {
@@ -243,6 +236,13 @@ class Cognigy extends Task {
this.reportedFinalAction = true;
this.notifyTaskDone();
}
_onTimeout(cs, ep, evt) {
this.logger.debug({evt}, 'Rasa: got timeout');
if (!this.hasReportedFinalAction) this.performAction({rasaResult: 'timeout'});
this.reportedFinalAction = true;
this.notifyTaskDone();
}
}
module.exports = Cognigy;

68
lib/tasks/cognigy/speech-config.js Normal file
View File

@@ -0,0 +1,68 @@
const Emitter = require('events');
const hasKeys = (obj) => typeof obj === 'object' && Object.keys(obj) > 0;
class SpeechConfig extends Emitter {
constructor({logger, ep, opts = {}}) {
super();
this.logger = logger;
this.ep = ep;
this.sessionConfig = opts.session || {};
this.turnConfig = opts.nextTurn || {};
this.update(opts);
}
update(opts = {}) {
const {session, nextTurn = {}} = opts;
if (session) this.sessionConfig = {...this.sessionConfig, ...session};
this.turnConfig = nextTurn;
this.logger.debug({opts, sessionLevel: this.sessionConfig, turnLevel: this.turnConfig}, 'SpeechConfig updated');
}
makeGatherTaskConfig(prompt) {
const opts = JSON.parse(JSON.stringify(this.sessionConfig || {}));
const nextTurnKeys = Object.keys(this.turnConfig || {});
const newKeys = nextTurnKeys.filter((k) => !(k in opts));
const bothKeys = nextTurnKeys.filter((k) => k in opts);
for (const key of newKeys) opts[key] = this.turnConfig[key];
for (const key of bothKeys) opts[key] = {...opts[key], ...this.turnConfig[key]};
this.logger.debug({
opts,
sessionConfig: this.sessionConfig,
turnConfig: this.turnConfig,
}, 'Congigy SpeechConfig:_makeGatherTask current options');
/* input type: speech and/or dtmf entry */
const input = [];
if (opts.recognizer) input.push('speech');
if (hasKeys(opts.dtmf)) input.push('digits');
/* bargein settings */
const bargein = opts.bargein || {};
const speechBargein = Array.isArray(bargein.enable) && bargein.enable.includes('speech');
const minBargeinWordCount = speechBargein ? (bargein.minWordCount || 1) : 0;
const sayConfig = {
text: prompt,
synthesizer: opts.synthesizer
};
const config = {
input,
listenDuringPrompt: speechBargein,
bargein: speechBargein,
minBargeinWordCount,
recognizer: opts?.recognizer,
timeout: opts?.user?.noInputTimeout || 0,
say: sayConfig
};
this.logger.debug({config}, 'Congigy SpeechConfig:_makeGatherTask config');
/* turn config can now be emptied for next turn of conversation */
this.turnConfig = {};
return config;
}
}
module.exports = SpeechConfig;

47
lib/tasks/gather.js
View File

@@ -17,10 +17,11 @@ class TaskGather extends Task {
[
'finishOnKey', 'hints', 'input', 'numDigits',
'partialResultHook',
'partialResultHook', 'bargein',
'speechTimeout', 'timeout', 'say', 'play'
].forEach((k) => this[k] = this.data[k]);
this.listenDuringPrompt = this.data.listenDuringPrompt === false ? false : true;
this.minBargeinWordCount = this.data.minBargeinWordCount || 1;
this.timeout = (this.timeout || 5) * 1000;
this.interim = this.partialResultCallback;
if (this.data.recognizer) {
@@ -80,22 +81,38 @@ class TaskGather extends Task {
throw new Error(`no speech-to-text service credentials for ${this.vendor} have been configured`);
}
const startListening = (cs, ep) => {
this._startTimer();
if (this.input.includes('speech') && !this.listenDuringPrompt) {
this._initSpeech(cs, ep)
.then(() => {
this._startTranscribing(ep);
return updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid);
})
.catch(() => {});
}
};
try {
if (this.sayTask) {
this.logger.debug('Gather: kicking off say task');
this.sayTask.exec(cs, ep); // kicked off, _not_ waiting for it to complete
this.sayTask.on('playDone', (err) => {
if (!this.killed) this._startTimer();
this.sayTask.on('playDone', async(err) => {
if (err) return this.logger.error({err}, 'Gather:exec Error playing tts');
this.logger.debug('Gather: say task completed');
if (!this.killed) startListening(cs, ep);
});
}
else if (this.playTask) {
this.playTask.exec(cs, ep); // kicked off, _not_ waiting for it to complete
this.playTask.on('playDone', (err) => {
if (!this.killed) this._startTimer();
this.playTask.on('playDone', async(err) => {
if (err) return this.logger.error({err}, 'Gather:exec Error playing url');
if (!this.killed) startListening(cs, ep);
});
}
else this._startTimer();
if (this.input.includes('speech')) {
if (this.input.includes('speech') && this.listenDuringPrompt) {
await this._initSpeech(cs, ep);
this._startTranscribing(ep);
updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid)
@@ -107,6 +124,7 @@ class TaskGather extends Task {
}
await this.awaitTaskDone();
this.logger.debug('Gather:exec task has completed');
} catch (err) {
this.logger.error(err, 'TaskGather:exec error');
}
@@ -118,6 +136,7 @@ class TaskGather extends Task {
}
kill(cs) {
this.logger.debug('Gather:kill');
super.kill(cs);
this._killAudio(cs);
this.ep.removeAllListeners('dtmf');
@@ -197,7 +216,7 @@ class TaskGather extends Task {
ep.startTranscription({
vendor: this.vendor,
locale: this.language,
interim: this.partialResultCallback ? true : false,
interim: this.partialResultCallback || this.bargein,
}).catch((err) => {
const {writeAlerts, AlertType} = this.cs.srf.locals;
this.logger.error(err, 'TaskGather:_startTranscribing error');
@@ -253,9 +272,15 @@ class TaskGather extends Task {
}
this.logger.debug(evt, 'TaskGather:_onTranscription');
if (evt.is_final) this._resolve('speech', evt);
else if (this.partialResultHook) {
this.cs.requestor.request(this.partialResultHook, Object.assign({speech: evt}, this.cs.callInfo))
.catch((err) => this.logger.info(err, 'GatherTask:_onTranscription error'));
else {
if (this.bargein && evt.alternatives[0].transcript.split(' ').length >= this.minBargeinWordCount) {
this.logger.debug('Gather:_onTranscription - killing audio due to bargein');
this._killAudio(cs);
}
if (this.partialResultHook) {
this.cs.requestor.request(this.partialResultHook, Object.assign({speech: evt}, this.cs.callInfo))
.catch((err) => this.logger.info(err, 'GatherTask:_onTranscription error'));
}
}
}
_onEndOfUtterance(cs, ep) {

6
lib/tasks/say.js
View File

@@ -79,7 +79,11 @@ class TaskSay extends Task {
const {memberId, confName, confUuid} = cs;
await this.playToConfMember(this.ep, memberId, confName, confUuid, filepath[segment]);
}
else await ep.play(filepath[segment]);
else {
this.logger.debug(`Say:exec sending command to play file ${filepath[segment]}`);
await ep.play(filepath[segment]);
this.logger.debug(`Say:exec completed play file ${filepath[segment]}`);
}
} while (!this.killed && ++segment < filepath.length);
}
} catch (err) {

3
lib/tasks/specs.json
View File

@@ -100,6 +100,9 @@
"numDigits": "number",
"partialResultHook": "object|string",
"speechTimeout": "number",
"listenDuringPrompt": "boolean",
"bargein": "boolean",
"minBargeinWordCount": "number",
"timeout": "number",
"recognizer": "#recognizer",
"play": "#play",