Compare commits

..

5 Commits

Author SHA1 Message Date
Dave Horton
77a696a0dc update to latest synthAudio with minor fixes 2022-01-27 13:52:35 -05:00
Dave Horton
62ff44540d more changes for wellsaid 2022-01-27 10:55:32 -05:00
Dave Horton
e5821cddf8 further fix for wellsaid tts 2022-01-27 10:46:16 -05:00
Dave Horton
25567a7842 add support for retrieving wellsaid speech credential 2022-01-27 10:34:30 -05:00
Dave Horton
40bd3c9c88 update to realtimedb-helpers with support for wellsaid tts 2022-01-27 10:13:18 -05:00
9 changed files with 101 additions and 271 deletions

View File

@@ -264,6 +264,12 @@ class CallSession extends Emitter {
region: credential.region
};
}
else if ('wellsaid' === vendor) {
return {
speech_credential_sid: credential.speech_credential_sid,
api_key: credential.api_key
};
}
}
else {
writeAlerts({

View File

@@ -1,8 +1,7 @@
const Task = require('../task');
const {TaskName, TaskPreconditions} = require('../../utils/constants');
const makeTask = require('../make_task');
const Task = require('./task');
const {TaskName, TaskPreconditions} = require('../utils/constants');
const makeTask = require('./make_task');
const { SocketClient } = require('@cognigy/socket-client');
const SpeechConfig = require('./speech-config');
const parseGallery = (obj = {}) => {
const {_default} = obj;
@@ -46,8 +45,6 @@ class Cognigy extends Task {
this.actionHook = this.data?.actionHook;
this.data = this.data.data || {};
this.prompts = [];
this.retry = {};
this.timeoutCount = 0;
}
get name() { return TaskName.Cognigy; }
@@ -59,33 +56,27 @@ class Cognigy extends Task {
async exec(cs, ep) {
await super.exec(cs);
const opts = {
session: {
synthesizer: this.data.synthesizer || {
vendor: 'default',
language: 'default',
voice: 'default'
},
recognizer: this.data.recognizer || {
vendor: 'default',
language: 'default'
},
bargein: this.data.bargein || {},
bot: this.data.bot || {},
user: this.data.user || {},
dtmf: this.data.dtmf || {}
}
};
this.config = new SpeechConfig({logger: this.logger, ep, opts});
this.ep = ep;
try {
/* set event handlers and start transcribing */
this.on('transcription', this._onTranscription.bind(this, cs, ep));
this.on('dtmf-collected', this._onDtmf.bind(this, cs, ep));
this.on('timeout', this._onTimeout.bind(this, cs, ep));
this.on('error', this._onError.bind(this, cs, ep));
this.transcribeTask = this._makeTranscribeTask();
this.transcribeTask.exec(cs, ep, this)
.catch((err) => {
this.logger.info({err}, 'Cognigy transcribe task returned error');
this.notifyTaskDone();
});
if (this.prompt) {
this.sayTask = this._makeSayTask(this.prompt);
this.sayTask.exec(cs, ep, this)
.catch((err) => {
this.logger.info({err}, 'Cognigy say task returned error');
this.notifyTaskDone();
});
}
/* connect to the bot and send initial data */
this.client = new SocketClient(
this.url,
@@ -101,6 +92,7 @@ class Cognigy extends Task {
}
);
this.client.on('output', this._onBotUtterance.bind(this, cs, ep));
this.client.on('typingStatus', this._onBotTypingStatus.bind(this, cs, ep));
this.client.on('error', this._onBotError.bind(this, cs, ep));
this.client.on('finalPing', this._onBotFinalPing.bind(this, cs, ep));
await this.client.connect();
@@ -135,12 +127,17 @@ class Cognigy extends Task {
this.notifyTaskDone();
}
_makeGatherTask({textPrompt, urlPrompt}) {
const config = this.config.makeGatherTaskConfig({textPrompt, urlPrompt});
const {retry, ...rest} = config;
this.retry = retry;
const gather = makeTask(this.logger, {gather: rest}, this);
return gather;
_makeTranscribeTask() {
const opts = {
recognizer: this.data.recognizer || {
vendor: 'default',
language: 'default',
outputFormat: 'detailed'
}
};
this.logger.debug({opts}, 'constructing a nested transcribe object');
const transcribe = makeTask(this.logger, {transcribe: opts}, this);
return transcribe;
}
_makeSayTask(text) {
@@ -153,6 +150,7 @@ class Cognigy extends Task {
voice: 'default'
}
};
this.logger.debug({opts}, 'constructing a nested say object');
const say = makeTask(this.logger, {say: opts}, this);
return say;
}
@@ -164,17 +162,23 @@ class Cognigy extends Task {
this.notifyTaskDone();
}
async _onBotTypingStatus(cs, ep, evt) {
this.logger.info({evt}, 'Cognigy:_onBotTypingStatus');
}
async _onBotFinalPing(cs, ep) {
this.logger.info({prompts: this.prompts}, 'Cognigy:_onBotFinalPing');
this.logger.info('Cognigy:_onBotFinalPing');
if (this.prompts.length) {
const text = this.prompts.join('.');
this.prompts = [];
if (text && !this.killed) {
this.gatherTask = this._makeGatherTask({textPrompt: text});
this.gatherTask.exec(cs, ep, this)
.catch((err) => this.logger.info({err}, 'Cognigy gather task returned error'));
this.sayTask = this._makeSayTask(text);
this.sayTask.exec(cs, ep, this)
.catch((err) => {
this.logger.info({err}, 'Cognigy say task returned error');
this.notifyTaskDone();
});
}
}
this.prompts = [];
}
async _onBotUtterance(cs, ep, evt) {
@@ -195,8 +199,7 @@ class Cognigy extends Task {
});
}
const text = parseBotText(evt);
if (evt.data) this.config.update(evt.data);
if (text) this.prompts.push(text);
this.prompts.push(text);
}
async _onTranscription(cs, ep, evt) {
@@ -234,46 +237,12 @@ class Cognigy extends Task {
this.notifyTaskDone();
}
}
_onDtmf(cs, ep, evt) {
this.logger.info({evt}, 'got dtmf');
/* send dtmf to bot */
try {
if (this.client && this.client.connected) {
this.client.sendMessage(evt.digits);
}
else {
this.logger.info('Cognigy_onTranscription - not sending user dtmf as bot is disconnected');
}
} catch (err) {
this.logger.error({err}, '_onDtmf: Error sending user dtmf to Cognigy - ending task');
this.performAction({cognigyResult: 'socketError'});
this.reportedFinalAction = true;
this.notifyTaskDone();
}
}
_onError(cs, ep, err) {
this.logger.info({err}, 'Cognigy: got error');
this.logger.debug({err}, 'Cognigy: got error');
if (!this.hasReportedFinalAction) this.performAction({cognigyResult: 'error', err});
this.reportedFinalAction = true;
this.notifyTaskDone();
}
_onTimeout(cs, ep, evt) {
const {noInputRetries, noInputSpeech, noInputUrl} = this.retry;
this.logger.debug({evt, retry: this.retry}, 'Cognigy: got timeout');
if (noInputRetries && this.timeoutCount++ < noInputRetries) {
this.gatherTask = this._makeGatherTask({textPrompt: noInputSpeech, urlPrompt: noInputUrl});
this.gatherTask.exec(cs, ep, this)
.catch((err) => this.logger.info({err}, 'Cognigy gather task returned error'));
}
else {
if (!this.hasReportedFinalAction) this.performAction({cognigyResult: 'timeout'});
this.reportedFinalAction = true;
this.notifyTaskDone();
}
}
}
module.exports = Cognigy;

View File

@@ -1,90 +0,0 @@
const Emitter = require('events');
const hasKeys = (obj) => typeof obj === 'object' && Object.keys(obj) > 0;
const stripNulls = (obj) => {
Object.keys(obj).forEach((k) => (obj[k] === null || typeof obj[k] === 'undefined') && delete obj[k]);
return obj;
};
class SpeechConfig extends Emitter {
constructor({logger, ep, opts = {}}) {
super();
this.logger = logger;
this.ep = ep;
this.sessionConfig = opts.session || {};
this.turnConfig = opts.nextTurn || {};
this.update(opts);
}
update(opts = {}) {
const {session, nextTurn = {}} = opts;
if (session) this.sessionConfig = {...this.sessionConfig, ...session};
this.turnConfig = nextTurn;
this.logger.debug({opts, sessionLevel: this.sessionConfig, turnLevel: this.turnConfig}, 'SpeechConfig updated');
}
makeGatherTaskConfig({textPrompt, urlPrompt}) {
const opts = JSON.parse(JSON.stringify(this.sessionConfig || {}));
const nextTurnKeys = Object.keys(this.turnConfig || {});
const newKeys = nextTurnKeys.filter((k) => !(k in opts));
const bothKeys = nextTurnKeys.filter((k) => k in opts);
for (const key of newKeys) opts[key] = this.turnConfig[key];
for (const key of bothKeys) opts[key] = {...opts[key], ...this.turnConfig[key]};
this.logger.debug({
opts,
sessionConfig: this.sessionConfig,
turnConfig: this.turnConfig,
}, 'Congigy SpeechConfig:_makeGatherTask current options');
/* input type: speech and/or dtmf entry */
const input = [];
if (opts.recognizer) input.push('speech');
if (hasKeys(opts.dtmf)) input.push('digits');
/* bargein settings */
const bargein = opts.bargein || {};
const speechBargein = Array.isArray(bargein.enable) && bargein.enable.includes('speech');
const dtmfBargein = Array.isArray(bargein.enable) && bargein.enable.includes('dtmf');
const minBargeinWordCount = speechBargein ? (bargein.minWordCount || 1) : 0;
const {interDigitTimeout = 0, maxDigits, minDigits = 1, submitDigit} = (opts.dtmf || {});
const {noInputTimeout, noInputRetries, noInputSpeech, noInputUrl} = (opts.user || {});
const sayConfig = {
text: textPrompt,
synthesizer: opts.synthesizer
};
const playConfig = {
url: urlPrompt
};
const config = {
input,
listenDuringPrompt: speechBargein,
bargein: speechBargein,
minBargeinWordCount,
dtmfBargein,
minDigits,
maxDigits,
interDigitTimeout,
finishOnKey: submitDigit,
recognizer: opts?.recognizer,
timeout: noInputTimeout,
retry : {
noInputRetries,
noInputSpeech,
noInputUrl
}
};
const final = stripNulls(config);
/* turn config can now be emptied for next turn of conversation */
this.turnConfig = {};
return textPrompt ?
{...final, say: sayConfig} :
{...final, play: playConfig};
}
}
module.exports = SpeechConfig;

View File

@@ -16,15 +16,13 @@ class TaskGather extends Task {
this.preconditions = TaskPreconditions.Endpoint;
[
'finishOnKey', 'hints', 'input', 'numDigits', 'minDigits', 'maxDigits',
'interDigitTimeout', 'submitDigit', 'partialResultHook', 'bargein', 'dtmfBargein',
'retries', 'retryPromptTts', 'retryPromptUrl',
'finishOnKey', 'hints', 'input', 'numDigits',
'partialResultHook',
'speechTimeout', 'timeout', 'say', 'play'
].forEach((k) => this[k] = this.data[k]);
this.listenDuringPrompt = this.data.listenDuringPrompt === false ? false : true;
this.minBargeinWordCount = this.data.minBargeinWordCount || 1;
this.timeout = (this.timeout || 15) * 1000;
this.interim = this.partialResultCallback || this.bargein;
this.timeout = (this.timeout || 5) * 1000;
this.interim = this.partialResultCallback;
if (this.data.recognizer) {
const recognizer = this.data.recognizer;
this.vendor = recognizer.vendor;
@@ -82,50 +80,33 @@ class TaskGather extends Task {
throw new Error(`no speech-to-text service credentials for ${this.vendor} have been configured`);
}
const startListening = (cs, ep) => {
this._startTimer();
if (this.input.includes('speech') && !this.listenDuringPrompt) {
this._initSpeech(cs, ep)
.then(() => {
this._startTranscribing(ep);
return updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid);
})
.catch(() => {});
}
};
try {
if (this.sayTask) {
this.logger.debug('Gather: kicking off say task');
this.sayTask.exec(cs, ep); // kicked off, _not_ waiting for it to complete
this.sayTask.on('playDone', async(err) => {
if (err) return this.logger.error({err}, 'Gather:exec Error playing tts');
this.logger.debug('Gather: say task completed');
if (!this.killed) startListening(cs, ep);
this.sayTask.on('playDone', (err) => {
if (!this.killed) this._startTimer();
});
}
else if (this.playTask) {
this.playTask.exec(cs, ep); // kicked off, _not_ waiting for it to complete
this.playTask.on('playDone', async(err) => {
if (err) return this.logger.error({err}, 'Gather:exec Error playing url');
if (!this.killed) startListening(cs, ep);
this.playTask.on('playDone', (err) => {
if (!this.killed) this._startTimer();
});
}
else this._startTimer();
if (this.input.includes('speech') && this.listenDuringPrompt) {
if (this.input.includes('speech')) {
await this._initSpeech(cs, ep);
this._startTranscribing(ep);
updateSpeechCredentialLastUsed(this.sttCredentials.speech_credential_sid)
.catch(() => {/*already logged error */});
}
if (this.input.includes('digits') || this.dtmfBargein) {
if (this.input.includes('digits')) {
ep.on('dtmf', this._onDtmf.bind(this, cs, ep));
}
await this.awaitTaskDone();
this.logger.debug('Gather:exec task has completed');
} catch (err) {
this.logger.error(err, 'TaskGather:exec error');
}
@@ -137,7 +118,6 @@ class TaskGather extends Task {
}
kill(cs) {
this.logger.debug('Gather:kill');
super.kill(cs);
this._killAudio(cs);
this.ep.removeAllListeners('dtmf');
@@ -146,28 +126,12 @@ class TaskGather extends Task {
_onDtmf(cs, ep, evt) {
this.logger.debug(evt, 'TaskGather:_onDtmf');
clearTimeout(this.interDigitTimer);
let resolved = false;
if (this.dtmfBargein) this._killAudio(cs);
if (evt.dtmf === this.finishOnKey) {
resolved = true;
this._resolve('dtmf-terminator-key');
}
if (evt.dtmf === this.finishOnKey) this._resolve('dtmf-terminator-key');
else {
this.digitBuffer += evt.dtmf;
const len = this.digitBuffer.length;
if (len === this.numDigits || len === this.maxDigits) {
resolved = true;
this._resolve('dtmf-num-digits');
}
}
if (!resolved && this.interDigitTimeout > 0 && this.digitBuffer.length >= this.minDigits) {
/* start interDigitTimer */
const ms = this.interDigitTimeout * 1000;
this.logger.debug(`starting interdigit timer of ${ms}`);
this.interDigitTimer = setTimeout(() => this._resolve('dtmf-interdigit-timeout'), ms);
if (this.digitBuffer.length === this.numDigits) this._resolve('dtmf-num-digits');
}
this._killAudio(cs);
}
async _initSpeech(cs, ep) {
@@ -233,7 +197,7 @@ class TaskGather extends Task {
ep.startTranscription({
vendor: this.vendor,
locale: this.language,
interim: this.interim,
interim: this.partialResultCallback ? true : false,
}).catch((err) => {
const {writeAlerts, AlertType} = this.cs.srf.locals;
this.logger.error(err, 'TaskGather:_startTranscribing error');
@@ -273,43 +237,25 @@ class TaskGather extends Task {
}
_onTranscription(cs, ep, evt) {
this.logger.debug(evt, 'TaskGather:_onTranscription');
if ('aws' === this.vendor && Array.isArray(evt) && evt.length > 0) evt = evt[0];
if ('microsoft' === this.vendor) {
const final = evt.RecognitionStatus === 'Success';
if (final) {
const nbest = evt.NBest;
evt = {
is_final: true,
alternatives: [
{
confidence: nbest[0].Confidence,
transcript: nbest[0].Display
}
]
};
}
else {
evt = {
is_final: false,
alternatives: [
{
transcript: evt.Text
}
]
};
}
const nbest = evt.NBest;
const newEvent = {
is_final: evt.RecognitionStatus === 'Success',
alternatives: [
{
confidence: nbest[0].Confidence,
transcript: nbest[0].Display
}
]
};
evt = newEvent;
}
this.logger.debug(evt, 'TaskGather:_onTranscription');
if (evt.is_final) this._resolve('speech', evt);
else {
if (this.bargein && evt.alternatives[0].transcript.split(' ').length >= this.minBargeinWordCount) {
this.logger.debug('Gather:_onTranscription - killing audio due to bargein');
this._killAudio(cs);
}
if (this.partialResultHook) {
this.cs.requestor.request(this.partialResultHook, Object.assign({speech: evt}, this.cs.callInfo))
.catch((err) => this.logger.info(err, 'GatherTask:_onTranscription error'));
}
else if (this.partialResultHook) {
this.cs.requestor.request(this.partialResultHook, Object.assign({speech: evt}, this.cs.callInfo))
.catch((err) => this.logger.info(err, 'GatherTask:_onTranscription error'));
}
}
_onEndOfUtterance(cs, ep) {
@@ -335,8 +281,7 @@ class TaskGather extends Task {
this._clearTimer();
if (reason.startsWith('dtmf')) {
if (this.parentTask) this.parentTask.emit('dtmf-collected', {reason, digits: this.digitBuffer});
else await this.performAction({digits: this.digitBuffer, reason: 'dtmfDetected'});
await this.performAction({digits: this.digitBuffer, reason: 'dtmfDetected'});
}
else if (reason.startsWith('speech')) {
if (this.parentTask) this.parentTask.emit('transcription', evt);

View File

@@ -79,11 +79,7 @@ class TaskSay extends Task {
const {memberId, confName, confUuid} = cs;
await this.playToConfMember(this.ep, memberId, confName, confUuid, filepath[segment]);
}
else {
this.logger.debug(`Say:exec sending command to play file ${filepath[segment]}`);
await ep.play(filepath[segment]);
this.logger.debug(`Say:exec completed play file ${filepath[segment]}`);
}
else await ep.play(filepath[segment]);
} while (!this.killed && ++segment < filepath.length);
}
} catch (err) {

View File

@@ -100,13 +100,6 @@
"numDigits": "number",
"partialResultHook": "object|string",
"speechTimeout": "number",
"listenDuringPrompt": "boolean",
"bargein": "boolean",
"minBargeinWordCount": "number",
"dtmfBargein": "boolean",
"minDigits": "number",
"maxDigits": "number",
"interDigitTimeout": "number",
"timeout": "number",
"recognizer": "#recognizer",
"play": "#play",

View File

@@ -36,6 +36,10 @@ const speechMapper = (cred) => {
obj.api_key = o.api_key;
obj.region = o.region;
}
else if ('wellsaid' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
}
return obj;
};
@@ -53,6 +57,7 @@ module.exports = (logger, srf) => {
const haveGoogle = speech.find((s) => s.vendor === 'google');
const haveAws = speech.find((s) => s.vendor === 'aws');
const haveMicrosoft = speech.find((s) => s.vendor === 'microsoft');
const haveWellsaid = speech.find((s) => s.vendor === 'wellsaid');
if (!haveGoogle || !haveAws || !haveMicrosoft) {
const [r3] = await pp.query(sqlSpeechCredentialsForSP, account_sid);
if (r3.length) {
@@ -68,6 +73,10 @@ module.exports = (logger, srf) => {
const ms = r3.find((s) => s.vendor === 'microsoft');
if (ms) speech.push(speechMapper(ms));
}
if (!haveWellsaid) {
const wellsaid = r3.find((s) => s.vendor === 'wellsaid');
if (wellsaid) speech.push(speechMapper(wellsaid));
}
}
}

16
package-lock.json generated
View File

@@ -13,7 +13,7 @@
"@jambonz/db-helpers": "^0.6.16",
"@jambonz/http-health-check": "^0.0.1",
"@jambonz/mw-registrar": "^0.2.1",
"@jambonz/realtimedb-helpers": "^0.4.17",
"@jambonz/realtimedb-helpers": "^0.4.19",
"@jambonz/stats-collector": "^0.1.6",
"@jambonz/time-series": "^0.1.5",
"aws-sdk": "^2.1060.0",
@@ -545,13 +545,14 @@
}
},
"node_modules/@jambonz/realtimedb-helpers": {
"version": "0.4.17",
"resolved": "https://registry.npmjs.org/@jambonz/realtimedb-helpers/-/realtimedb-helpers-0.4.17.tgz",
"integrity": "sha512-ERE+VLAmbZJ4DFIHXS689xr3EDVPWvpLxx69kSb/Z+QAZtbbv7a2E7XmJ6/QxF25VgKxNHOg949qYCFoWxb2RQ==",
"version": "0.4.19",
"resolved": "https://registry.npmjs.org/@jambonz/realtimedb-helpers/-/realtimedb-helpers-0.4.19.tgz",
"integrity": "sha512-wEc7OXogQ5SZ9mgLNxviBNY0WvUKGGztyDzze3oi44edsZp0vNneVHUmdZuDzuDvUMBqsvCa0fvri8LGaxovlw==",
"dependencies": {
"@google-cloud/text-to-speech": "^3.4.0",
"@jambonz/promisify-redis": "^0.0.6",
"aws-sdk": "^2.1060.0",
"bent": "^7.3.12",
"debug": "^4.3.3",
"microsoft-cognitiveservices-speech-sdk": "^1.19.0",
"redis": "^3.1.2"
@@ -5875,13 +5876,14 @@
}
},
"@jambonz/realtimedb-helpers": {
"version": "0.4.17",
"resolved": "https://registry.npmjs.org/@jambonz/realtimedb-helpers/-/realtimedb-helpers-0.4.17.tgz",
"integrity": "sha512-ERE+VLAmbZJ4DFIHXS689xr3EDVPWvpLxx69kSb/Z+QAZtbbv7a2E7XmJ6/QxF25VgKxNHOg949qYCFoWxb2RQ==",
"version": "0.4.19",
"resolved": "https://registry.npmjs.org/@jambonz/realtimedb-helpers/-/realtimedb-helpers-0.4.19.tgz",
"integrity": "sha512-wEc7OXogQ5SZ9mgLNxviBNY0WvUKGGztyDzze3oi44edsZp0vNneVHUmdZuDzuDvUMBqsvCa0fvri8LGaxovlw==",
"requires": {
"@google-cloud/text-to-speech": "^3.4.0",
"@jambonz/promisify-redis": "^0.0.6",
"aws-sdk": "^2.1060.0",
"bent": "^7.3.12",
"debug": "^4.3.3",
"microsoft-cognitiveservices-speech-sdk": "^1.19.0",
"redis": "^3.1.2"

View File

@@ -30,7 +30,7 @@
"@jambonz/db-helpers": "^0.6.16",
"@jambonz/http-health-check": "^0.0.1",
"@jambonz/mw-registrar": "^0.2.1",
"@jambonz/realtimedb-helpers": "^0.4.17",
"@jambonz/realtimedb-helpers": "^0.4.19",
"@jambonz/stats-collector": "^0.1.6",
"@jambonz/time-series": "^0.1.5",
"aws-sdk": "^2.1060.0",