diff --git a/lib/session/call-session.js b/lib/session/call-session.js index 47ee2ad8..6ddfbfb7 100644 --- a/lib/session/call-session.js +++ b/lib/session/call-session.js @@ -462,11 +462,11 @@ class CallSession extends Emitter { * @param {Task} task - task to be executed */ async _evalEndpointPrecondition(task) { - this.logger.debug('_evalEndpointPrecondition'); + this.logger.debug('CallSession:_evalEndpointPrecondition'); if (this.callGone) new Error(`${BADPRECONDITIONS}: call gone`); if (this.ep) { - if (!task.earlyMedia || this.dlg) return this.ep; + if (task.earlyMedia === true || this.dlg) return this.ep; // we are going from an early media connection to answer await this.propagateAnswer(); diff --git a/lib/tasks/dialogflow/digit-buffer.js b/lib/tasks/dialogflow/digit-buffer.js new file mode 100644 index 00000000..d6011f18 --- /dev/null +++ b/lib/tasks/dialogflow/digit-buffer.js @@ -0,0 +1,70 @@ +const Emitter = require('events'); + +/** + * A dtmf collector + * @class + */ +class DigitBuffer extends Emitter { + /** + * Creates a DigitBuffer + * @param {*} logger - a pino logger + * @param {*} opts - dtmf collection instructions + */ + constructor(logger, opts) { + super(); + this.logger = logger; + this.minDigits = opts.min || 1; + this.maxDigits = opts.max || 99; + this.termDigit = opts.term; + this.interdigitTimeout = opts.idt || 8000; + this.template = opts.template; + this.buffer = ''; + this.logger.debug(`digitbuffer min: ${this.minDigits} max: ${this.maxDigits} term digit: ${this.termDigit}`); + } + + /** + * process a received dtmf digit + * @param {String} a single digit entered by the caller + */ + process(digit) { + this.logger.debug(`digitbuffer process: ${digit}`); + if (digit === this.termDigit) return this._fulfill(); + this.buffer += digit; + if (this.buffer.length === this.maxDigits) return this._fulfill(); + if (this.buffer.length >= this.minDigits) this._startInterDigitTimer(); + this.logger.debug(`digitbuffer buffer: ${this.buffer}`); + } + + /** + * clear the digit buffer + */ + flush() { + if (this.idtimer) clearTimeout(this.idtimer); + this.buffer = ''; + } + + _fulfill() { + this.logger.debug(`digit buffer fulfilled with ${this.buffer}`); + if (this.template && this.template.includes('${digits}')) { + const text = this.template.replace('${digits}', this.buffer); + this.logger.info(`reporting dtmf as ${text}`); + this.emit('fulfilled', text); + } + else { + this.emit('fulfilled', this.buffer); + } + this.flush(); + } + + _startInterDigitTimer() { + if (this.idtimer) clearTimeout(this.idtimer); + this.idtimer = setTimeout(this._onInterDigitTimeout.bind(this), this.interdigitTimeout); + } + + _onInterDigitTimeout() { + this.logger.debug('digit buffer timeout'); + this._fulfill(); + } +} + +module.exports = DigitBuffer; diff --git a/lib/tasks/dialogflow/index.js b/lib/tasks/dialogflow/index.js new file mode 100644 index 00000000..acb6c221 --- /dev/null +++ b/lib/tasks/dialogflow/index.js @@ -0,0 +1,337 @@ +const Task = require('../task'); +const {TaskName, TaskPreconditions} = require('../../utils/constants'); +const Intent = require('./intent'); +const DigitBuffer = require('./digit-buffer'); +const Transcription = require('./transcription'); + +class Dialogflow extends Task { + constructor(logger, opts) { + super(logger, opts); + this.preconditions = TaskPreconditions.Endpoint; + + this.credentials = this.data.credentials; + this.project = this.data.project; + this.lang = this.data.lang || 'en-US'; + this.welcomeEvent = this.data.welcomeEvent || ''; + if (this.welcomeEvent.length && this.data.welcomeEventParams && typeof this.data.welcomeEventParams === 'object') { + this.welcomeEventParams = this.data.welcomeEventParams; + } + this.noInputTimeout = this.data.noInputTimeout ; + this.passDtmfAsInputText = this.passDtmfAsInputText === true; + if (this.data.eventHook) this.eventHook = this.data.eventHook; + if (this.eventHook && Array.isArray(this.data.events)) { + this.events = this.data.events; + } + else if (this.eventHook) { + // send all events by default + this.events = [ + 'intent', + 'transcription', + 'dtmf', + 'end-utterance', + 'start-play', + 'end-play', + 'no-input' + ]; + } + else { + this.events = []; + } + if (this.data.actionHook) this.actionHook = this.data.actionHook; + if (this.data.thinkingMusic) this.thinkingMusic = this.data.thinkingMusic; + } + + get name() { return TaskName.Dialogflow; } + + async exec(cs, ep) { + await super.exec(cs); + + try { + await this.init(cs, ep); + + this.logger.debug(`starting dialogflow bot ${this.project}`); + + // kick it off + if (this.welcomeEventParams) { + this.ep.api('dialogflow_start', + `${this.ep.uuid} ${this.project} ${this.lang} ${this.welcomeEvent} '${JSON.stringify(this.welcomeEventParams)}'`); + } + else if (this.welcomeEvent.length) { + this.ep.api('dialogflow_start', + `${this.ep.uuid} ${this.project} ${this.lang} ${this.welcomeEvent}`); + } + else { + this.ep.api('dialogflow_start', `${this.ep.uuid} ${this.project} ${this.lang}`); + } + this.logger.debug(`started dialogflow bot ${this.project}`); + + await this.awaitTaskDone(); + } catch (err) { + this.logger.error({err}, 'Dialogflow:exec error'); + } + } + + async kill(cs) { + super.kill(cs); + if (this.ep.connected) { + this.logger.debug('TaskDialogFlow:kill'); + this.ep.removeCustomEventListener('dialogflow::intent'); + this.ep.removeCustomEventListener('dialogflow::transcription'); + this.ep.removeCustomEventListener('dialogflow::audio_provided'); + this.ep.removeCustomEventListener('dialogflow::end_of_utterance'); + this.ep.removeCustomEventListener('dialogflow::error'); + + await this.ep.api('uuid_break', this.ep.uuid).catch((err) => this.logger.info(err, 'Error killing audio')); + } + this.notifyTaskDone(); + } + + async init(cs, ep) { + this.ep = ep; + try { + this.ep.addCustomEventListener('dialogflow::intent', this._onIntent.bind(this, ep, cs)); + this.ep.addCustomEventListener('dialogflow::transcription', this._onTranscription.bind(this, ep, cs)); + this.ep.addCustomEventListener('dialogflow::audio_provided', this._onAudioProvided.bind(this, ep, cs)); + this.ep.addCustomEventListener('dialogflow::end_of_utterance', this._onEndOfUtterance.bind(this, ep, cs)); + this.ep.addCustomEventListener('dialogflow::error', this._onError.bind(this, ep, cs)); + + const creds = JSON.stringify(JSON.parse(this.credentials)); + await this.ep.set('GOOGLE_APPLICATION_CREDENTIALS', creds); + + } catch (err) { + this.logger.error({err}, 'Error setting credentials'); + throw err; + } + } + + /** + * An intent has been returned. Since we are using SINGLE_UTTERANCE on the dialogflow side, + * we may get an empty intent, signified by the lack of a 'response_id' attribute. + * In such a case, we just start another StreamingIntentDetectionRequest. + * @param {*} ep - media server endpoint + * @param {*} evt - event data + */ + _onIntent(ep, cs, evt) { + const intent = new Intent(this.logger, evt); + + if (intent.isEmpty) { + /** + * An empty intent is returned in 3 conditions: + * 1. Our no-input timer fired + * 2. We collected dtmf that needs to be fed to dialogflow + * 3. A normal dialogflow timeout + */ + if (this.noinput && this.greetingPlayed) { + this.logger.info('no input timer fired, reprompting..'); + this.noinput = false; + ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang} actions_intent_NO_INPUT`); + } + else if (this.dtmfEntry && this.greetingPlayed) { + this.logger.info('dtmf detected, reprompting..'); + ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang} none \'${this.dtmfEntry}\'`); + this.dtmfEntry = null; + } + else if (this.greetingPlayed) { + this.logger.info('starting another intent'); + ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`); + } + else { + this.logger.info('got empty intent'); + } + return; + } + + if (this.events.includes('intent')) { + this.performHook(this.eventHook, {event: 'intent', data: evt}); + } + + // clear the no-input timer and the digit buffer + this._clearNoinputTimer(); + if (this.digitBuffer) this.digitBuffer.flush(); + + /* hang up (or tranfer call) after playing next audio file? */ + if (intent.saysEndInteraction) { + // if 'end_interaction' is true, end the dialog after playing the final prompt + // (or in 1 second if there is no final prompt) + this.hangupAfterPlayDone = true; + this.waitingForPlayStart = true; + setTimeout(() => { + if (this.waitingForPlayStart) { + this.logger.info('hanging up since intent was marked end interaction'); + this.notifyTaskDone(); + } + }, 1000); + } + + /* collect digits? */ + else if (intent.saysCollectDtmf || this.enableDtmfAlways) { + const opts = Object.assign({ + idt: this.opts.interDigitTimeout + }, intent.dtmfInstructions || {term: '#'}); + this.digitBuffer = new DigitBuffer(this.logger, opts); + this.digitBuffer.once('fulfilled', this._onDtmfEntryComplete.bind(this, ep)); + } + } + + /** + * A transcription - either interim or final - has been returned. + * If we are doing barge-in based on hotword detection, check for the hotword or phrase. + * If we are playing a filler sound, like typing, during the fullfillment phase, start that + * if this is a final transcript. + * @param {*} ep - media server endpoint + * @param {*} evt - event data + */ + _onTranscription(ep, cs, evt) { + const transcription = new Transcription(this.logger, evt); + + if (this.events.includes('transcription') && transcription.isFinal) { + this.performHook(this.eventHook, {event: 'transcription', data: evt}); + } + else if (this.events.includes('interim-transcription') && !transcription.isFinal) { + this.performHook(this.eventHook, {event: 'transcription', data: evt}); + } + + // if a final transcription, start a typing sound + if (this.thinkingSound > 0 && !transcription.isEmpty && transcription.isFinal && + transcription.confidence > 0.8) { + ep.play(this.data.thinkingSound).catch((err) => this.logger.info(err, 'Error playing typing sound')); + } + } + + /** + * The caller has just finished speaking. No action currently taken. + * @param {*} evt - event data + */ + _onEndOfUtterance(cs, evt) { + if (this.events.includes('end-of-utterance')) { + this.performHook(this.eventHook, {event: 'end-of-utterance'}); + } + } + + /** + * Dialogflow has returned an error of some kind. + * @param {*} evt - event data + */ + _onError(evt) { + this.logger.error(`got error: ${JSON.stringify(evt)}`); + } + + /** + * Audio has been received from dialogflow and written to a temporary disk file. + * Start playing the audio, after killing any filler sound that might be playing. + * When the audio completes, start the no-input timer. + * @param {*} ep - media server endpoint + * @param {*} evt - event data + */ + async _onAudioProvided(ep, cs, evt) { + this.waitingForPlayStart = false; + + // kill filler audio + await ep.api('uuid_break', ep.uuid); + + // start a new intent, (we want to continue to listen during the audio playback) + // _unless_ we are transferring or ending the session + if (/*this.greetingPlayed &&*/ !this.hangupAfterPlayDone) { + ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`); + } + + this.playInProgress = true; + this.curentAudioFile = evt.path; + + this.logger.info(`starting to play ${evt.path}`); + if (this.events.includes('start-play')) { + this.performHook(this.eventHook, {event: 'start-play', data: {path: evt.path}}); + } + await ep.play(evt.path); + if (this.events.includes('end-play')) { + this.performHook(this.eventHook, {event: 'stop-play', data: {path: evt.path}}); + } + this.logger.info(`finished ${evt.path}`); + + if (this.curentAudioFile === evt.path) { + this.playInProgress = false; + } + /* + if (!this.inbound && !this.greetingPlayed) { + this.logger.info('finished greeting on outbound call, starting new intent'); + this.ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`); + } + */ + this.greetingPlayed = true; + + if (this.hangupAfterPlayDone) { + this.logger.info('hanging up since intent was marked end interaction and we completed final prompt'); + this.notifyTaskDone(); + } + else { + // every time we finish playing a prompt, start the no-input timer + this._startNoinputTimer(ep, cs); + } + } + + /** + * receive a dmtf entry from the caller. + * If we have active dtmf instructions, collect and process accordingly. + */ + _onDtmf(ep, cs, evt) { + if (this.digitBuffer) this.digitBuffer.process(evt.dtmf); + if (this.events.includes('dtmf')) { + this.performHook(this.eventHook, {event: 'dtmf', data: evt}); + } + } + + _onDtmfEntryComplete(ep, dtmfEntry) { + this.logger.info(`collected dtmf entry: ${dtmfEntry}`); + this.dtmfEntry = dtmfEntry; + this.digitBuffer = null; + // if a final transcription, start a typing sound + if (this.thinkingSound > 0) { + ep.play(this.thinkingSound).catch((err) => this.logger.info(err, 'Error playing typing sound')); + } + + // kill the current dialogflow, which will result in us getting an immediate intent + ep.api('dialogflow_stop', `${ep.uuid}`) + .catch((err) => this.logger.info(`dialogflow_stop failed: ${err.message}`)); + } + + /** + * The user has not provided any input for some time. + * Set the 'noinput' member to true and kill the current dialogflow. + * This will result in us re-prompting with an event indicating no input. + * @param {*} ep + */ + _onNoInput(ep, cs) { + this.noinput = true; + + if (this.events.includes('no-input')) { + this.performHook(this.eventHook, {event: 'no-input'}); + } + + // kill the current dialogflow, which will result in us getting an immediate intent + ep.api('dialogflow_stop', `${ep.uuid}`) + .catch((err) => this.logger.info(`dialogflow_stop failed: ${err.message}`)); + } + + /** + * Stop the no-input timer, if it is running + */ + _clearNoinputTimer() { + if (this.noinputTimer) { + clearTimeout(this.noinputTimer); + this.noinputTimer = null; + } + } + + /** + * Start the no-input timer. The duration is set in the configuration file. + * @param {*} ep + */ + _startNoinputTimer(ep, cs) { + if (!this.noInputTimeout) return; + this._clearNoinputTimer(); + this.noinputTimer = setTimeout(this._onNoInput.bind(this, ep, cs), this.noInputTimeout); + } + +} + +module.exports = Dialogflow; diff --git a/lib/tasks/dialogflow/intent.js b/lib/tasks/dialogflow/intent.js new file mode 100644 index 00000000..d09936bd --- /dev/null +++ b/lib/tasks/dialogflow/intent.js @@ -0,0 +1,89 @@ +class Intent { + constructor(logger, evt) { + this.logger = logger; + this.evt = evt; + + this.logger.debug({evt}, 'intent'); + this.dtmfRequest = checkIntentForDtmfEntry(logger, evt); + } + + get isEmpty() { + return this.evt.response_id.length === 0; + } + + get fulfillmentText() { + return this.evt.query_result.fulfillment_text; + } + + get saysEndInteraction() { + return this.evt.query_result.intent.end_interaction ; + } + + get saysCollectDtmf() { + return !!this.dtmfRequest; + } + + get dtmfInstructions() { + return this.dtmfRequest; + } + + get name() { + if (!this.isEmpty) return this.evt.query_result.intent.display_name; + } + + toJSON() { + return { + name: this.name, + fulfillmentText: this.fulfillmentText + }; + } + +} + +module.exports = Intent; + +/** + * Parse a returned intent for DTMF entry information + * i.e. + * allow-dtmf-x-y-z + * x = min number of digits + * y = optional, max number of digits + * z = optional, terminating character + * e.g. + * allow-dtmf-5 : collect 5 digits + * allow-dtmf-1-4 : collect between 1 to 4 (inclusive) digits + * allow-dtmf-1-4-# : collect 1-4 digits, terminating if '#' is entered + * @param {*} intent - dialogflow intent + */ +const checkIntentForDtmfEntry = (logger, intent) => { + const qr = intent.query_result; + if (!qr || !qr.fulfillment_messages || !qr.output_contexts) { + logger.info({f: qr.fulfillment_messages, o: qr.output_contexts}, 'no dtmfs'); + return; + } + + // check for custom payloads with a gather verb + const custom = qr.fulfillment_messages.find((f) => f.payload && f.payload.verb === 'gather'); + if (custom && custom.payload && custom.payload.verb === 'gather') { + logger.info({custom}, 'found dtmf custom payload'); + return { + max: custom.payload.numDigits, + term: custom.payload.finishOnKey, + template: custom.payload.responseTemplate + }; + } + + // check for an output context with a specific naming convention + const context = qr.output_contexts.find((oc) => oc.name.includes('/contexts/allow-dtmf-')); + if (context) { + const arr = /allow-dtmf-(\d+)(?:-(\d+))?(?:-(.*))?/.exec(context.name); + if (arr) { + logger.info({custom}, 'found dtmf output context'); + return { + min: parseInt(arr[1]), + max: arr.length > 2 ? parseInt(arr[2]) : null, + term: arr.length > 3 ? arr[3] : null + }; + } + } +}; diff --git a/lib/tasks/dialogflow/transcription.js b/lib/tasks/dialogflow/transcription.js new file mode 100644 index 00000000..02a0754b --- /dev/null +++ b/lib/tasks/dialogflow/transcription.js @@ -0,0 +1,41 @@ +class Transcription { + constructor(logger, evt) { + this.logger = logger; + + this.recognition_result = evt.recognition_result; + } + + get isEmpty() { + return !this.recognition_result; + } + + get isFinal() { + return this.recognition_result && this.recognition_result.is_final === true; + } + + get confidence() { + if (!this.isEmpty) return this.recognition_result.confidence; + } + + get text() { + if (!this.isEmpty) return this.recognition_result.transcript; + } + + startsWith(str) { + return (this.text.toLowerCase() || '').startsWith(str.toLowerCase()); + } + + includes(str) { + return (this.text.toLowerCase() || '').includes(str.toLowerCase()); + } + + toJSON() { + return { + final: this.recognition_result.is_final === true, + text: this.text, + confidence: this.confidence + }; + } +} + +module.exports = Transcription; diff --git a/lib/tasks/make_task.js b/lib/tasks/make_task.js index 586e64ea..85cb6274 100644 --- a/lib/tasks/make_task.js +++ b/lib/tasks/make_task.js @@ -24,6 +24,9 @@ function makeTask(logger, obj, parent) { case TaskName.Dial: const TaskDial = require('./dial'); return new TaskDial(logger, data, parent); + case TaskName.Dialogflow: + const TaskDialogflow = require('./dialogflow'); + return new TaskDialogflow(logger, data, parent); case TaskName.Dequeue: const TaskDequeue = require('./dequeue'); return new TaskDequeue(logger, data, parent); diff --git a/lib/tasks/specs.json b/lib/tasks/specs.json index 2c2c9990..7ca16d09 100644 --- a/lib/tasks/specs.json +++ b/lib/tasks/specs.json @@ -118,6 +118,26 @@ "target" ] }, + "dialogflow": { + "properties": { + "credentials": "string", + "project": "string", + "lang": "string", + "actionHook": "object|string", + "eventHook": "object|string", + "events": "[string]", + "welcomeEvent": "string", + "welcomeEventParams": "object", + "noInputTimeout": "number", + "passDtmfAsTextInput": "boolean", + "thinkingMusic": "string" + }, + "required": [ + "project", + "credentials", + "lang" + ] + }, "listen": { "properties": { "actionHook": "object|string", diff --git a/lib/tasks/task.js b/lib/tasks/task.js index e8d6bff9..ad8bec66 100644 --- a/lib/tasks/task.js +++ b/lib/tasks/task.js @@ -99,6 +99,18 @@ class Task extends Emitter { } } + async performHook(hook, results, expectResponse = true) { + const json = await this.cs.requestor.request(hook, results); + if (expectResponse && json && Array.isArray(json)) { + const makeTask = require('./make_task'); + const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata)); + if (tasks && tasks.length > 0) { + this.logger.info({tasks: tasks}, `${this.name} replacing application with ${tasks.length} tasks`); + this.callSession.replaceApplication(tasks); + } + } + } + async transferCallToFeatureServer(cs, sipAddress, opts) { const uuid = uuidv4(); const {addKey} = cs.srf.locals.dbHelpers; diff --git a/lib/utils/constants.json b/lib/utils/constants.json index d35cac92..50456251 100644 --- a/lib/utils/constants.json +++ b/lib/utils/constants.json @@ -3,6 +3,7 @@ "Conference": "conference", "Dequeue": "dequeue", "Dial": "dial", + "Dialogflow": "dialogflow", "Enqueue": "enqueue", "Gather": "gather", "Hangup": "hangup", diff --git a/package-lock.json b/package-lock.json index 97a52d14..6b9a342c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1118,9 +1118,9 @@ } }, "drachtio-srf": { - "version": "4.4.34", - "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-4.4.34.tgz", - "integrity": "sha512-hTaO45Hc9SOJlxucZxnNGtvoIXG1loeu0H8W4U3WU9va3BY2U0kK2f2sT8mvqURA3c7/jFRfSicvDniZYKWwyg==", + "version": "4.4.36", + "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-4.4.36.tgz", + "integrity": "sha512-8Tkj0RcPyGZb3wkgL3cWZhpqhCimLYwy8goFAYykNcEl7R4R14pQcMuCi/OhAvOQLBY1XCHXhJZhPyU3/A/8iQ==", "requires": { "async": "^1.4.2", "debug": "^3.1.0", diff --git a/package.json b/package.json index 01ab44be..3a7f2e2c 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "cidr-matcher": "^2.1.1", "debug": "^4.1.1", "drachtio-fsmrf": "^2.0.1", - "drachtio-srf": "^4.4.34", + "drachtio-srf": "^4.4.36", "express": "^4.17.1", "ip": "^1.1.5", "@jambonz/db-helpers": "^0.3.7",