added initial support for dialogflow

lib/session/call-session.js

@@ -462,11 +462,11 @@ class CallSession extends Emitter {
    * @param {Task} task - task to be executed
    */
   async _evalEndpointPrecondition(task) {
-    this.logger.debug('_evalEndpointPrecondition');
+    this.logger.debug('CallSession:_evalEndpointPrecondition');
     if (this.callGone) new Error(`${BADPRECONDITIONS}: call gone`);
 
     if (this.ep) {
-      if (!task.earlyMedia || this.dlg) return this.ep;
+      if (task.earlyMedia === true || this.dlg) return this.ep;
 
       // we are going from an early media connection to answer
       await this.propagateAnswer();

lib/tasks/dialogflow/digit-buffer.js

@@ -0,0 +1,70 @@
+const Emitter = require('events');
+
+/**
+ * A dtmf collector
+ * @class
+ */
+class DigitBuffer extends Emitter {
+  /**
+   * Creates a DigitBuffer
+   * @param {*} logger - a pino logger
+   * @param {*} opts - dtmf collection instructions
+   */
+  constructor(logger, opts) {
+    super();
+    this.logger = logger;
+    this.minDigits = opts.min || 1;
+    this.maxDigits = opts.max || 99;
+    this.termDigit = opts.term;
+    this.interdigitTimeout = opts.idt || 8000;
+    this.template = opts.template;
+    this.buffer = '';
+    this.logger.debug(`digitbuffer min: ${this.minDigits} max: ${this.maxDigits} term digit: ${this.termDigit}`);
+  }
+
+  /**
+   * process a received dtmf digit
+   * @param {String} a single digit entered by the caller
+   */
+  process(digit) {
+    this.logger.debug(`digitbuffer process: ${digit}`);
+    if (digit === this.termDigit) return this._fulfill();
+    this.buffer += digit;
+    if (this.buffer.length === this.maxDigits) return this._fulfill();
+    if (this.buffer.length >= this.minDigits) this._startInterDigitTimer();
+    this.logger.debug(`digitbuffer buffer: ${this.buffer}`);
+  }
+
+  /**
+   * clear the digit buffer
+   */
+  flush() {
+    if (this.idtimer) clearTimeout(this.idtimer);
+    this.buffer = '';
+  }
+
+  _fulfill() {
+    this.logger.debug(`digit buffer fulfilled with ${this.buffer}`);
+    if (this.template && this.template.includes('${digits}')) {
+      const text = this.template.replace('${digits}', this.buffer);
+      this.logger.info(`reporting dtmf as ${text}`);
+      this.emit('fulfilled', text);
+    }
+    else {
+      this.emit('fulfilled', this.buffer);
+    }
+    this.flush();
+  }
+
+  _startInterDigitTimer() {
+    if (this.idtimer) clearTimeout(this.idtimer);
+    this.idtimer = setTimeout(this._onInterDigitTimeout.bind(this), this.interdigitTimeout);
+  }
+
+  _onInterDigitTimeout() {
+    this.logger.debug('digit buffer timeout');
+    this._fulfill();
+  }
+}
+
+module.exports = DigitBuffer;

lib/tasks/dialogflow/index.js

@@ -0,0 +1,337 @@
+const Task = require('../task');
+const {TaskName, TaskPreconditions} = require('../../utils/constants');
+const Intent = require('./intent');
+const DigitBuffer = require('./digit-buffer');
+const Transcription = require('./transcription');
+
+class Dialogflow extends Task {
+  constructor(logger, opts) {
+    super(logger, opts);
+    this.preconditions = TaskPreconditions.Endpoint;
+
+    this.credentials = this.data.credentials;
+    this.project = this.data.project;
+    this.lang = this.data.lang || 'en-US';
+    this.welcomeEvent = this.data.welcomeEvent || '';
+    if (this.welcomeEvent.length && this.data.welcomeEventParams && typeof this.data.welcomeEventParams === 'object') {
+      this.welcomeEventParams = this.data.welcomeEventParams;
+    }
+    this.noInputTimeout = this.data.noInputTimeout ;
+    this.passDtmfAsInputText = this.passDtmfAsInputText === true;
+    if (this.data.eventHook) this.eventHook = this.data.eventHook;
+    if (this.eventHook && Array.isArray(this.data.events)) {
+      this.events = this.data.events;
+    }
+    else if (this.eventHook) {
+      // send all events by default
+      this.events = [
+        'intent',
+        'transcription',
+        'dtmf',
+        'end-utterance',
+        'start-play',
+        'end-play',
+        'no-input'
+      ];
+    }
+    else {
+      this.events = [];
+    }
+    if (this.data.actionHook) this.actionHook = this.data.actionHook;
+    if (this.data.thinkingMusic) this.thinkingMusic = this.data.thinkingMusic;
+  }
+
+  get name() { return TaskName.Dialogflow; }
+
+  async exec(cs, ep) {
+    await super.exec(cs);
+
+    try {
+      await this.init(cs, ep);
+
+      this.logger.debug(`starting dialogflow bot ${this.project}`);
+
+      // kick it off
+      if (this.welcomeEventParams) {
+        this.ep.api('dialogflow_start',
+          `${this.ep.uuid} ${this.project} ${this.lang} ${this.welcomeEvent} '${JSON.stringify(this.welcomeEventParams)}'`);
+      }
+      else if (this.welcomeEvent.length) {
+        this.ep.api('dialogflow_start',
+          `${this.ep.uuid} ${this.project} ${this.lang} ${this.welcomeEvent}`);
+      }
+      else {
+        this.ep.api('dialogflow_start', `${this.ep.uuid} ${this.project} ${this.lang}`);
+      }
+      this.logger.debug(`started dialogflow bot ${this.project}`);
+
+      await this.awaitTaskDone();
+    } catch (err) {
+      this.logger.error({err}, 'Dialogflow:exec error');
+    }
+  }
+
+  async kill(cs) {
+    super.kill(cs);
+    if (this.ep.connected) {
+      this.logger.debug('TaskDialogFlow:kill');
+      this.ep.removeCustomEventListener('dialogflow::intent');
+      this.ep.removeCustomEventListener('dialogflow::transcription');
+      this.ep.removeCustomEventListener('dialogflow::audio_provided');
+      this.ep.removeCustomEventListener('dialogflow::end_of_utterance');
+      this.ep.removeCustomEventListener('dialogflow::error');
+
+      await this.ep.api('uuid_break', this.ep.uuid).catch((err) => this.logger.info(err, 'Error killing audio'));
+    }
+    this.notifyTaskDone();
+  }
+
+  async init(cs, ep) {
+    this.ep = ep;
+    try {
+      this.ep.addCustomEventListener('dialogflow::intent', this._onIntent.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow::transcription', this._onTranscription.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow::audio_provided', this._onAudioProvided.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow::end_of_utterance', this._onEndOfUtterance.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow::error', this._onError.bind(this, ep, cs));
+
+      const creds = JSON.stringify(JSON.parse(this.credentials));
+      await this.ep.set('GOOGLE_APPLICATION_CREDENTIALS', creds);
+
+    } catch (err) {
+      this.logger.error({err}, 'Error setting credentials');
+      throw err;
+    }
+  }
+
+  /**
+   * An intent has been returned.  Since we are using SINGLE_UTTERANCE on the dialogflow side,
+   * we may get an empty intent, signified by the lack of a 'response_id' attribute.
+   * In such a case, we just start another StreamingIntentDetectionRequest.
+   * @param {*} ep -  media server endpoint
+   * @param {*} evt - event data
+   */
+  _onIntent(ep, cs, evt) {
+    const intent = new Intent(this.logger, evt);
+
+    if (intent.isEmpty) {
+      /**
+       * An empty intent is returned in 3 conditions:
+       * 1. Our no-input timer fired
+       * 2. We collected dtmf that needs to be fed to dialogflow
+       * 3. A normal dialogflow timeout
+       */
+      if (this.noinput && this.greetingPlayed) {
+        this.logger.info('no input timer fired, reprompting..');
+        this.noinput = false;
+        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang} actions_intent_NO_INPUT`);
+      }
+      else if (this.dtmfEntry && this.greetingPlayed) {
+        this.logger.info('dtmf detected, reprompting..');
+        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang} none \'${this.dtmfEntry}\'`);
+        this.dtmfEntry = null;
+      }
+      else if (this.greetingPlayed) {
+        this.logger.info('starting another intent');
+        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
+      }
+      else {
+        this.logger.info('got empty intent');
+      }
+      return;
+    }
+
+    if (this.events.includes('intent')) {
+      this.performHook(this.eventHook, {event: 'intent', data: evt});
+    }
+
+    // clear the no-input timer and the digit buffer
+    this._clearNoinputTimer();
+    if (this.digitBuffer) this.digitBuffer.flush();
+
+    /* hang up (or tranfer call) after playing next audio file? */
+    if (intent.saysEndInteraction) {
+      //  if 'end_interaction' is true, end the dialog after playing the final prompt
+      //  (or in 1 second if there is no final prompt)
+      this.hangupAfterPlayDone = true;
+      this.waitingForPlayStart = true;
+      setTimeout(() => {
+        if (this.waitingForPlayStart) {
+          this.logger.info('hanging up since intent was marked end interaction');
+          this.notifyTaskDone();
+        }
+      }, 1000);
+    }
+
+    /* collect digits? */
+    else if (intent.saysCollectDtmf || this.enableDtmfAlways) {
+      const opts = Object.assign({
+        idt: this.opts.interDigitTimeout
+      }, intent.dtmfInstructions || {term: '#'});
+      this.digitBuffer = new DigitBuffer(this.logger, opts);
+      this.digitBuffer.once('fulfilled', this._onDtmfEntryComplete.bind(this, ep));
+    }
+  }
+
+  /**
+   * A transcription - either interim or final - has been returned.
+   * If we are doing barge-in based on hotword detection, check for the hotword or phrase.
+   * If we are playing a filler sound, like typing, during the fullfillment phase, start that
+   * if this is a final transcript.
+   * @param {*} ep  -  media server endpoint
+   * @param {*} evt - event data
+   */
+  _onTranscription(ep, cs, evt) {
+    const transcription = new Transcription(this.logger, evt);
+
+    if (this.events.includes('transcription') && transcription.isFinal) {
+      this.performHook(this.eventHook, {event: 'transcription', data: evt});
+    }
+    else if (this.events.includes('interim-transcription') && !transcription.isFinal) {
+      this.performHook(this.eventHook, {event: 'transcription', data: evt});
+    }
+
+    // if a final transcription, start a typing sound
+    if (this.thinkingSound > 0 && !transcription.isEmpty && transcription.isFinal &&
+      transcription.confidence > 0.8) {
+      ep.play(this.data.thinkingSound).catch((err) => this.logger.info(err, 'Error playing typing sound'));
+    }
+  }
+
+  /**
+   * The caller has just finished speaking.  No action currently taken.
+   * @param {*} evt - event data
+   */
+  _onEndOfUtterance(cs, evt) {
+    if (this.events.includes('end-of-utterance')) {
+      this.performHook(this.eventHook, {event: 'end-of-utterance'});
+    }
+  }
+
+  /**
+   * Dialogflow has returned an error of some kind.
+   * @param {*} evt - event data
+   */
+  _onError(evt) {
+    this.logger.error(`got error: ${JSON.stringify(evt)}`);
+  }
+
+  /**
+   * Audio has been received from dialogflow and written to a temporary disk file.
+   * Start playing the audio, after killing any filler sound that might be playing.
+   * When the audio completes, start the no-input timer.
+   * @param {*} ep -  media server endpoint
+   * @param {*} evt - event data
+   */
+  async _onAudioProvided(ep, cs, evt) {
+    this.waitingForPlayStart = false;
+
+    // kill filler audio
+    await ep.api('uuid_break', ep.uuid);
+
+    // start a new intent, (we want to continue to listen during the audio playback)
+    // _unless_ we are transferring or ending the session
+    if (/*this.greetingPlayed &&*/ !this.hangupAfterPlayDone) {
+      ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
+    }
+
+    this.playInProgress = true;
+    this.curentAudioFile = evt.path;
+
+    this.logger.info(`starting to play ${evt.path}`);
+    if (this.events.includes('start-play')) {
+      this.performHook(this.eventHook, {event: 'start-play', data: {path: evt.path}});
+    }
+    await ep.play(evt.path);
+    if (this.events.includes('end-play')) {
+      this.performHook(this.eventHook, {event: 'stop-play', data: {path: evt.path}});
+    }
+    this.logger.info(`finished ${evt.path}`);
+
+    if (this.curentAudioFile === evt.path) {
+      this.playInProgress = false;
+    }
+    /*
+    if (!this.inbound && !this.greetingPlayed) {
+      this.logger.info('finished greeting on outbound call, starting new intent');
+      this.ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
+    }
+    */
+    this.greetingPlayed = true;
+
+    if (this.hangupAfterPlayDone) {
+      this.logger.info('hanging up since intent was marked end interaction and we completed final prompt');
+      this.notifyTaskDone();
+    }
+    else {
+      // every time we finish playing a prompt, start the no-input timer
+      this._startNoinputTimer(ep, cs);
+    }
+  }
+
+  /**
+   * receive a dmtf entry from the caller.
+   * If we have active dtmf instructions, collect and process accordingly.
+   */
+  _onDtmf(ep, cs, evt) {
+    if (this.digitBuffer) this.digitBuffer.process(evt.dtmf);
+    if (this.events.includes('dtmf')) {
+      this.performHook(this.eventHook, {event: 'dtmf', data: evt});
+    }
+  }
+
+  _onDtmfEntryComplete(ep, dtmfEntry) {
+    this.logger.info(`collected dtmf entry: ${dtmfEntry}`);
+    this.dtmfEntry = dtmfEntry;
+    this.digitBuffer = null;
+    // if a final transcription, start a typing sound
+    if (this.thinkingSound > 0) {
+      ep.play(this.thinkingSound).catch((err) => this.logger.info(err, 'Error playing typing sound'));
+    }
+
+    // kill the current dialogflow, which will result in us getting an immediate intent
+    ep.api('dialogflow_stop', `${ep.uuid}`)
+      .catch((err) => this.logger.info(`dialogflow_stop failed: ${err.message}`));
+  }
+
+  /**
+   * The user has not provided any input for some time.
+   * Set the 'noinput' member to true and kill the current dialogflow.
+   * This will result in us re-prompting with an event indicating no input.
+   * @param {*} ep
+   */
+  _onNoInput(ep, cs) {
+    this.noinput = true;
+
+    if (this.events.includes('no-input')) {
+      this.performHook(this.eventHook,  {event: 'no-input'});
+    }
+
+    // kill the current dialogflow, which will result in us getting an immediate intent
+    ep.api('dialogflow_stop', `${ep.uuid}`)
+      .catch((err) => this.logger.info(`dialogflow_stop failed: ${err.message}`));
+  }
+
+  /**
+   * Stop the no-input timer, if it is running
+   */
+  _clearNoinputTimer() {
+    if (this.noinputTimer) {
+      clearTimeout(this.noinputTimer);
+      this.noinputTimer = null;
+    }
+  }
+
+  /**
+   * Start the no-input timer.  The duration is set in the configuration file.
+   * @param {*} ep
+   */
+  _startNoinputTimer(ep, cs) {
+    if (!this.noInputTimeout) return;
+    this._clearNoinputTimer();
+    this.noinputTimer = setTimeout(this._onNoInput.bind(this, ep, cs), this.noInputTimeout);
+  }
+
+}
+
+module.exports = Dialogflow;

lib/tasks/dialogflow/intent.js

@@ -0,0 +1,89 @@
+class Intent {
+  constructor(logger, evt) {
+    this.logger = logger;
+    this.evt = evt;
+
+    this.logger.debug({evt}, 'intent');
+    this.dtmfRequest = checkIntentForDtmfEntry(logger, evt);
+  }
+
+  get isEmpty() {
+    return this.evt.response_id.length === 0;
+  }
+
+  get fulfillmentText() {
+    return this.evt.query_result.fulfillment_text;
+  }
+
+  get saysEndInteraction() {
+    return this.evt.query_result.intent.end_interaction ;
+  }
+
+  get saysCollectDtmf() {
+    return !!this.dtmfRequest;
+  }
+
+  get dtmfInstructions() {
+    return this.dtmfRequest;
+  }
+
+  get name() {
+    if (!this.isEmpty) return this.evt.query_result.intent.display_name;
+  }
+
+  toJSON() {
+    return {
+      name: this.name,
+      fulfillmentText: this.fulfillmentText
+    };
+  }
+
+}
+
+module.exports = Intent;
+
+/**
+ * Parse a returned intent for DTMF entry information
+ * i.e.
+ * allow-dtmf-x-y-z
+ * x = min number of digits
+ * y = optional, max number of digits
+ * z = optional, terminating character
+ * e.g.
+ * allow-dtmf-5 :     collect 5 digits
+ * allow-dtmf-1-4 :   collect between 1 to 4 (inclusive) digits
+ * allow-dtmf-1-4-# : collect 1-4 digits, terminating if '#' is entered
+ * @param {*} intent - dialogflow intent
+ */
+const checkIntentForDtmfEntry = (logger, intent) => {
+  const qr = intent.query_result;
+  if (!qr || !qr.fulfillment_messages || !qr.output_contexts) {
+    logger.info({f: qr.fulfillment_messages, o: qr.output_contexts}, 'no dtmfs');
+    return;
+  }
+
+  // check for custom payloads with a gather verb
+  const custom = qr.fulfillment_messages.find((f) => f.payload && f.payload.verb === 'gather');
+  if (custom && custom.payload && custom.payload.verb === 'gather') {
+    logger.info({custom}, 'found dtmf custom payload');
+    return {
+      max: custom.payload.numDigits,
+      term: custom.payload.finishOnKey,
+      template: custom.payload.responseTemplate
+    };
+  }
+
+  // check for an output context with a specific naming convention
+  const context = qr.output_contexts.find((oc) => oc.name.includes('/contexts/allow-dtmf-'));
+  if (context) {
+    const arr = /allow-dtmf-(\d+)(?:-(\d+))?(?:-(.*))?/.exec(context.name);
+    if (arr) {
+      logger.info({custom}, 'found dtmf output context');
+      return {
+        min: parseInt(arr[1]),
+        max: arr.length > 2 ? parseInt(arr[2]) : null,
+        term: arr.length > 3 ? arr[3] : null
+      };
+    }
+  }
+};

lib/tasks/dialogflow/transcription.js

@@ -0,0 +1,41 @@
+class Transcription {
+  constructor(logger, evt) {
+    this.logger = logger;
+
+    this.recognition_result = evt.recognition_result;
+  }
+
+  get isEmpty() {
+    return !this.recognition_result;
+  }
+
+  get isFinal() {
+    return this.recognition_result && this.recognition_result.is_final === true;
+  }
+
+  get confidence() {
+    if (!this.isEmpty) return this.recognition_result.confidence;
+  }
+
+  get text() {
+    if (!this.isEmpty) return this.recognition_result.transcript;
+  }
+
+  startsWith(str) {
+    return (this.text.toLowerCase() || '').startsWith(str.toLowerCase());
+  }
+
+  includes(str) {
+    return (this.text.toLowerCase() || '').includes(str.toLowerCase());
+  }
+
+  toJSON() {
+    return {
+      final: this.recognition_result.is_final === true,
+      text: this.text,
+      confidence: this.confidence
+    };
+  }
+}
+
+module.exports = Transcription;

lib/tasks/make_task.js

@@ -24,6 +24,9 @@ function makeTask(logger, obj, parent) {
     case TaskName.Dial:
       const TaskDial = require('./dial');
       return new TaskDial(logger, data, parent);
+    case TaskName.Dialogflow:
+      const TaskDialogflow = require('./dialogflow');
+      return new TaskDialogflow(logger, data, parent);
     case TaskName.Dequeue:
       const TaskDequeue = require('./dequeue');
       return new TaskDequeue(logger, data, parent);

lib/tasks/specs.json

@@ -118,6 +118,26 @@
       "target"
     ]
   },
+  "dialogflow": {
+    "properties": {
+      "credentials": "string",
+      "project": "string",
+      "lang": "string",
+      "actionHook": "object|string",
+      "eventHook": "object|string",
+      "events": "[string]",
+      "welcomeEvent": "string",
+      "welcomeEventParams": "object",
+      "noInputTimeout": "number",
+      "passDtmfAsTextInput": "boolean",
+      "thinkingMusic": "string"
+    },
+    "required": [
+      "project",
+      "credentials",
+      "lang"
+    ]
+  },
   "listen": {
     "properties": {
       "actionHook": "object|string",

lib/tasks/task.js

@@ -99,6 +99,18 @@ class Task extends Emitter {
     }
   }
 
+  async performHook(hook, results, expectResponse = true) {
+    const json = await this.cs.requestor.request(hook, results);
+    if (expectResponse && json && Array.isArray(json)) {
+      const makeTask = require('./make_task');
+      const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
+      if (tasks && tasks.length > 0) {
+        this.logger.info({tasks: tasks}, `${this.name} replacing application with ${tasks.length} tasks`);
+        this.callSession.replaceApplication(tasks);
+      }
+    }
+  }
+
   async transferCallToFeatureServer(cs, sipAddress, opts) {
     const uuid = uuidv4();
     const {addKey} = cs.srf.locals.dbHelpers;

lib/utils/constants.json

@@ -3,6 +3,7 @@
     "Conference": "conference",
     "Dequeue": "dequeue",
     "Dial": "dial",
+    "Dialogflow": "dialogflow",
     "Enqueue": "enqueue",
     "Gather": "gather",
     "Hangup": "hangup",

package-lock.json

@@ -1118,9 +1118,9 @@
       }
     },
     "drachtio-srf": {
-      "version": "4.4.34",
-      "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-4.4.34.tgz",
-      "integrity": "sha512-hTaO45Hc9SOJlxucZxnNGtvoIXG1loeu0H8W4U3WU9va3BY2U0kK2f2sT8mvqURA3c7/jFRfSicvDniZYKWwyg==",
+      "version": "4.4.36",
+      "resolved": "https://registry.npmjs.org/drachtio-srf/-/drachtio-srf-4.4.36.tgz",
+      "integrity": "sha512-8Tkj0RcPyGZb3wkgL3cWZhpqhCimLYwy8goFAYykNcEl7R4R14pQcMuCi/OhAvOQLBY1XCHXhJZhPyU3/A/8iQ==",
       "requires": {
         "async": "^1.4.2",
         "debug": "^3.1.0",

package.json

@@ -30,7 +30,7 @@
     "cidr-matcher": "^2.1.1",
     "debug": "^4.1.1",
     "drachtio-fsmrf": "^2.0.1",
-    "drachtio-srf": "^4.4.34",
+    "drachtio-srf": "^4.4.36",
     "express": "^4.17.1",
     "ip": "^1.1.5",
     "@jambonz/db-helpers": "^0.3.7",