wip

lib/tasks/dialogflow/index.js

@@ -1,3 +1,4 @@
+const assert = require('assert');
 const Task = require('../task');
 const {TaskName, TaskPreconditions} = require('../../utils/constants');
 const Intent = require('./intent');
@@ -10,19 +11,29 @@ class Dialogflow extends Task {
     super(logger, opts);
     this.preconditions = TaskPreconditions.Endpoint;
     this.credentials = this.data.credentials;
+    this.project = this.data.project;
+    this.agent = this.data.agent;
+    this.region = this.data.region || 'default';
+    this.model = this.data.model || 'es';
+    this.queryInput = this.data.queryInput || {};
 
-    /* set project id with environment and region (optionally) */
-    if (this.data.environment && this.data.region) {
-      this.project = `${this.data.project}:${this.data.environment}:${this.data.region}`;
-    }
-    else if (this.data.environment) {
-      this.project = `${this.data.project}:${this.data.environment}`;
-    }
-    else if (this.data.region) {
-      this.project = `${this.data.project}::${this.data.region}`;
+    assert(this.agent || !this.isCX, 'agent is required for dialogflow cx');
+    assert(this.credentials, 'dialogflow credentials are required');
+
+    if (this.isCX) {
+      this.environment = this.data.environment || 'default';
     }
     else {
-      this.project = this.data.project;
+      /* ES: set project id with environment and region (optionally) */
+      if (this.data.environment && this.data.region) {
+        this.project = `${this.data.project}:${this.data.environment}:${this.data.region}`;
+      }
+      else if (this.data.environment) {
+        this.project = `${this.data.project}:${this.data.environment}`;
+      }
+      else if (this.data.region) {
+        this.project = `${this.data.project}::${this.data.region}`;
+      }
     }
 
     this.lang = this.data.lang || 'en-US';
@@ -67,31 +78,23 @@ class Dialogflow extends Task {
       this.fallbackLabel = this.data.tts.fallbackLabel;
     }
     this.bargein = this.data.bargein;
+
+    this.cmd = this.model === 'cx' ? 'dialogflow_cx_start' : 'dialogflow_start';
+    this.cmdStop = this.model === 'cx' ? 'dialogflow_cx_stop' : 'dialogflow_stop';
   }
 
   get name() { return TaskName.Dialogflow; }
 
+  get isCX() { return this.model === 'cx'; }
+
+  get isES() { return !this.isCX; }
+
   async exec(cs, {ep}) {
     await super.exec(cs);
 
     try {
       await this.init(cs, ep);
-
-      this.logger.debug(`starting dialogflow bot ${this.project}`);
-
-      // kick it off
-      const baseArgs = `${this.ep.uuid} ${this.project} ${this.lang} ${this.welcomeEvent}`;
-      if (this.welcomeEventParams) {
-        this.ep.api('dialogflow_start', `${baseArgs} '${JSON.stringify(this.welcomeEventParams)}'`);
-      }
-      else if (this.welcomeEvent.length) {
-        this.ep.api('dialogflow_start', baseArgs);
-      }
-      else {
-        this.ep.api('dialogflow_start', `${this.ep.uuid} ${this.project} ${this.lang}`);
-      }
-      this.logger.debug(`started dialogflow bot ${this.project}`);
-
+      await this.startBot('default');
       await this.awaitTaskDone();
     } catch (err) {
       this.logger.error({err}, 'Dialogflow:exec error');
@@ -108,6 +111,12 @@ class Dialogflow extends Task {
       this.ep.removeCustomEventListener('dialogflow::end_of_utterance');
       this.ep.removeCustomEventListener('dialogflow::error');
 
+      this.ep.removeCustomEventListener('dialogflow_cx::intent');
+      this.ep.removeCustomEventListener('dialogflow_cx::transcription');
+      this.ep.removeCustomEventListener('dialogflow_cx::audio_provided');
+      this.ep.removeCustomEventListener('dialogflow_cx::end_of_utterance');
+      this.ep.removeCustomEventListener('dialogflow_cx::error');
+
       this._clearNoinputTimer();
 
       if (!this.reportedFinalAction) this.performAction({dialogflowResult: 'caller hungup'})
@@ -141,6 +150,12 @@ class Dialogflow extends Task {
       this.ep.addCustomEventListener('dialogflow::end_of_utterance', this._onEndOfUtterance.bind(this, ep, cs));
       this.ep.addCustomEventListener('dialogflow::error', this._onError.bind(this, ep, cs));
 
+      this.ep.addCustomEventListener('dialogflow_cx::intent', this._onIntent.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow_cx::transcription', this._onTranscription.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow_cx::audio_provided', this._onAudioProvided.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow_cx::end_of_utterance', this._onEndOfUtterance.bind(this, ep, cs));
+      this.ep.addCustomEventListener('dialogflow_cx::error', this._onError.bind(this, ep, cs));
+
       const obj = typeof this.credentials === 'string' ? JSON.parse(this.credentials) : this.credentials;
       const creds = JSON.stringify(obj);
       await this.ep.set('GOOGLE_APPLICATION_CREDENTIALS', creds);
@@ -151,6 +166,51 @@ class Dialogflow extends Task {
     }
   }
 
+  async startBot(intent) {
+    if (this.isCX) {
+      await this.startBotCX(intent);
+    }
+    else {
+      await this.startBotES(intent);
+    }
+  }
+
+  async startBotES() {
+    this.logger.info('starting dialogflow ES bot');
+    const baseArgs = `${this.ep.uuid} ${this.project} ${this.lang} ${this.welcomeEvent}`;
+    if (this.welcomeEventParams) {
+      await this.ep.api(this.cmd, `${baseArgs} '${JSON.stringify(this.welcomeEventParams)}'`);
+    }
+    else if (this.welcomeEvent.length) {
+      await this.ep.api(this.cmd, baseArgs);
+    }
+    else {
+      await this.ep.api(this.cmd, `${this.ep.uuid} ${this.project} ${this.lang}`);
+    }
+  }
+
+  async startBotCX(intent) {
+    const baseArgs = [
+      this.ep.uuid,
+      this.region,
+      this.project,
+      this.agent,
+      this.environment,
+      this.lang,
+    ];
+    if (intent) {
+      baseArgs.push(intent);
+    }
+    /*
+    if (Object.keys(this.queryInput).length > 0) {
+      baseArgs.push(`'${JSON.stringify(this.queryInput)}'`);
+    }
+      */
+    this.logger.info({args: baseArgs}, 'starting dialogflow CX bot');
+
+    await this.ep.api(this.cmd, `${baseArgs.join(' ')}`);
+  }
+
   /**
    * An intent has been returned.  Since we are using SINGLE_UTTERANCE on the dialogflow side,
    * we may get an empty intent, signified by the lack of a 'response_id' attribute.
@@ -171,20 +231,20 @@ class Dialogflow extends Task {
       if (this.noinput && this.greetingPlayed) {
         this.logger.info('no input timer fired, reprompting..');
         this.noinput = false;
-        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang} ${this.noInputEvent}`);
+        ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang} ${this.noInputEvent}`);
       }
       else if (this.dtmfEntry && this.greetingPlayed) {
         this.logger.info('dtmf detected, reprompting..');
-        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang} none \'${this.dtmfEntry}\'`);
+        ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang} none \'${this.dtmfEntry}\'`);
         this.dtmfEntry = null;
       }
       else if (this.greetingPlayed) {
         this.logger.info('starting another intent');
-        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
+        ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang}`);
       }
       else {
         this.logger.info('got empty intent');
-        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
+        ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang}`);
       }
       return;
     }
@@ -231,7 +291,7 @@ class Dialogflow extends Task {
       // start a new intent, (we want to continue to listen during the audio playback)
       // _unless_ we are transferring or ending the session
       if (!this.hangupAfterPlayDone) {
-        ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
+        ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang}`);
       }
 
       try {
@@ -386,10 +446,12 @@ class Dialogflow extends Task {
     // kill filler audio
     await ep.api('uuid_break', ep.uuid);
 
-    // start a new intent, (we want to continue to listen during the audio playback)
+    // if ES start a new intent (for CX we do not set single_utterance on),
+    // (we want to continue to listen during the audio playback)
     // _unless_ we are transferring or ending the session
-    if (/*this.greetingPlayed &&*/ !this.hangupAfterPlayDone) {
-      ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
+    if (!this.hangupAfterPlayDone) {
+      this.startBot();
+      //ep.api(this.cmd, `${ep.uuid} ${this.project} ${this.lang}`);
     }
 
     this.playInProgress = true;
@@ -414,12 +476,7 @@ class Dialogflow extends Task {
         return;
       }
     }
-    /*
-    if (!this.inbound && !this.greetingPlayed) {
-      this.logger.info('finished greeting on outbound call, starting new intent');
-      this.ep.api('dialogflow_start', `${ep.uuid} ${this.project} ${this.lang}`);
-    }
-    */
+
     this.greetingPlayed = true;
 
     if (this.hangupAfterPlayDone) {
@@ -454,7 +511,7 @@ class Dialogflow extends Task {
     }
 
     // kill the current dialogflow, which will result in us getting an immediate intent
-    ep.api('dialogflow_stop', `${ep.uuid}`)
+    ep.api(this.cmdStop, `${ep.uuid}`)
       .catch((err) => this.logger.info(`dialogflow_stop failed: ${err.message}`));
   }
 
@@ -472,7 +529,7 @@ class Dialogflow extends Task {
     }
 
     // kill the current dialogflow, which will result in us getting an immediate intent
-    ep.api('dialogflow_stop', `${ep.uuid}`)
+    ep.api(this.cmdStop, `${ep.uuid}`)
       .catch((err) => this.logger.info(`dialogflow_stop failed: ${err.message}`));
   }
 

lib/tasks/dialogflow/intent.js

@@ -4,19 +4,31 @@ class Intent {
     this.evt = evt;
 
     this.logger.debug({evt}, 'intent');
-    this.dtmfRequest = checkIntentForDtmfEntry(logger, evt);
+    this.qr = this.isCX ? evt.detect_intent_response.query_result : evt.query_result;
+    this.dtmfRequest = this._checkIntentForDtmfEntry(logger, evt);
+  }
+
+  get response_id() {
+    return this.isCX ? this.evt.detect_intent_response.response_id : this.evt.response_id;
   }
 
   get isEmpty() {
-    return this.evt.response_id.length === 0;
+    return !(this.response_id?.length > 0);
   }
 
   get fulfillmentText() {
-    return this.evt.query_result.fulfillment_text;
+    return this.qr.fulfillment_text;
   }
 
   get saysEndInteraction() {
-    return this.evt.query_result.intent.end_interaction ;
+    if (this.isCX) {
+      const end_interaction = this.qr.response_messages
+        .find((m) => typeof m === 'object' && 'end_interaction' in m)?.end_interaction;
+
+      //TODO: need to do more checking on the actual contents
+      return end_interaction && Object.keys(end_interaction).length > 0;
+    }
+    else return this.qr.intent.end_interaction ;
   }
 
   get saysCollectDtmf() {
@@ -28,7 +40,22 @@ class Intent {
   }
 
   get name() {
-    if (!this.isEmpty) return this.evt.query_result.intent.display_name;
+    if (!this.isEmpty) {
+      if (this.isCX) {
+        return this.qr.match?.intent?.display_name;
+      }
+      else {
+        return this.qr.intent.display_name;
+      }
+    }
+  }
+
+  get isCX() {
+    return typeof this.evt.detect_intent_response === 'object';
+  }
+
+  get isES() {
+    return !this.isCX;
   }
 
   toJSON() {
@@ -38,11 +65,7 @@ class Intent {
     };
   }
 
-}
-
-module.exports = Intent;
-
-/**
+  /**
  * Parse a returned intent for DTMF entry information
  * i.e.
  * allow-dtmf-x-y-z
@@ -55,35 +78,39 @@ module.exports = Intent;
  * allow-dtmf-1-4-# : collect 1-4 digits, terminating if '#' is entered
  * @param {*} intent - dialogflow intent
  */
-const checkIntentForDtmfEntry = (logger, intent) => {
-  const qr = intent.query_result;
-  if (!qr || !qr.fulfillment_messages || !qr.output_contexts) {
-    logger.info({f: qr.fulfillment_messages, o: qr.output_contexts}, 'no dtmfs');
-    return;
-  }
+  _checkIntentForDtmfEntry(logger, intent) {
+    const qr = this.isCX ? intent.detect_intent_response.query_result : intent.query_result;
 
-  // check for custom payloads with a gather verb
-  const custom = qr.fulfillment_messages.find((f) => f.payload && f.payload.verb === 'gather');
-  if (custom && custom.payload && custom.payload.verb === 'gather') {
-    logger.info({custom}, 'found dtmf custom payload');
-    return {
-      max: custom.payload.numDigits,
-      term: custom.payload.finishOnKey,
-      template: custom.payload.responseTemplate
-    };
-  }
+    if (!qr || !qr.fulfillment_messages || !qr.output_contexts) {
+      logger.info({f: qr.fulfillment_messages, o: qr.output_contexts}, 'no dtmfs');
+      return;
+    }
 
-  // check for an output context with a specific naming convention
-  const context = qr.output_contexts.find((oc) => oc.name.includes('/contexts/allow-dtmf-'));
-  if (context) {
-    const arr = /allow-dtmf-(\d+)(?:-(\d+))?(?:-(.*))?/.exec(context.name);
-    if (arr) {
-      logger.info({custom}, 'found dtmf output context');
+    // check for custom payloads with a gather verb
+    const custom = qr.fulfillment_messages.find((f) => f.payload && f.payload.verb === 'gather');
+    if (custom && custom.payload && custom.payload.verb === 'gather') {
+      logger.info({custom}, 'found dtmf custom payload');
       return {
-        min: parseInt(arr[1]),
-        max: arr.length > 2 ? parseInt(arr[2]) : null,
-        term: arr.length > 3 ? arr[3] : null
+        max: custom.payload.numDigits,
+        term: custom.payload.finishOnKey,
+        template: custom.payload.responseTemplate
       };
     }
+
+    // check for an output context with a specific naming convention
+    const context = qr.output_contexts.find((oc) => oc.name.includes('/contexts/allow-dtmf-'));
+    if (context) {
+      const arr = /allow-dtmf-(\d+)(?:-(\d+))?(?:-(.*))?/.exec(context.name);
+      if (arr) {
+        logger.info({custom}, 'found dtmf output context');
+        return {
+          min: parseInt(arr[1]),
+          max: arr.length > 2 ? parseInt(arr[2]) : null,
+          term: arr.length > 3 ? arr[3] : null
+        };
+      }
+    }
   }
-};
+}
+
+module.exports = Intent;

package-lock.json

@@ -18,7 +18,7 @@
         "@jambonz/speech-utils": "^0.1.15",
         "@jambonz/stats-collector": "^0.1.10",
         "@jambonz/time-series": "^0.2.9",
-        "@jambonz/verb-specifications": "^0.0.76",
+        "@jambonz/verb-specifications": "^0.0.79",
         "@opentelemetry/api": "^1.8.0",
         "@opentelemetry/exporter-jaeger": "^1.23.0",
         "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
@@ -1575,9 +1575,9 @@
       }
     },
     "node_modules/@jambonz/verb-specifications": {
-      "version": "0.0.76",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.76.tgz",
-      "integrity": "sha512-7s61qAsG07xLLaEAHW236rSYzEoh9Qg0aRWHPbTfxCsuTKDNeq+5EwGAShDU5R5ZpjgweZJLhArQm8Ym+4xJ2A==",
+      "version": "0.0.79",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.79.tgz",
+      "integrity": "sha512-SJpUfRivPaBBF16sUhkKPuXC4KFf2vE03LuSNYGhtjzZ03PnIGXbsuz16cK+XeQow5tkof+ptmxwFgfv6TM5RQ==",
       "dependencies": {
         "debug": "^4.3.4",
         "pino": "^8.8.0"
@@ -10539,9 +10539,9 @@
       }
     },
     "@jambonz/verb-specifications": {
-      "version": "0.0.76",
-      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.76.tgz",
-      "integrity": "sha512-7s61qAsG07xLLaEAHW236rSYzEoh9Qg0aRWHPbTfxCsuTKDNeq+5EwGAShDU5R5ZpjgweZJLhArQm8Ym+4xJ2A==",
+      "version": "0.0.79",
+      "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.79.tgz",
+      "integrity": "sha512-SJpUfRivPaBBF16sUhkKPuXC4KFf2vE03LuSNYGhtjzZ03PnIGXbsuz16cK+XeQow5tkof+ptmxwFgfv6TM5RQ==",
       "requires": {
         "debug": "^4.3.4",
         "pino": "^8.8.0"

package.json

@@ -34,7 +34,7 @@
     "@jambonz/speech-utils": "^0.1.15",
     "@jambonz/stats-collector": "^0.1.10",
     "@jambonz/time-series": "^0.2.9",
-    "@jambonz/verb-specifications": "^0.0.76",
+    "@jambonz/verb-specifications": "^0.0.79",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/exporter-jaeger": "^1.23.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",

test/docker-compose-testbed.yaml

@@ -57,7 +57,7 @@ services:
         condition: service_healthy
 
   freeswitch:
-    image: drachtio/drachtio-freeswitch-mrf:0.7.3
+    image: drachtio/drachtio-freeswitch-mrf:latest
     restart: always
     command: freeswitch --rtp-range-start 20000 --rtp-range-end 20100
     environment: