reanchor media for A leg

* wip, create outbound dial from webhook ws

* wip, create outbound dial from webhook ws

* clean

README.md

@@ -1,4 +1,4 @@
-# jambonz-feature-server ![Build Status](https://github.com/jambonz/jambonz-feature-server/workflows/CI/badge.svg)
+# jambonz-feature-server [![CI](https://github.com/jambonz/jambonz-feature-server/actions/workflows/build.yml/badge.svg)](https://github.com/jambonz/jambonz-feature-server/actions/workflows/build.yml)
 
 This application implements the core feature server of the jambones platform.
 
@@ -40,6 +40,8 @@ Configuration is provided via environment variables:
 |JAMBONZ_RECORD_WS_BASE_URL| recording websocket URL to send the recording audio|no|
 |JAMBONZ_RECORD_WS_USERNAME| recording websocket username|no|
 |JAMBONZ_RECORD_WS_PASSWORD| recording websocket password|no|
+|ANCHOR_MEDIA_ALWAYS| keep media on media server|no|
+|JAMBONZ_DISABLE_DIAL_PAI_HEADER| control P-Asserted-Identity header on B-Leg|no|
 
 ### running under pm2
 Typically, this application runs under [pm2](https://pm2.io) using an [ecosystem.config.js](https://pm2.keymetrics.io/docs/usage/application-declaration/) file similar to this:

app.js

@@ -109,7 +109,7 @@ const disconnect = () => {
     httpServer?.on('close', resolve);
     httpServer?.close();
     srf.disconnect();
-    srf.locals.mediaservers.forEach((ms) => ms.disconnect());
+    srf.locals.mediaservers?.forEach((ms) => ms.disconnect());
   });
 };
 

docs/contributing.md

@@ -45,7 +45,7 @@ The GCP credential is the JSON service key in stringified format.
 
 #### Install Docker
 
-The test suite ralso equires [Docker](https://www.docker.com/) and docker-compose to be installed on your laptop.  Docker is used to set up a network with all of the elements required to test the jambonz-feature-server in a black-box type of fashion.
+The test suite also requires [Docker](https://www.docker.com/) and docker-compose to be installed on your laptop.  Docker is used to set up a network with all of the elements required to test the jambonz-feature-server in a black-box type of fashion.
 
 Once you have docker installed, you can optionally make sure everything Docker-wise is working properly by running this command from the project folder:
 

lib/config.js

@@ -28,10 +28,6 @@ const JAMBONES_MYSQL_PORT = parseInt(process.env.JAMBONES_MYSQL_PORT, 10) || 330
 const JAMBONES_MYSQL_REFRESH_TTL = parseInt(process.env.JAMBONES_MYSQL_REFRESH_TTL, 10) || 0;
 const JAMBONES_MYSQL_CONNECTION_LIMIT = parseInt(process.env.JAMBONES_MYSQL_CONNECTION_LIMIT, 10) || 10;
 
-/* redis */
-const JAMBONES_REDIS_HOST = process.env.JAMBONES_REDIS_HOST;
-const JAMBONES_REDIS_PORT = parseInt(process.env.JAMBONES_REDIS_PORT, 10) || 6379;
-
 /* gather and hints */
 const JAMBONES_GATHER_EARLY_HINTS_MATCH = process.env.JAMBONES_GATHER_EARLY_HINTS_MATCH;
 const JAMBONZ_GATHER_EARLY_HINTS_MATCH = process.env.JAMBONZ_GATHER_EARLY_HINTS_MATCH;
@@ -124,35 +120,20 @@ const HTTP_TIMEOUT = 10000;
 const HTTP_PROXY_IP = process.env.JAMBONES_HTTP_PROXY_IP;
 const HTTP_PROXY_PORT = process.env.JAMBONES_HTTP_PROXY_PORT;
 const HTTP_PROXY_PROTOCOL = process.env.JAMBONES_HTTP_PROXY_PROTOCOL || 'http';
+const HTTP_USER_AGENT_HEADER = process.env.JAMBONES_HTTP_USER_AGENT_HEADER || 'jambonz';
 
 const OPTIONS_PING_INTERVAL = parseInt(process.env.OPTIONS_PING_INTERVAL, 10) || 30000;
 
-const JAMBONES_REDIS_SENTINELS = process.env.JAMBONES_REDIS_SENTINELS ? {
-  sentinels: process.env.JAMBONES_REDIS_SENTINELS.split(',').map((sentinel) => {
-    let host, port = 26379;
-    if (sentinel.includes(':')) {
-      const arr = sentinel.split(':');
-      host = arr[0];
-      port = parseInt(arr[1], 10);
-    } else {
-      host = sentinel;
-    }
-    return {host, port};
-  }),
-  name: process.env.JAMBONES_REDIS_SENTINEL_MASTER_NAME,
-  ...(process.env.JAMBONES_REDIS_SENTINEL_PASSWORD && {
-    password: process.env.JAMBONES_REDIS_SENTINEL_PASSWORD
-  }),
-  ...(process.env.JAMBONES_REDIS_SENTINEL_USERNAME && {
-    username: process.env.JAMBONES_REDIS_SENTINEL_USERNAME
-  })
-} : null;
 const JAMBONZ_RECORD_WS_BASE_URL = process.env.JAMBONZ_RECORD_WS_BASE_URL || process.env.JAMBONES_RECORD_WS_BASE_URL;
 const JAMBONZ_RECORD_WS_USERNAME = process.env.JAMBONZ_RECORD_WS_USERNAME || process.env.JAMBONES_RECORD_WS_USERNAME;
 const JAMBONZ_RECORD_WS_PASSWORD = process.env.JAMBONZ_RECORD_WS_PASSWORD || process.env.JAMBONES_RECORD_WS_PASSWORD;
 const JAMBONZ_DISABLE_DIAL_PAI_HEADER = process.env.JAMBONZ_DISABLE_DIAL_PAI_HEADER || false;
 const JAMBONES_DISABLE_DIRECT_P2P_CALL = process.env.JAMBONES_DISABLE_DIRECT_P2P_CALL || false;
 
+const JAMBONES_EAGERLY_PRE_CACHE_AUDIO = process.env.JAMBONES_EAGERLY_PRE_CACHE_AUDIO;
+
+const JAMBONES_USE_FREESWITCH_TIMER_FD = process.env.JAMBONES_USE_FREESWITCH_TIMER_FD;
+
 module.exports = {
   JAMBONES_MYSQL_HOST,
   JAMBONES_MYSQL_USER,
@@ -170,14 +151,12 @@ module.exports = {
   JAMBONZ_GATHER_EARLY_HINTS_MATCH,
   JAMBONES_GATHER_CLEAR_GLOBAL_HINTS_ON_EMPTY_HINTS,
   JAMBONES_FREESWITCH,
-  JAMBONES_REDIS_HOST,
-  JAMBONES_REDIS_PORT,
-  JAMBONES_REDIS_SENTINELS,
   SMPP_URL,
   JAMBONES_NETWORK_CIDR,
   JAMBONES_API_BASE_URL,
   JAMBONES_TIME_SERIES_HOST,
   JAMBONES_INJECT_CONTENT,
+  JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
   JAMBONES_ESL_LISTEN_ADDRESS,
   JAMBONES_SBCS,
   JAMBONES_OTEL_ENABLED,
@@ -220,6 +199,7 @@ module.exports = {
   HTTP_PROXY_IP,
   HTTP_PROXY_PORT,
   HTTP_PROXY_PROTOCOL,
+  HTTP_USER_AGENT_HEADER,
   OPTIONS_PING_INTERVAL,
   RESPONSE_TIMEOUT_MS,
   JAMBONES_WS_HANDSHAKE_TIMEOUT_MS,
@@ -235,5 +215,6 @@ module.exports = {
   JAMBONZ_RECORD_WS_USERNAME,
   JAMBONZ_RECORD_WS_PASSWORD,
   JAMBONZ_DISABLE_DIAL_PAI_HEADER,
-  JAMBONES_DISABLE_DIRECT_P2P_CALL
+  JAMBONES_DISABLE_DIRECT_P2P_CALL,
+  JAMBONES_USE_FREESWITCH_TIMER_FD
 };

lib/http-routes/api/create-call.js

@@ -5,7 +5,7 @@ const CallInfo = require('../../session/call-info');
 const {CallDirection, CallStatus} = require('../../utils/constants');
 const uuidv4 = require('uuid-random');
 const SipError = require('drachtio-srf').SipError;
-const { validationResult } = require('express-validator');
+const { validationResult, body } = require('express-validator');
 const { validate } = require('@jambonz/verb-specifications');
 const sysError = require('./error');
 const HttpRequestor = require('../../utils/http-requestor');
@@ -13,7 +13,7 @@ const WsRequestor = require('../../utils/ws-requestor');
 const RootSpan = require('../../utils/call-tracer');
 const dbUtils = require('../../utils/db-utils');
 const { mergeSdpMedia, extractSdpMedia } = require('../../utils/sdp-utils');
-const { createCallSchema } = require('../schemas/create-call');
+const { createCallSchema, customSanitizeFunction } = require('../schemas/create-call');
 
 const removeNullProperties = (obj) => (Object.keys(obj).forEach((key) => obj[key] === null && delete obj[key]), obj);
 const removeNulls = (req, res, next) => {
@@ -24,6 +24,12 @@ const removeNulls = (req, res, next) => {
 router.post('/',
   removeNulls,
   createCallSchema,
+  body('tag').custom((value) => {
+    if (value) {
+      customSanitizeFunction(value);
+    }
+    return true;
+  }),
   async(req, res) => {
     const {logger} = req.app.locals;
     const errors = validationResult(req);

lib/http-routes/api/update-call.js

@@ -45,7 +45,7 @@ function retrieveCallSession(callSid, opts) {
 router.post('/:callSid', async(req, res) => {
   const logger = req.app.locals.logger;
   const callSid = req.params.callSid;
-  logger.debug({body: req.body}, 'got upateCall request');
+  logger.debug({body: req.body}, 'got updateCall request');
   try {
     const cs = retrieveCallSession(callSid, req.body);
     if (!cs) {

lib/http-routes/schemas/create-call.js

@@ -46,11 +46,6 @@ const createCallSchema = checkSchema({
     optional: true,
     errorMessage: 'Invalid tag',
   },
-  'tag.*': {
-    trim: true,
-    escape: true,
-    stripLow: true,
-  },
   app_json: {
     isString: true,
     optional: true,
@@ -109,6 +104,31 @@ const createCallSchema = checkSchema({
   }
 }, ['body']);
 
-module.exports = {
+const customSanitizeFunction = (value) => {
-  createCallSchema
+  try {
+    if (Array.isArray(value)) {
+      value = value.map((item) => customSanitizeFunction(item));
+    } else if (typeof value === 'object') {
+      Object.keys(value).forEach((key) => {
+        value[key] = customSanitizeFunction(value[key]);
+      });
+    } else if (typeof value === 'string') {
+      /* trims characters at the beginning and at the end of a string */
+      value = value.trim();
+
+      /* Verify strings including 'http' via new URL */
+      if (value.includes('http')) {
+        value = new URL(value).toString();
+      }
+    }
+  } catch (error) {
+    value = `Error: ${error.message}`;
+  }
+
+  return value;
+};
+
+module.exports = {
+  createCallSchema,
+  customSanitizeFunction
 };

lib/middleware.js

@@ -23,7 +23,8 @@ module.exports = function(srf, logger) {
     lookupAppBySid,
     lookupAppByRealm,
     lookupAppByTeamsTenant,
-    registrar
+    registrar,
+    lookupClientByAccountAndUsername
   }  = srf.locals.dbHelpers;
   const {
     writeAlerts,
@@ -48,10 +49,23 @@ module.exports = function(srf, logger) {
     const account_sid = req.get('X-Account-Sid');
     req.locals = {callSid, account_sid, callId};
 
-    if (req.has('X-Authenticated-User')) req.locals.originatingUser = req.get('X-Authenticated-User');
+    let clientDb = null;
+    if (req.has('X-Authenticated-User')) {
+      req.locals.originatingUser = req.get('X-Authenticated-User');
+      let clientSettings;
+      const arr = /^(.*)@(.*)/.exec(req.locals.originatingUser);
+      if (arr) {
+        [clientSettings] = await lookupClientByAccountAndUsername(account_sid, arr[1]);
+      }
+      clientDb = await registrar.query(req.locals.originatingUser);
+      clientDb = {
+        ...clientDb,
+        ...clientSettings,
+      };
+    }
 
     // check for call to application
-    if (uri.user?.startsWith('app-') && req.locals.originatingUser) {
+    if (uri.user?.startsWith('app-') && req.locals.originatingUser && clientDb?.allow_direct_app_calling) {
       const application_sid = uri.user.match(/app-(.*)/)[1];
       logger.debug(`got application from Request URI header: ${application_sid}`);
       req.locals.application_sid = application_sid;
@@ -61,13 +75,13 @@ module.exports = function(srf, logger) {
       req.locals.application_sid = application_sid;
     }
     // check for call to queue
-    if (uri.user?.startsWith('queue-') && req.locals.originatingUser) {
+    if (uri.user?.startsWith('queue-') && req.locals.originatingUser && clientDb?.allow_direct_queue_calling) {
       const queue_name = uri.user.match(/queue-(.*)/)[1];
       logger.debug(`got Queue from Request URI header: ${queue_name}`);
       req.locals.queue_name = queue_name;
     }
     // check for call to registered user
-    if (!JAMBONES_DISABLE_DIRECT_P2P_CALL && req.locals.originatingUser) {
+    if (!JAMBONES_DISABLE_DIRECT_P2P_CALL && req.locals.originatingUser && clientDb?.allow_direct_user_calling) {
       const arr = /^(.*)@(.*)/.exec(req.locals.originatingUser);
       if (arr) {
         const sipRealm = arr[2];
@@ -83,7 +97,7 @@ module.exports = function(srf, logger) {
     if (req.has('X-MS-Teams-Tenant-FQDN')) req.locals.msTeamsTenant = req.get('X-MS-Teams-Tenant-FQDN');
     if (req.has('X-Cisco-Recording-Participant')) {
       const ciscoParticipants = req.get('X-Cisco-Recording-Participant');
-      const regex = /sip:[\d]+@[\d]+\.[\d]+\.[\d]+\.[\d]+/g;
+      const regex = /sip:[a-zA-Z0-9]+@[a-zA-Z0-9.-_]+/g;
       const sipURIs = ciscoParticipants.match(regex);
       logger.info(`X-Cisco-Recording-Participant : ${sipURIs} `);
       if (sipURIs && sipURIs.length > 0) {
@@ -152,7 +166,7 @@ module.exports = function(srf, logger) {
         };
         logger.info({callId, metadata, sdp}, 'successfully parsed SIPREC payload');
       } catch (err) {
-        logger.info({callId}, 'Error parsing multipart payload');
+        logger.info({err, callId}, 'Error parsing multipart payload');
         return res.send(503);
       }
     }
@@ -368,7 +382,7 @@ module.exports = function(srf, logger) {
           },
           recognizer: {
             vendor: app.speech_recognizer_vendor,
-            ...(app.speech_synthesis_label && {label: app.speech_synthesis_label}),
+            ...(app.speech_recognizer_label && {label: app.speech_recognizer_label}),
             language: app.speech_recognizer_language,
             ...(app.fallback_speech_recognizer_vendor && {fallback_vendor: app.fallback_speech_recognizer_vendor}),
             ...(app.fallback_speech_recognizer_label && {fallback_label: app.fallback_speech_recognizer_label}),

lib/session/adulting-call-session.js

@@ -1,4 +1,6 @@
 const CallSession = require('./call-session');
+const {CallStatus} = require('../utils/constants');
+const moment = require('moment');
 
 /**
  * @classdesc Subclass of CallSession.  Represents a CallSession
@@ -19,12 +21,14 @@ class AdultingCallSession extends CallSession {
       rootSpan
     });
     this.sd = singleDialer;
+    this.req = callInfo.req;
 
     this.sd.dlg.on('destroy', () => {
       this.logger.info('AdultingCallSession: called party hung up');
       this._callReleased();
     });
     this.sd.emit('adulting');
+    this.on('callStatusChange', this._notifyCallStatusChange.bind(this));
   }
 
   get dlg() {
@@ -49,6 +53,18 @@ class AdultingCallSession extends CallSession {
   }
 
   _callerHungup() {
+    if (this.dlg.connectTime) {
+      const duration = moment().diff(this.dlg.connectTime, 'seconds');
+      this.rootSpan.setAttributes({'call.termination': 'hangup by caller'});
+      this.callInfo.callTerminationBy = 'caller';
+      this.emit('callStatusChange', {
+        callStatus: CallStatus.Completed,
+        duration
+      });
+    }
+    this.logger.info('InboundCallSession: caller hung up');
+    this._callReleased();
+    this.req.removeAllListeners('cancel');
   }
 }
 

lib/session/call-session.js

@@ -19,11 +19,12 @@ const HttpRequestor = require('../utils/http-requestor');
 const WsRequestor = require('../utils/ws-requestor');
 const {
   JAMBONES_INJECT_CONTENT,
+  JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
   AWS_REGION,
-  JAMBONZ_RECORD_WS_BASE_URL,
+  JAMBONES_USE_FREESWITCH_TIMER_FD
-  JAMBONZ_RECORD_WS_USERNAME,
-  JAMBONZ_RECORD_WS_PASSWORD,
 } = require('../config');
+const bent = require('bent');
+const BackgroundTaskManager = require('../utils/background-task-manager');
 const BADPRECONDITIONS = 'preconditions not met';
 const CALLER_CANCELLED_ERR_MSG = 'Response not sent due to unknown transaction';
 
@@ -66,6 +67,11 @@ class CallSession extends Emitter {
     this.callGone = false;
     this.notifiedComplete = false;
     this.rootSpan = rootSpan;
+    this.backgroundTaskManager = new BackgroundTaskManager({
+      cs: this,
+      logger,
+      rootSpan
+    });
 
     this._origRecognizerSettings = {
       vendor: this.application?.speech_recognizer_vendor,
@@ -136,8 +142,7 @@ class CallSession extends Emitter {
   }
 
   get isBackGroundListen() {
-    return !(this.backgroundListenTask === null ||
+    return this.backgroundTaskManager.isTaskRunning('listen');
-    this.backgroundListenTask === undefined);
   }
 
   /**
@@ -170,6 +175,18 @@ class CallSession extends Emitter {
     return this.application.notifier;
   }
 
+  /**
+   * syntheizer
+   */
+
+  get synthesizer() {
+    return this._synthesizer;
+  }
+
+  set synthesizer(synth) {
+    this._synthesizer = synth;
+  }
+
   /**
    * default vendor to use for speech synthesis if not provided in the app
    */
@@ -252,6 +269,16 @@ class CallSession extends Emitter {
   set fallbackSpeechRecognizerVendor(vendor) {
     this.application.fallback_speech_recognizer_vendor = vendor;
   }
+  /**
+   * recognizer
+   */
+  get recognizer() {
+    return this._recognizer;
+  }
+
+  set recognizer(rec) {
+    this._recognizer = rec;
+  }
   /**
    * default vendor to use for speech recognition if not provided in the app
    */
@@ -357,11 +384,11 @@ class CallSession extends Emitter {
   }
 
   get isBotModeEnabled() {
-    return this.backgroundGatherTask;
+    return this.backgroundTaskManager.isTaskRunning('bargeIn');
   }
 
   get isListenEnabled() {
-    return this.backgroundListenTask;
+    return this.backgroundTaskManager.isTaskRunning('listen');
   }
 
   get b3() {
@@ -590,117 +617,41 @@ class CallSession extends Emitter {
     }
   }
 
-  async startBackgroundListen(opts, bugname) {
-    if (this.isListenEnabled) {
-      this.logger.info('CallSession:startBackgroundListen - listen is already enabled, ignoring request');
-      return;
-    }
-    try {
-      this.logger.debug({opts}, 'CallSession:startBackgroundListen');
-      const t = normalizeJambones(this.logger, [opts]);
-      this.backgroundListenTask = makeTask(this.logger, t[0]);
-      this.backgroundListenTask.bugname = bugname;
-      // Remove unneeded customer data to be sent to api server.
-      this.backgroundListenTask.ignoreCustomerData = true;
-      const resources = await this._evaluatePreconditions(this.backgroundListenTask);
-      const {span, ctx} = this.rootSpan.startChildSpan(`background-listen:${this.backgroundListenTask.summary}`);
-      this.backgroundListenTask.span = span;
-      this.backgroundListenTask.ctx = ctx;
-      this.backgroundListenTask.exec(this, resources)
-        .then(() => {
-          this.logger.info('CallSession:startBackgroundListen: listen completed');
-          this.backgroundListenTask && this.backgroundListenTask.removeAllListeners();
-          this.backgroundListenTask && this.backgroundListenTask.span.end();
-          this.backgroundListenTask = null;
-          return;
-        })
-        .catch((err) => {
-          this.logger.info({err}, 'CallSession:startBackgroundListen: listen threw error');
-          this.backgroundListenTask && this.backgroundListenTask.removeAllListeners();
-          this.backgroundListenTask && this.backgroundListenTask.span.end();
-          this.backgroundListenTask = null;
-        });
-    } catch (err) {
-      this.logger.info({err, opts}, 'CallSession:startBackgroundListen - Error creating listen task');
-    }
-  }
-
-  async stopBackgroundListen() {
-    this.logger.debug('CallSession:stopBackgroundListen');
-    try {
-      if (this.backgroundListenTask) {
-        this.backgroundListenTask.removeAllListeners();
-        this.backgroundListenTask.kill().catch(() => {});
-      }
-    } catch (err) {
-      this.logger.info({err}, 'CallSession:stopBackgroundListen - Error stopping listen task');
-    }
-  }
-
   async enableBotMode(gather, autoEnable) {
     try {
-      const t = normalizeJambones(this.logger, [gather]);
+      let task;
-      const task = makeTask(this.logger, t[0]);
-
       if (this.isBotModeEnabled) {
-        const currInput = this.backgroundGatherTask.input;
+        task = this.backgroundTaskManager.getTask('bargeIn');
-        const newInput = task.input;
+        const currInput = task.input;
+        const t = normalizeJambones(this.logger, [gather]);
+        const tmpTask = makeTask(this.logger, t[0]);
+        const newInput = tmpTask.input;
         if (JSON.stringify(currInput) === JSON.stringify(newInput)) {
           this.logger.info('CallSession:enableBotMode - bot mode currently enabled, ignoring request to start again');
           return;
-        }
+        } else {
-        else {
           this.logger.info({currInput, newInput},
-            'CallSession:enableBotMode - restarting background gather to apply new input type');
+            'CallSession:enableBotMode - restarting background bargeIn to apply new input type');
-          this.backgroundGatherTask.sticky = false;
+          task.sticky = false;
           await this.disableBotMode();
         }
       }
-      this.backgroundGatherTask = task;
+      task = await this.backgroundTaskManager.newTask('bargeIn', gather);
-      this._bargeInEnabled = true;
+      task.sticky = autoEnable;
-      this.backgroundGatherTask
+      task.once('bargeIn-done', () => {
-        .once('dtmf', this._clearTasks.bind(this, this.backgroundGatherTask))
+        if (this.requestor instanceof WsRequestor) {
-        .once('vad', this._clearTasks.bind(this, this.backgroundGatherTask))
+          try {
-        .once('transcription', this._clearTasks.bind(this, this.backgroundGatherTask))
+            this.kill(true);
-        .once('timeout', this._clearTasks.bind(this, this.backgroundGatherTask));
+          } catch (err) {}
-      this.logger.info({gather}, 'CallSession:enableBotMode - starting background gather');
+        }
-      const resources = await this._evaluatePreconditions(this.backgroundGatherTask);
+      });
-      const {span, ctx} = this.rootSpan.startChildSpan(`background-gather:${this.backgroundGatherTask.summary}`);
+      this.logger.info({gather}, 'CallSession:enableBotMode - starting background bargeIn');
-      this.backgroundGatherTask.span = span;
-      this.backgroundGatherTask.ctx = ctx;
-      this.backgroundGatherTask.sticky = autoEnable;
-      this.backgroundGatherTask.exec(this, resources)
-        .then(() => {
-          this.logger.info('CallSession:enableBotMode: gather completed');
-          this.backgroundGatherTask && this.backgroundGatherTask.removeAllListeners();
-          this.backgroundGatherTask && this.backgroundGatherTask.span.end();
-          const sticky = this.backgroundGatherTask?.sticky;
-          this.backgroundGatherTask = null;
-          if (sticky && !this.callGone && !this._stopping && this._bargeInEnabled) {
-            this.logger.info('CallSession:enableBotMode: restarting background gather');
-            setImmediate(() => this.enableBotMode(gather, true));
-          }
-          return;
-        })
-        .catch((err) => {
-          this.logger.info({err}, 'CallSession:enableBotMode: gather threw error');
-          this.backgroundGatherTask && this.backgroundGatherTask.removeAllListeners();
-          this.backgroundGatherTask && this.backgroundGatherTask.span.end();
-          this.backgroundGatherTask = null;
-        });
     } catch (err) {
-      this.logger.info({err, gather}, 'CallSession:enableBotMode - Error creating gather task');
+      this.logger.info({err, gather}, 'CallSession:enableBotMode - Error creating bargeIn task');
     }
   }
   async disableBotMode() {
-    this._bargeInEnabled = false;
+    this.backgroundTaskManager.stop('bargeIn');
-    if (this.backgroundGatherTask) {
-      try {
-        this.backgroundGatherTask.removeAllListeners();
-        await this.backgroundGatherTask.kill();
-      } catch (err) {}
-      this.backgroundGatherTask = null;
-    }
   }
 
   setConferenceDetails(memberId, confName, confUuid) {
@@ -740,7 +691,7 @@ class CallSession extends Emitter {
         (type === 'stt' && credential.use_for_stt)
       )) {
         this.logger.info(
-          `Speech vendor: ${credential.vendor} ${credential.label ? `, label: ${credential.label}` : ''} selected`);
+          `${type}: ${credential.vendor} ${credential.label ? `, label: ${credential.label}` : ''}  `);
         if ('google' === vendor) {
           if (type === 'tts' && !credential.tts_tested_ok ||
             type === 'stt' && !credential.stt_tested_ok) {
@@ -801,7 +752,9 @@ class CallSession extends Emitter {
         else if ('deepgram' === vendor) {
           return {
             speech_credential_sid: credential.speech_credential_sid,
-            api_key: credential.api_key
+            api_key: credential.api_key,
+            deepgram_stt_uri: credential.deepgram_stt_uri,
+            deepgram_stt_use_tls: credential.deepgram_stt_use_tls
           };
         }
         else if ('soniox' === vendor) {
@@ -833,13 +786,19 @@ class CallSession extends Emitter {
         } else if ('elevenlabs' === vendor) {
           return {
             api_key: credential.api_key,
-            model_id: credential.model_id
+            model_id: credential.model_id,
+            options: credential.options
           };
         } else if ('assemblyai' === vendor) {
           return {
             speech_credential_sid: credential.speech_credential_sid,
             api_key: credential.api_key
           };
+        } else if ('whisper' === vendor) {
+          return {
+            api_key: credential.api_key,
+            model_id: credential.model_id
+          };
         } else if (vendor.startsWith('custom:')) {
           return {
             speech_credential_sid: credential.speech_credential_sid,
@@ -873,12 +832,16 @@ class CallSession extends Emitter {
       const task = this.tasks.shift();
       this.logger.info(`CallSession:exec starting task #${stackNum}:${taskNum}: ${task.name}`);
       this._notifyTaskStatus(task, {event: 'starting'});
+      // Register verbhook span wait for end
+      task.on('VerbHookSpanWaitForEnd', ({span}) => {
+        this.verbHookSpan = span;
+      });
       try {
         const resources = await this._evaluatePreconditions(task);
         let skip = false;
         this.currentTask = task;
         if (TaskName.Gather === task.name && this.isBotModeEnabled) {
-          if (this.backgroundGatherTask.updateTaskInProgress(task) !== false) {
+          if (this.backgroundTaskManager.getTask('bargeIn').updateTaskInProgress(task) !== false) {
             this.logger.info(`CallSession:exec skipping #${stackNum}:${taskNum}: ${task.name}`);
             skip = true;
           }
@@ -913,7 +876,8 @@ class CallSession extends Emitter {
       if (0 === this.tasks.length &&
         this.requestor instanceof WsRequestor &&
         !this.requestor.closedGracefully &&
-        !this.callGone
+        !this.callGone &&
+        !this.isConfirmCallSession
       ) {
         try {
           await this._awaitCommandsOrHangup();
@@ -1014,6 +978,64 @@ class CallSession extends Emitter {
     }
   }
 
+  /**
+   * perform live call control - create rest:dial
+   * @param {obj} opts create call options
+   */
+  async _lccCallDial(opts) {
+    try {
+      const restDialUrl = `${this.srf.locals.serviceUrl}/v1/createCall`;
+      await this.transformInputIfRequired(opts);
+      const resp = bent('POST', 'json', 201)(restDialUrl, opts);
+      this.logger.info(resp.body, 'successfully create outbound call');
+      return resp.body;
+    } catch (err) {
+      if (err.json) {
+        err.body = await err.json();
+      }
+      this.logger.error(err, 'failed to create outbound call from ' + this.callSid);
+      this._notifyTaskError(err.body);
+    }
+  }
+
+  async transformInputIfRequired(opts) {
+    const {
+      lookupAppBySid
+    }  = this.srf.locals.dbHelpers;
+    opts.account_sid = this.accountSid;
+
+    if (opts.application_sid) {
+      this.logger.debug(`Callsession:_validateCreateCall retrieving application ${opts.application_sid}`);
+      const application = await lookupAppBySid(opts.application_sid);
+      Object.assign(opts, {
+        call_hook: application.call_hook,
+        app_json: application.app_json,
+        call_status_hook: application.call_status_hook,
+        speech_synthesis_vendor: application.speech_synthesis_vendor,
+        speech_synthesis_language: application.speech_synthesis_language,
+        speech_synthesis_voice: application.speech_synthesis_voice,
+        speech_recognizer_vendor: application.speech_recognizer_vendor,
+        speech_recognizer_language: application.speech_recognizer_language
+      });
+      this.logger.debug({opts, application}, 'Callsession:_validateCreateCall augmented with application settings');
+    }
+
+    if (typeof opts.call_hook === 'string') {
+      const url = opts.call_hook;
+      opts.call_hook = {
+        url,
+        method: 'POST'
+      };
+    }
+    if (typeof opts.call_status_hook === 'string') {
+      const url = opts.call_status_hook;
+      opts.call_status_hook = {
+        url,
+        method: 'POST'
+      };
+    }
+  }
+
   /**
    * perform live call control -- set a new call_hook
    * @param {object} opts
@@ -1021,30 +1043,22 @@ class CallSession extends Emitter {
    * @param {object} [opts.call_hook] - new call_status_hook
    */
   async _lccCallHook(opts) {
-    const webhooks = [];
     let sd, tasks, childTasks;
     const b3 = this.b3;
     const httpHeaders = b3 && {b3};
 
     if (opts.call_hook || opts.child_call_hook) {
       if (opts.call_hook) {
-        webhooks.push(this.requestor.request('session:redirect', opts.call_hook, this.callInfo.toJSON(), httpHeaders));
+        tasks = await this.requestor.request('session:redirect', opts.call_hook, this.callInfo.toJSON(), httpHeaders);
       }
       if (opts.child_call_hook) {
         /* child call hook only allowed from a connected Dial state */
         const task = this.currentTask;
         sd = task.sd;
         if (task && TaskName.Dial === task.name && sd) {
-          webhooks.push(this.requestor.request(
+          childTasks = [];
-            'session:redirect', opts.child_call_hook, sd.callInfo.toJSON(), httpHeaders));
         }
       }
-      const [tasks1, tasks2] = await Promise.all(webhooks);
-      if (opts.call_hook) {
-        tasks = tasks1;
-        if (opts.child_call_hook) childTasks = tasks2;
-      }
-      else childTasks = tasks1;
     }
     else if (opts.parent_call || opts.child_call) {
       const {parent_call, child_call} = opts;
@@ -1064,21 +1078,31 @@ class CallSession extends Emitter {
       const cs = await sd.doAdulting({
         logger: childLogger,
         application: this.application,
-        tasks: t
+        tasks: t,
+        call_hook_url: opts.child_call_hook
       });
 
       /* need to update the callSid of the child with its own (new) AdultingCallSession */
       sessionTracker.add(cs.callSid, cs);
     }
     if (tasks) {
+      if (!this.ep) {
+        await this.reAnchorMedia();
+      }
       const t = normalizeJambones(this.logger, tasks).map((tdata) => makeTask(this.logger, tdata));
       this.logger.info({tasks: listTaskNames(t)}, 'CallSession:_lccCallHook new task list');
       this.replaceApplication(t);
+      if (this.wakeupResolver) {
+        //this.logger.debug({resolution}, 'CallSession:_onCommand - got commands, waking up..');
+        this.wakeupResolver({reason: 'lcc: new tasks'});
+        this.wakeupResolver = null;
+      }
     }
     else {
       /* we started a new app on the child leg, but nothing given for parent so hang him up */
       this.currentTask.kill(this);
     }
+    this._endVerbHookSpan();
   }
 
   /**
@@ -1104,6 +1128,9 @@ class CallSession extends Emitter {
    * @param {string} opts.transcribe_status - 'pause' or 'resume'
   */
   async _lccTranscribeStatus(opts) {
+    if (this.backgroundTaskManager.isTaskRunning('transcribe')) {
+      this.backgroundTaskManager.getTask('transcribe').updateTranscribe(opts.transcribe_status);
+    }
     const task = this.currentTask;
     if (!task || ![TaskName.Dial, TaskName.Transcribe].includes(task.name)) {
       return this.logger.info(`CallSession:_lccTranscribeStatus - invalid transcribe_status in task ${task.name}`);
@@ -1116,6 +1143,21 @@ class CallSession extends Emitter {
     transcribeTask.updateTranscribe(opts.transcribe_status);
   }
 
+  /**
+   * perform live call control -- update customer data
+   * @param {object} opts
+   * @param {object} opts.tag - customer data
+  */
+  _lccTag(opts) {
+    const {tag} = opts;
+    if (typeof tag !== 'object' || Array.isArray(tag) || tag === null) {
+      this.logger.info('CallSession:_lccTag - invalid tag data');
+      return;
+    }
+    this.logger.debug({customerData: tag}, 'CallSession:_lccTag set customer data in callInfo');
+    this.callInfo.customerData = tag;
+  }
+
   async _lccMuteStatus(callSid, mute) {
     // this whole thing requires us to be in a Dial or Conference verb
     const task = this.currentTask;
@@ -1166,6 +1208,38 @@ class CallSession extends Emitter {
     }
   }
 
+  /**
+   * perform live call control - send RFC 2833 DTMF
+   * @param {obj} opts
+   * @param {string} opts.dtmf.digit - DTMF digit
+   * @param {string} opts.dtmf.duration - Optional, Duration
+   */
+  async _lccDtmf(opts, callSid) {
+    const {dtmf} = opts;
+    const {digit, duration = 250} = dtmf;
+    if (!this.hasStableDialog) {
+      this.logger.info('CallSession:_lccDtmf - invalid command as we do not have a stable call');
+      return;
+    }
+    try {
+      const dlg = callSid === this.callSid ? this.dlg : this.currentTask.dlg;
+      const res = await dlg.request({
+        method: 'INFO',
+        headers: {
+          'Content-Type': 'application/dtmf',
+          'X-Reason': 'Dtmf'
+        },
+        body: `Signal=${digit}
+Duration=${duration} `
+      });
+      this.logger.debug({res}, `CallSession:_lccDtmf
+ got response to INFO DTMF digit=${digit} and duration=${duration}`);
+      return res;
+    } catch (err) {
+      this.logger.error({err}, 'CallSession:_lccDtmf - error sending INFO RFC 2833 DTMF');
+    }
+  }
+
   /**
    * perform live call control -- whisper to one party or the other on a call
    * @param {array} opts - array of play or say tasks
@@ -1220,7 +1294,7 @@ class CallSession extends Emitter {
   /**
    * perform live call control
    * @param {object} opts - update instructions
-   * @param {string} callSid - identifies call toupdate
+   * @param {string} callSid - identifies call to update
    */
   async updateCall(opts, callSid) {
     this.logger.debug(opts, 'CallSession:updateCall');
@@ -1249,13 +1323,18 @@ class CallSession extends Emitter {
     else if (opts.sip_request) {
       const res = await this._lccSipRequest(opts, callSid);
       return {status: res.status, reason: res.reason};
+    } else if (opts.dtmf) {
+      await this._lccDtmf(opts, callSid);
     }
     else if (opts.record) {
       await this.notifyRecordOptions(opts.record);
     }
+    else if (opts.tag) {
+      return this._lccTag(opts);
+    }
 
     // whisper may be the only thing we are asked to do, or it may that
-    // we are doing a whisper after having muted, paused reccording etc..
+    // we are doing a whisper after having muted, paused recording etc..
     if (opts.whisper) {
       return this._lccWhisper(opts, callSid);
     }
@@ -1298,6 +1377,7 @@ class CallSession extends Emitter {
   kill(onBackgroundGatherBargein = false) {
     if (this.isConfirmCallSession) this.logger.debug('CallSession:kill (ConfirmSession)');
     else this.logger.info('CallSession:kill');
+    this._endVerbHookSpan();
     if (this.currentTask) {
       this.currentTask.kill(this);
       this.currentTask = null;
@@ -1313,7 +1393,6 @@ class CallSession extends Emitter {
             get the full transcription.
           */
           delete t.bargeIn.enable;
-          this._bargeInEnabled = false;
           this.logger.info('CallSession:kill - found bargein disabled in the stack, clearing to that point');
           break;
         }
@@ -1324,6 +1403,35 @@ class CallSession extends Emitter {
     this.taskIdx = 0;
   }
 
+  _preCacheAudio(newTasks) {
+    for (const task of newTasks) {
+      if (task.name === TaskName.Config && task.hasSynthesizer) {
+        /* if they change synthesizer settings don't try to precache */
+        break;
+      }
+      if (task.name === TaskName.Say) {
+        /* identify vendor language, voice, and label */
+        const vendor = task.synthesizer.vendor && task.synthesizer.vendor !== 'default' ?
+          task.synthesizer.vendor :
+          this.speechSynthesisVendor;
+        const language = task.synthesizer.language && task.synthesizer.language !== 'default' ?
+          task.synthesizer.language :
+          this.speechSynthesisLanguage ;
+        const voice =  task.synthesizer.voice && task.synthesizer.voice !== 'default' ?
+          task.synthesizer.voice :
+          this.speechSynthesisVoice;
+        const label = task.synthesizer.label && task.synthesizer.label !== 'default' ?
+          task.synthesizer.label :
+          this.speechSynthesisLabel;
+
+        this.logger.info({vendor, language, voice, label},
+          'CallSession:_preCacheAudio - precaching audio for future prompt');
+        task._synthesizeWithSpecificVendor(this, this.ep, {vendor, language, voice, label, preCache: true})
+          .catch((err) => this.logger.error(err, 'CallSession:_preCacheAudio - error precaching audio'));
+      }
+    }
+  }
+
   /**
    * Append tasks to the current execution stack UNLESS there is a gather in the stack.
    * in that case, insert the tasks before the gather AND if the tasks include
@@ -1370,10 +1478,11 @@ class CallSession extends Emitter {
 
   _onCommand({msgid, command, call_sid, queueCommand, data}) {
     this.logger.info({msgid, command, queueCommand, data}, 'CallSession:_onCommand - received command');
-    const resolution = {reason: 'received command', queue: queueCommand, command};
+    let resolution;
     switch (command) {
       case 'redirect':
         if (Array.isArray(data)) {
+          this._endVerbHookSpan();
           const t = normalizeJambones(this.logger, data)
             .map((tdata) => makeTask(this.logger, tdata));
           if (!queueCommand) {
@@ -1381,13 +1490,16 @@ class CallSession extends Emitter {
             this.replaceApplication(t);
           }
           else if (JAMBONES_INJECT_CONTENT) {
+            if (JAMBONES_EAGERLY_PRE_CACHE_AUDIO) this._preCacheAudio(t);
             this._injectTasks(t);
             this.logger.info({tasks: listTaskNames(this.tasks)}, 'CallSession:_onCommand - updated task list');
           }
           else {
+            if (JAMBONES_EAGERLY_PRE_CACHE_AUDIO) this._preCacheAudio(t);
             this.tasks.push(...t);
             this.logger.info({tasks: listTaskNames(this.tasks)}, 'CallSession:_onCommand - updated task list');
           }
+          resolution = {reason: 'received command, new tasks', queue: queueCommand, command};
           resolution.command = listTaskNames(t);
         }
         else this._lccCallHook(data);
@@ -1397,6 +1509,14 @@ class CallSession extends Emitter {
         this._lccCallStatus(data);
         break;
 
+      case 'dial':
+        this._lccCallDial(data);
+        break;
+
+      case 'record':
+        this.notifyRecordOptions(data);
+        break;
+
       case 'mute:status':
         this._lccMuteStatus(call_sid, data);
         break;
@@ -1428,10 +1548,17 @@ class CallSession extends Emitter {
           });
         break;
 
+      case 'dtmf':
+        this._lccDtmf(data, call_sid)
+          .catch((err) => {
+            this.logger.info({err, data}, `CallSession:_onCommand - error sending RFC 2833 DTMF ${data}`);
+          });
+        break;
+
       default:
         this.logger.info(`CallSession:_onCommand - invalid command ${command}`);
     }
-    if (this.wakeupResolver) {
+    if (this.wakeupResolver && resolution) {
       //this.logger.debug({resolution}, 'CallSession:_onCommand - got commands, waking up..');
       this.wakeupResolver(resolution);
       this.wakeupResolver = null;
@@ -1612,6 +1739,8 @@ class CallSession extends Emitter {
     this.notifier && this.notifier.close();
 
     this.rootSpan && this.rootSpan.end();
+    // close all background tasks
+    this.backgroundTaskManager.stopAll();
   }
 
   /**
@@ -1697,7 +1826,7 @@ class CallSession extends Emitter {
           res.send(200, {body: this.ep.local.sdp});
         }
         else {
-          if (this.currentTask.name === TaskName.Dial && this.currentTask.isOnHold) {
+          if (this.currentTask.name === TaskName.Dial && this.currentTask.isOnHoldEnabled) {
             this.logger.info('onholdMusic reINVITE after media has been released');
             await this.currentTask.handleReinviteAfterMediaReleased(req, res);
           } else {
@@ -1868,7 +1997,7 @@ class CallSession extends Emitter {
   wrapDialog(dlg) {
     dlg.connectTime = moment();
     const origDestroy = dlg.destroy.bind(dlg);
-    dlg.destroy = () => {
+    dlg.destroy = (opts) => {
       if (dlg.connected) {
         dlg.connected = false;
         dlg.destroy = origDestroy;
@@ -1877,7 +2006,7 @@ class CallSession extends Emitter {
         this.emit('callStatusChange', {callStatus: CallStatus.Completed, duration});
         this.logger.debug('CallSession: call terminated by jambonz');
         this.rootSpan.setAttributes({'call.termination': 'hangup by jambonz'});
-        origDestroy().catch((err) => this.logger.info({err}, 'CallSession - error destroying dialog'));
+        origDestroy(opts).catch((err) => this.logger.info({err}, 'CallSession - error destroying dialog'));
         if (this.wakeupResolver) {
           this.wakeupResolver({reason: 'session ended'});
           this.wakeupResolver = null;
@@ -1928,12 +2057,14 @@ class CallSession extends Emitter {
   async _notifyCallStatusChange({callStatus, sipStatus, sipReason, duration}) {
     if (this.callMoved) return;
 
+    // manage record all call.
     if (callStatus === CallStatus.InProgress) {
-      // nice, call is in progress, good time to enable record
+      if (this.accountInfo.account.record_all_calls ||
-      await this.enableRecordAllCall();
+        this.application.record_all_calls) {
-    } else if (callStatus == CallStatus.Completed && this.isBackGroundListen) {
+        this.backgroundTaskManager.newTask('record');
-      this.stopBackgroundListen().catch((err) => this.logger.error(
+      }
-        {err}, 'CallSession:_notifyCallStatusChange - error stopping background listen'));
+    } else if (callStatus == CallStatus.Completed) {
+      this.backgroundTaskManager.stop('record');
     }
 
     /* race condition: we hang up at the same time as the caller */
@@ -1970,32 +2101,13 @@ class CallSession extends Emitter {
     }
   }
 
-  async enableRecordAllCall() {
-    if (this.accountInfo.account.record_all_calls || this.application.record_all_calls) {
-      if (!JAMBONZ_RECORD_WS_BASE_URL || !this.accountInfo.account.bucket_credential) {
-        this.logger.error('Record all calls: invalid configuration');
-        return;
-      }
-      const listenOpts = {
-        url: `${JAMBONZ_RECORD_WS_BASE_URL}/record/${this.accountInfo.account.bucket_credential.vendor}`,
-        disableBidirectionalAudio: true,
-        mixType : 'stereo',
-        passDtmf: true
-      };
-      if (JAMBONZ_RECORD_WS_USERNAME && JAMBONZ_RECORD_WS_PASSWORD) {
-        listenOpts.wsAuth = {
-          username: JAMBONZ_RECORD_WS_USERNAME,
-          password: JAMBONZ_RECORD_WS_PASSWORD
-        };
-      }
-      this.logger.debug({listenOpts}, 'Record all calls: enabling listen');
-      await this.startBackgroundListen({verb: 'listen', ...listenOpts}, 'jambonz-session-record');
-    }
-  }
-
   _configMsEndpoint() {
-    if (this.onHoldMusic) {
+    const opts = {
-      this.ep.set({hold_music: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`});
+      ...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
+      ...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'})
+    };
+    if (Object.keys(opts).length > 0) {
+      this.ep.set(opts);
     }
   }
 
@@ -2035,6 +2147,25 @@ class CallSession extends Emitter {
       } catch (err) {}
     }
   }
+
+  /**
+   * startBackgroundTask - Start background task
+   */
+
+  async startBackgroundTask(type, opts) {
+    await this.backgroundTaskManager.newTask(type, opts);
+  }
+
+  stopBackgroundTask(type) {
+    this.backgroundTaskManager.stop(type);
+  }
+
+  _endVerbHookSpan() {
+    if (this.verbHookSpan) {
+      this.verbHookSpan.end();
+      this.verbHookSpan = null;
+    }
+  }
 }
 
 module.exports = CallSession;

lib/tasks/config.js

@@ -9,7 +9,8 @@ class TaskConfig extends Task {
       'recognizer',
       'bargeIn',
       'record',
-      'listen'
+      'listen',
+      'transcribe'
     ].forEach((k) => this[k] = this.data[k] || {});
 
     if ('notifyEvents' in this.data) {
@@ -30,6 +31,13 @@ class TaskConfig extends Task {
         if (this.bargeIn[k]) this.gatherOpts[k] = this.bargeIn[k];
       });
     }
+    if (this.transcribe?.enable) {
+      this.transcribeOpts = {
+        verb: 'transcribe',
+        ...this.transcribe
+      };
+      delete this.transcribeOpts.enable;
+    }
 
     if (this.data.reset) {
       if (typeof this.data.reset === 'string') this.data.reset = [this.data.reset];
@@ -37,7 +45,11 @@ class TaskConfig extends Task {
     else this.data.reset = [];
 
     if (this.bargeIn.sticky) this.autoEnable = true;
-    this.preconditions = (this.bargeIn.enable || this.record?.action || this.listen?.url || this.data.amd) ?
+    this.preconditions = (this.bargeIn.enable ||
+      this.record?.action ||
+      this.listen?.url ||
+      this.data.amd ||
+      this.transcribe?.enable) ?
       TaskPreconditions.Endpoint :
       TaskPreconditions.None;
 
@@ -50,6 +62,7 @@ class TaskConfig extends Task {
   get hasRecognizer() { return Object.keys(this.recognizer).length; }
   get hasRecording() { return Object.keys(this.record).length; }
   get hasListen() { return Object.keys(this.listen).length; }
+  get hasTranscribe() { return Object.keys(this.transcribe).length; }
 
   get summary() {
     const phrase = [];
@@ -72,6 +85,9 @@ class TaskConfig extends Task {
     if (this.hasListen) {
       phrase.push(this.listen.enable ? `listen ${this.listen.url}` : 'stop listen');
     }
+    if (this.hasTranscribe) {
+      phrase.push(this.transcribe.enable ? `transcribe ${this.transcribe.transcriptionHook}` : 'stop transcribe');
+    }
     if (this.data.amd) phrase.push('enable amd');
     if (this.notifyEvents) phrase.push(`event notification ${this.notifyEvents ? 'on' : 'off'}`);
     if (this.onHoldMusic) phrase.push(`onHoldMusic: ${this.onHoldMusic}`);
@@ -109,6 +125,7 @@ class TaskConfig extends Task {
     });
 
     if (this.hasSynthesizer) {
+      cs.synthesizer = this.synthesizer;
       cs.speechSynthesisVendor = this.synthesizer.vendor !== 'default'
         ? this.synthesizer.vendor
         : cs.speechSynthesisVendor;
@@ -138,6 +155,7 @@ class TaskConfig extends Task {
       this.logger.info({synthesizer: this.synthesizer}, 'Config: updated synthesizer');
     }
     if (this.hasRecognizer) {
+      cs.recognizer = this.recognizer;
       cs.speechRecognizerVendor = this.recognizer.vendor !== 'default'
         ? this.recognizer.vendor
         : cs.speechRecognizerVendor;
@@ -210,10 +228,25 @@ class TaskConfig extends Task {
       const {enable, ...opts} = this.listen;
       if (enable) {
         this.logger.debug({opts}, 'Config: enabling listen');
-        cs.startBackgroundListen({verb: 'listen', ...opts});
+        cs.startBackgroundTask('listen', {verb: 'listen', ...opts});
       } else {
         this.logger.info('Config: disabling listen');
-        cs.stopBackgroundListen();
+        cs.stopBackgroundTask('listen');
+      }
+    }
+    if (this.hasTranscribe) {
+      if (this.transcribe.enable) {
+        this.transcribeOpts.recognizer = this.hasRecognizer ?
+          this.recognizer :
+          {
+            vendor: cs.speechRecognizerVendor,
+            language: cs.speechRecognizerLanguage
+          };
+        this.logger.debug(this.transcribeOpts, 'Config: enabling transcribe');
+        cs.startBackgroundTask('transcribe', this.transcribeOpts);
+      } else {
+        this.logger.info('Config: disabling transcribe');
+        cs.stopBackgroundTask('transcribe');
       }
     }
     if (this.data.sipRequestWithinDialogHook) {

lib/tasks/dequeue.js

@@ -73,7 +73,8 @@ class TaskDequeue extends Task {
         try {
           let url;
           if (this.callSid) {
-            url = await retrieveByPatternSortedSet(this.queueName, `*${this.callSid}`);
+            const r = await retrieveByPatternSortedSet(this.queueName, `*${this.callSid}`);
+            url = r[0];
           } else {
             url = await retrieveFromSortedSet(this.queueName);
           }

lib/tasks/dial.js

@@ -17,7 +17,7 @@ const dbUtils = require('../utils/db-utils');
 const debug = require('debug')('jambonz:feature-server');
 const {parseUri} = require('drachtio-srf');
 const {ANCHOR_MEDIA_ALWAYS, JAMBONZ_DISABLE_DIAL_PAI_HEADER} = require('../config');
-const { isOnhold } = require('../utils/sdp-utils');
+const { isOnhold, isOpusFirst } = require('../utils/sdp-utils');
 const { normalizeJambones } = require('@jambonz/verb-specifications');
 
 function parseDtmfOptions(logger, dtmfCapture) {
@@ -100,6 +100,7 @@ class TaskDial extends Task {
     this.referHook = this.data.referHook;
     this.dtmfHook = this.data.dtmfHook;
     this.proxy = this.data.proxy;
+    this.tag = this.data.tag;
 
     if (this.dtmfHook) {
       const {parentDtmfCollector, childDtmfCollector} = parseDtmfOptions(logger, this.data.dtmfCapture || {});
@@ -138,13 +139,14 @@ class TaskDial extends Task {
 
   get name() { return TaskName.Dial; }
 
-  get isOnHold() {
+  get isOnHoldEnabled() {
-    return this.isIncomingLegHold || this.isOutgoingLegHold;
+    return !!this.data.onHoldHook;
   }
 
   get canReleaseMedia() {
     const keepAnchor = this.data.anchorMedia ||
     this.cs.isBackGroundListen ||
+    this.cs.onHoldMusic ||
     ANCHOR_MEDIA_ALWAYS ||
     this.listenTask ||
     this.transcribeTask ||
@@ -323,7 +325,7 @@ class TaskDial extends Task {
         const by = parseUri(req.getParsedHeader('Referred-By').uri);
         this.logger.info({to}, 'refer to parsed');
         const json = await cs.requestor.request('verb:hook', this.referHook, {
-          ...callInfo,
+          ...(callInfo.toJSON()),
           refer_details: {
             sip_refer_to: req.get('Refer-To'),
             sip_referred_by: req.get('Referred-By'),
@@ -488,7 +490,8 @@ class TaskDial extends Task {
       headers: this.headers,
       proxy: `sip:${sbcAddress}`,
       callingNumber: this.callerId || req.callingNumber,
-      ...(this.callerName && {callingName: this.callerName})
+      ...(this.callerName && {callingName: this.callerName}),
+      opusFirst: isOpusFirst(this.cs.ep.remote.sdp)
     };
 
     const t = this.target.find((t) => t.type === 'teams');
@@ -569,7 +572,8 @@ class TaskDial extends Task {
           accountInfo: cs.accountInfo,
           rootSpan: cs.rootSpan,
           startSpan: this.startSpan.bind(this),
-          dialTask: this
+          dialTask: this,
+          onHoldMusic: this.cs.onHoldMusic
         });
         this.dials.set(sd.callSid, sd);
 
@@ -678,22 +682,43 @@ class TaskDial extends Task {
   async _onReinvite(req, res) {
     try {
       let isHandled = false;
-      if (this.cs.onHoldMusic) {
+      if (this.isOnHoldEnabled) {
-        if (isOnhold(req.body) && !this.epOther && !this.ep) {
+        if (isOnhold(req.body)) {
-          await this.cs.handleReinviteAfterMediaReleased(req, res);
+          this.logger.debug('Dial: _onReinvite receive hold Request');
-          // Onhold but media is already released
+          if (!this.epOther && !this.ep) {
-          // reconnect A Leg and Response B leg
+            this.logger.debug(`Dial: _onReinvite receive hold Request,
-          await this.reAnchorMedia(this.cs, this.sd);
+ media already released, reconnect media server`);
-          this.isOutgoingLegHold = true;
+            // update caller leg for new SDP from callee.
+            await this.cs.handleReinviteAfterMediaReleased(req, res);
+            // Freeswitch media is released, reconnect
+            await this.reAnchorMedia(this.cs, this.sd);
+            this.isOutgoingLegHold = true;
+          } else {
+            this.logger.debug('Dial: _onReinvite receive hold Request, update SDP');
+            const newSdp = await this.ep.modify(req.body);
+            res.send(200, {body: newSdp});
+          }
           isHandled = true;
-          this._onHoldHook();
+          // Media already connected, ask for onHoldHook
-        } else if (!isOnhold(req.body) && this.epOther && this.ep && this.isOutgoingLegHold && this.canReleaseMedia) {
+          this._onHoldHook(req);
-          // Offhold, time to release media
+        } else if (!isOnhold(req.body)) {
-          const newSdp = await this.ep.modify(req.body);
+          this.logger.debug('Dial: _onReinvite receive unhold Request');
-          await res.send(200, {body: newSdp});
+          if (this.epOther && this.ep && this.isOutgoingLegHold && this.canReleaseMedia) {
-          await this._releaseMedia(this.cs, this.sd);
+            this.logger.debug('Dial: _onReinvite receive unhold Request, release media');
+            // Offhold, time to release media
+            const newSdp = await this.ep.modify(req.body);
+            await res.send(200, {body: newSdp});
+            await this._releaseMedia(this.cs, this.sd);
+            this.isOutgoingLegHold = false;
+          } else {
+            this.logger.debug('Dial: _onReinvite receive unhold Request, update media server');
+            const newSdp = await this.ep.modify(req.body);
+            res.send(200, {body: newSdp});
+          }
+          if (this._onHoldSession) {
+            this._onHoldSession.kill();
+          }
           isHandled = true;
-          this.isOutgoingLegHold = false;
         }
       }
       if (!isHandled) {
@@ -790,9 +815,11 @@ class TaskDial extends Task {
     assert(cs.ep && sd.ep);
 
     try {
+      // Wait until we got new SDP from B leg to ofter to A Leg
       const aLegSdp = cs.ep.remote.sdp;
+      await sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp);
       const bLegSdp = sd.dlg.remote.sdp;
-      await Promise.all[sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp), cs.releaseMediaToSBC(bLegSdp)];
+      await cs.releaseMediaToSBC(bLegSdp);
       this.epOther = null;
       this.logger.info('Dial:_releaseMedia - successfully released media from freewitch');
     } catch (err) {
@@ -808,23 +835,34 @@ class TaskDial extends Task {
     this.epOther = cs.ep;
   }
 
+  // Handle RE-INVITE hold from caller leg.
   async handleReinviteAfterMediaReleased(req, res) {
     let isHandled = false;
-    if (isOnhold(req.body) && !this.epOther && !this.ep) {
+    if (this.isOnHoldEnabled) {
-      const sdp = await this.dlg.modify(req.body);
+      if (isOnhold(req.body)) {
-      res.send(200, {body: sdp});
+        if (!this.epOther && !this.ep) {
-      // Onhold but media is already released
+          // update callee leg for new SDP from caller.
-      await this.reAnchorMedia(this.cs, this.sd);
+          const sdp = await this.dlg.modify(req.body);
-      isHandled = true;
+          res.send(200, {body: sdp});
-      this.isIncomingLegHold = true;
+          // Onhold but media is already released, reconnect
-      this._onHoldHook();
+          await this.reAnchorMedia(this.cs, this.sd);
-    } else if (!isOnhold(req.body) && this.epOther && this.ep && this.isIncomingLegHold && this.canReleaseMedia) {
+          isHandled = true;
-      // Offhold, time to release media
+          this.isIncomingLegHold = true;
-      const newSdp = await this.epOther.modify(req.body);
+        }
-      await res.send(200, {body: newSdp});
+        this._onHoldHook(req);
-      await this._releaseMedia(this.cs, this.sd);
+      } else if (!isOnhold(req.body)) {
-      isHandled = true;
+        if (this.epOther && this.ep && this.isIncomingLegHold && this.canReleaseMedia) {
-      this.isIncomingLegHold = false;
+          // Offhold, time to release media
+          const newSdp = await this.epOther.modify(req.body);
+          await res.send(200, {body: newSdp});
+          await this._releaseMedia(this.cs, this.sd);
+          isHandled = true;
+        }
+        this.isIncomingLegHold = false;
+        if (this._onHoldSession) {
+          this._onHoldSession.kill();
+        }
+      }
     }
 
     if (!isHandled) {
@@ -843,7 +881,7 @@ class TaskDial extends Task {
       });
   }
 
-  async _onHoldHook(allowed = [TaskName.Play, TaskName.Say, TaskName.Pause]) {
+  async _onHoldHook(req, allowed = [TaskName.Play, TaskName.Say, TaskName.Pause]) {
     if (this.data.onHoldHook) {
       // send silence for keep Voice quality
       await this.epOther.play('silence_stream://500');
@@ -853,7 +891,13 @@ class TaskDial extends Task {
           const b3 = this.getTracingPropagation();
           const httpHeaders = b3 && {b3};
           const json = await this.cs.application.requestor.
-            request('verb:hook', this.data.onHoldHook, this.cs.callInfo.toJSON(), httpHeaders);
+            request('verb:hook', this.data.onHoldHook, {
+              ...this.cs.callInfo.toJSON(),
+              hold_detail: {
+                from: req.get('From'),
+                to: req.get('To')
+              }
+            }, httpHeaders);
           const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
           allowedTasks = tasks.filter((t) => allowed.includes(t.name));
           if (tasks.length !== allowedTasks.length) {
@@ -862,7 +906,7 @@ class TaskDial extends Task {
           }
           this.logger.debug(`DialTask:_onHoldHook: executing ${tasks.length} tasks`);
           if (tasks.length) {
-            this._playSession = new ConfirmCallSession({
+            this._onHoldSession = new ConfirmCallSession({
               logger: this.logger,
               application: this.cs.application,
               dlg: this.isIncomingLegHold ? this.dlg : this.cs.dlg,
@@ -872,12 +916,12 @@ class TaskDial extends Task {
               tasks,
               rootSpan: this.cs.rootSpan
             });
-            await this._playSession.exec();
+            await this._onHoldSession.exec();
-            this._playSession = null;
+            this._onHoldSession = null;
           }
         } catch (error) {
           this.logger.info(error, 'DialTask:_onHoldHook: failed retrieving waitHook');
-          this._playSession = null;
+          this._onHoldSession = null;
           break;
         }
       } while (allowedTasks && allowedTasks.length > 0 && !this.killed && this.isOnHold);

lib/tasks/dialogflow/index.js

@@ -58,13 +58,13 @@ class Dialogflow extends Task {
       this.vendor = this.data.tts.vendor || 'default';
       this.language = this.data.tts.language || 'default';
       this.voice = this.data.tts.voice || 'default';
-      this.speechSynthesisLabel = this.data.tts.label || 'default';
+      this.speechSynthesisLabel = this.data.tts.label;
 
       // fallback tts
       this.fallbackVendor = this.data.tts.fallbackVendor || 'default';
       this.fallbackLanguage = this.data.tts.fallbackLanguage || 'default';
       this.fallbackVoice = this.data.tts.fallbackLanguage || 'default';
-      this.fallbackLabel = this.data.tts.fallbackLabel || 'default';
+      this.fallbackLabel = this.data.tts.fallbackLabel;
     }
     this.bargein = this.data.bargein;
   }

lib/tasks/gather.js

@@ -30,6 +30,10 @@ class TaskGather extends SttTask {
       'speechTimeout', 'timeout', 'say', 'play'
     ].forEach((k) => this[k] = this.data[k]);
 
+    // gather default input is digits
+    if (!this.input) {
+      this.input = ['digits'];
+    }
     /* when collecting dtmf, bargein on dtmf is true unless explicitly set to false */
     if (this.dtmfBargein !== false  && this.input.includes('digits')) this.dtmfBargein = true;
 
@@ -71,6 +75,7 @@ class TaskGather extends SttTask {
     /* buffer speech for continuous asr */
     this._bufferedTranscripts = [];
     this.partialTranscriptsCount = 0;
+    this.bugname_prefix = 'gather_';
   }
 
   get name() { return TaskName.Gather; }
@@ -103,7 +108,7 @@ class TaskGather extends SttTask {
 
   async exec(cs, {ep}) {
     this.logger.debug({options: this.data}, 'Gather:exec');
-    await super.exec(cs);
+    await super.exec(cs, {ep});
     const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
 
     if (cs.hasGlobalSttHints && !this.maskGlobalSttHints) {
@@ -116,14 +121,6 @@ class TaskGather extends SttTask {
       this.logger.debug({hints: this.data.recognizer.hints, hintsBoost: this.data.recognizer.hintsBoost},
         'Gather:exec - applying global sttHints');
     }
-    if (cs.hasAltLanguages) {
-      this.data.recognizer.altLanguages = this.data.recognizer.altLanguages.concat(cs.altLanguages);
-      this.logger.debug({altLanguages: this.data.recognizer?.altLanguages},
-        'Gather:exec - applying altLanguages');
-    }
-    if (cs.hasGlobalSttPunctuation && !this.data.recognizer.punctuation) {
-      this.data.recognizer.punctuation = cs.globalSttPunctuation;
-    }
     if (!this.isContinuousAsr && cs.isContinuousAsr) {
       this.isContinuousAsr = true;
       this.asrTimeout = cs.asrTimeout * 1000;
@@ -141,59 +138,6 @@ class TaskGather extends SttTask {
       this.interim = true;
       this.logger.debug('Gather:exec - early hints match enabled');
     }
-
-    this.ep = ep;
-    if ('default' === this.vendor || !this.vendor) {
-      this.vendor = cs.speechRecognizerVendor;
-      if (this.data.recognizer) this.data.recognizer.vendor = this.vendor;
-    }
-    if ('default' === this.language || !this.language) {
-      this.language = cs.speechRecognizerLanguage;
-      if (this.data.recognizer) this.data.recognizer.language = this.language;
-    }
-    if ('default' === this.label || !this.label) {
-      this.label = cs.speechRecognizerLabel;
-      if (this.data.recognizer) this.data.recognizer.label = this.label;
-    }
-    // Fallback options
-    if ('default' === this.fallbackVendor || !this.fallbackVendor) {
-      this.fallbackVendor = cs.fallbackSpeechRecognizerVendor;
-      if (this.data.recognizer) this.data.recognizer.fallbackVendor = this.fallbackVendor;
-    }
-    if ('default' === this.fallbackLanguage || !this.fallbackLanguage) {
-      this.fallbackLanguage = cs.fallbackSpeechRecognizerLanguage;
-      if (this.data.recognizer) this.data.recognizer.fallbackLanguage = this.fallbackLanguage;
-    }
-    if ('default' === this.fallbackLabel || !this.fallbackLabel) {
-      this.fallbackLabel = cs.fallbackSpeechRecognizerLabel;
-      if (this.data.recognizer) this.data.recognizer.fallbackLabel = this.fallbackLabel;
-    }
-    if (!this.data.recognizer.vendor) {
-      this.data.recognizer.vendor = this.vendor;
-    }
-    if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
-      // By default, application saves cobalt model in language
-      this.data.recognizer.model = cs.speechRecognizerLanguage;
-    }
-
-    if (this.needsStt && !this.sttCredentials) {
-      try {
-        this.sttCredentials = await this._initSpeechCredentials(cs, this.vendor, this.label);
-      } catch (error) {
-        if (this.fallbackVendor && this.isHandledByPrimaryProvider) {
-          await this._fallback();
-        } else {
-          throw error;
-        }
-      }
-    }
-
-    /* when using cobalt model is required */
-    if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
-      this.notifyError({ msg: 'ASR error', details:'Cobalt requires a model to be specified'});
-      throw new Error('Cobalt requires a model to be specified');
-    }
-
     const startListening = async(cs, ep) => {
       this._startTimer();
       if (this.isContinuousAsr && 0 === this.timeout) this._startAsrTimer();
@@ -281,7 +225,7 @@ class TaskGather extends SttTask {
     } catch (err) {
       this.logger.error(err, 'TaskGather:exec error');
     }
-    this.removeSpeechListeners(ep);
+    this.removeCustomEventListeners();
   }
 
   kill(cs) {
@@ -319,6 +263,16 @@ class TaskGather extends SttTask {
       this._resolve('dtmf-terminator-key');
     }
     else if (this.input.includes('digits')) {
+      if (this.digitBuffer.length === 0 && this.needsStt) {
+        // DTMF is higher priority than STT.
+        this.removeCustomEventListeners();
+        ep.stopTranscription({
+          vendor: this.vendor,
+          bugname: this.bugname,
+        })
+          .catch((err) => this.logger.error({err},
+            ` Received DTMF, Error stopping transcription for vendor ${this.vendor}`));
+      }
       this.digitBuffer += evt.dtmf;
       const len = this.digitBuffer.length;
       if (len === this.numDigits || len === this.maxDigits) {
@@ -351,37 +305,41 @@ class TaskGather extends SttTask {
       if (this.data.recognizer?.deepgramOptions?.shortUtterance) this.shortUtterance = true;
     }
 
-    const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.data.recognizer);
+    const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.language, this.data.recognizer);
     switch (this.vendor) {
       case 'google':
-        this.bugname = 'google_transcribe';
+        this.bugname = `${this.bugname_prefix}google_transcribe`;
-        ep.addCustomEventListener(GoogleTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(
-        ep.addCustomEventListener(GoogleTranscriptionEvents.EndOfUtterance, this._onEndOfUtterance.bind(this, cs, ep));
+          ep, GoogleTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
-        ep.addCustomEventListener(GoogleTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
+        this.addCustomEventListener(
+          ep, GoogleTranscriptionEvents.EndOfUtterance, this._onEndOfUtterance.bind(this, cs, ep));
+        this.addCustomEventListener(
+          ep, GoogleTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
         break;
 
       case 'aws':
       case 'polly':
-        this.bugname = 'aws_transcribe';
+        this.bugname = `${this.bugname_prefix}aws_transcribe`;
-        ep.addCustomEventListener(AwsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(ep, AwsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
-        ep.addCustomEventListener(AwsTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
+        this.addCustomEventListener(ep, AwsTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
         break;
       case 'microsoft':
-        this.bugname = 'azure_transcribe';
+        this.bugname = `${this.bugname_prefix}azure_transcribe`;
-        ep.addCustomEventListener(AzureTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(
-        ep.addCustomEventListener(AzureTranscriptionEvents.NoSpeechDetected,
+          ep, AzureTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(ep, AzureTranscriptionEvents.NoSpeechDetected,
           this._onNoSpeechDetected.bind(this, cs, ep));
-        ep.addCustomEventListener(AzureTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
+        this.addCustomEventListener(ep, AzureTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
         break;
       case 'nuance':
-        this.bugname = 'nuance_transcribe';
+        this.bugname = `${this.bugname_prefix}nuance_transcribe`;
-        ep.addCustomEventListener(NuanceTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, NuanceTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep));
-        ep.addCustomEventListener(NuanceTranscriptionEvents.StartOfSpeech,
+        this.addCustomEventListener(ep, NuanceTranscriptionEvents.StartOfSpeech,
           this._onStartOfSpeech.bind(this, cs, ep));
-        ep.addCustomEventListener(NuanceTranscriptionEvents.TranscriptionComplete,
+        this.addCustomEventListener(ep, NuanceTranscriptionEvents.TranscriptionComplete,
           this._onTranscriptionComplete.bind(this, cs, ep));
-        ep.addCustomEventListener(NuanceTranscriptionEvents.VadDetected,
+        this.addCustomEventListener(ep, NuanceTranscriptionEvents.VadDetected,
           this._onVadDetected.bind(this, cs, ep));
 
         /* stall timers until prompt finishes playing */
@@ -391,24 +349,24 @@ class TaskGather extends SttTask {
         break;
 
       case 'deepgram':
-        this.bugname = 'deepgram_transcribe';
+        this.bugname = `${this.bugname_prefix}deepgram_transcribe`;
-        ep.addCustomEventListener(DeepgramTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(
-        ep.addCustomEventListener(DeepgramTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+          ep, DeepgramTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
-        ep.addCustomEventListener(DeepgramTranscriptionEvents.ConnectFailure,
+        this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, DeepgramTranscriptionEvents.ConnectFailure,
           this._onVendorConnectFailure.bind(this, cs, ep));
-
-        /* if app sets deepgramOptions.utteranceEndMs they essentially want continuous asr */
-        if (opts.DEEPGRAM_SPEECH_UTTERANCE_END_MS) this.isContinuousAsr = true;
         break;
 
       case 'soniox':
-        this.bugname = 'soniox_transcribe';
+        this.bugname = `${this.bugname_prefix}soniox_transcribe`;
-        ep.addCustomEventListener(SonioxTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(
+          ep, SonioxTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
         break;
 
       case 'cobalt':
-        this.bugname = 'cobalt_transcribe';
+        this.bugname = `${this.bugname_prefix}cobalt_transcribe`;
-        ep.addCustomEventListener(CobaltTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(
+          ep, CobaltTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
 
         /* cobalt doesnt have language, it has model, which is required */
         if (!this.data.recognizer.model) {
@@ -437,22 +395,22 @@ class TaskGather extends SttTask {
         break;
 
       case 'ibm':
-        this.bugname = 'ibm_transcribe';
+        this.bugname = `${this.bugname_prefix}ibm_transcribe`;
-        ep.addCustomEventListener(IbmTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+        this.addCustomEventListener(ep, IbmTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
-        ep.addCustomEventListener(IbmTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep, IbmTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
-        ep.addCustomEventListener(IbmTranscriptionEvents.ConnectFailure,
+        this.addCustomEventListener(ep, IbmTranscriptionEvents.ConnectFailure,
           this._onVendorConnectFailure.bind(this, cs, ep));
         break;
 
       case 'nvidia':
-        this.bugname = 'nvidia_transcribe';
+        this.bugname = `${this.bugname_prefix}nvidia_transcribe`;
-        ep.addCustomEventListener(NvidiaTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, NvidiaTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep));
-        ep.addCustomEventListener(NvidiaTranscriptionEvents.StartOfSpeech,
+        this.addCustomEventListener(ep, NvidiaTranscriptionEvents.StartOfSpeech,
           this._onStartOfSpeech.bind(this, cs, ep));
-        ep.addCustomEventListener(NvidiaTranscriptionEvents.TranscriptionComplete,
+        this.addCustomEventListener(ep, NvidiaTranscriptionEvents.TranscriptionComplete,
           this._onTranscriptionComplete.bind(this, cs, ep));
-        ep.addCustomEventListener(NvidiaTranscriptionEvents.VadDetected,
+        this.addCustomEventListener(ep, NvidiaTranscriptionEvents.VadDetected,
           this._onVadDetected.bind(this, cs, ep));
 
         /* I think nvidia has this (??) - stall timers until prompt finishes playing */
@@ -462,20 +420,22 @@ class TaskGather extends SttTask {
         break;
 
       case 'assemblyai':
-        this.bugname = 'assemblyai_transcribe';
+        this.bugname = `${this.bugname_prefix}assemblyai_transcribe`;
-        ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep));
-        ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(
-        ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
+          ep, AssemblyAiTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
-        ep.addCustomEventListener(AssemblyAiTranscriptionEvents.ConnectFailure,
+        this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
+        this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.ConnectFailure,
           this._onVendorConnectFailure.bind(this, cs, ep));
         break;
       default:
         if (this.vendor.startsWith('custom:')) {
-          this.bugname = `${this.vendor}_transcribe`;
+          this.bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
-          ep.addCustomEventListener(JambonzTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+          this.addCustomEventListener(
-          ep.addCustomEventListener(JambonzTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+            ep, JambonzTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
-          ep.addCustomEventListener(JambonzTranscriptionEvents.ConnectFailure,
+          this.addCustomEventListener(ep, JambonzTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+          this.addCustomEventListener(ep, JambonzTranscriptionEvents.ConnectFailure,
             this._onVendorConnectFailure.bind(this, cs, ep));
           break;
         }
@@ -487,7 +447,7 @@ class TaskGather extends SttTask {
     }
 
     /* common handler for all stt engine errors */
-    ep.addCustomEventListener(JambonzTranscriptionEvents.Error, this._onJambonzError.bind(this, cs, ep));
+    this.addCustomEventListener(ep, JambonzTranscriptionEvents.Error, this._onJambonzError.bind(this, cs, ep));
     await ep.set(opts)
       .catch((err) => this.logger.info(err, 'Error setting channel variables'));
   }
@@ -528,7 +488,10 @@ class TaskGather extends SttTask {
     this._clearTimer();
     this._timeoutTimer = setTimeout(() => {
       if (this.isContinuousAsr) this._startAsrTimer();
-      else this._resolve(this.digitBuffer.length >= this.minDigits ? 'dtmf-num-digits' : 'timeout');
+      if (this.interDigitTimer) return; // let the inter-digit timer complete
+      else {
+        this._resolve(this.digitBuffer.length >= this.minDigits ? 'dtmf-num-digits' : 'timeout');
+      }
     }, this.timeout);
   }
 
@@ -536,7 +499,9 @@ class TaskGather extends SttTask {
     if (this._timeoutTimer) {
       clearTimeout(this._timeoutTimer);
       this._timeoutTimer = null;
+      return true;
     }
+    return false;
   }
 
   _startAsrTimer() {
@@ -632,7 +597,8 @@ class TaskGather extends SttTask {
       return;
     }
 
-    evt = this.normalizeTranscription(evt, this.vendor, 1, this.language, this.shortUtterance);
+    evt = this.normalizeTranscription(evt, this.vendor, 1, this.language,
+      this.shortUtterance, this.data.recognizer.punctuation);
     if (evt.alternatives.length === 0) {
       this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, continue listening');
       return;
@@ -658,7 +624,9 @@ class TaskGather extends SttTask {
     if (evt.is_final) {
       if (evt.alternatives[0].transcript === '' && !this.callSession.callGone && !this.killed) {
         emptyTranscript = true;
-        if (finished === 'true' && ['microsoft', 'deepgram'].includes(this.vendor)) {
+        if (finished === 'true' &&
+          ['microsoft', 'deepgram'].includes(this.vendor) &&
+          this._bufferedTranscripts.length === 0) {
           this.logger.debug({evt}, 'TaskGather:_onTranscription - got empty transcript from old gather, disregarding');
           return;
         }
@@ -705,7 +673,7 @@ class TaskGather extends SttTask {
           this.logger.debug({evt, words, bufferedWords},
             'TaskGather:_onTranscription - final transcript but < min barge words');
           this._bufferedTranscripts.push(evt);
-          this._startTranscribing(ep);
+          if (!['soniox', 'aws', 'microsoft', 'deepgram'].includes(this.vendor)) this._startTranscribing(ep);
           return;
         }
         else {
@@ -718,14 +686,11 @@ class TaskGather extends SttTask {
           else if (this.vendor === 'deepgram') {
             /* compile transcripts into one */
             if (!emptyTranscript) this._bufferedTranscripts.push(evt);
-            if (this.data.recognizer?.deepgramOptions?.utteranceEndMs) {
+
-              this.logger.debug('TaskGather:_onTranscription - got speech_final waiting for UtteranceEnd event');
+            /* deepgram can send an empty and final transcript; only if we have any buffered should we resolve */
-              return;
+            if (this._bufferedTranscripts.length === 0) return;
-            }
-            this.logger.debug({evt}, 'TaskGather:_onTranscription - compiling deepgram transcripts');
             evt = this.consolidateTranscripts(this._bufferedTranscripts, 1, this.language);
             this._bufferedTranscripts = [];
-            this.logger.debug({evt}, 'TaskGather:_onTranscription - compiled deepgram transcripts');
           }
 
           /* here is where we return a final transcript */
@@ -734,8 +699,7 @@ class TaskGather extends SttTask {
       }
     }
     else {
-      this._clearTimer();
+      if (this._clearTimer()) this._startTimer();
-      this._startTimer();
       if (this.bargein && (words + bufferedWords) >= this.minBargeinWordCount) {
         if (!this.playComplete) {
           this.logger.debug({transcript: evt.alternatives[0].transcript}, 'killing audio due to speech');
@@ -801,7 +765,10 @@ class TaskGather extends SttTask {
   async _onJambonzError(cs, ep, evt) {
     this.logger.info({evt}, 'TaskGather:_onJambonzError');
     if (this.isHandledByPrimaryProvider && this.fallbackVendor) {
-      ep.stopTranscription({vendor: this.vendor})
+      ep.stopTranscription({
+        vendor: this.vendor,
+        bugname: this.bugname
+      })
         .catch((err) => this.logger.error({err}, `Error stopping transcription for primary vendor ${this.vendor}`));
       const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
       try {
@@ -868,6 +835,9 @@ class TaskGather extends SttTask {
     if (this.resolved) return;
 
     this.resolved = true;
+    // If bargin is false and ws application return ack to verb:hook
+    // the gather should not play any audio
+    this._killAudio(this.cs);
     // Clear dtmf event
     if (this.dtmfBargein) {
       this.ep.removeAllListeners('dtmf');
@@ -882,7 +852,10 @@ class TaskGather extends SttTask {
       'stt.result': JSON.stringify(evt)
     });
     if (this.needsStt && this.ep && this.ep.connected) {
-      this.ep.stopTranscription({vendor: this.vendor})
+      this.ep.stopTranscription({
+        vendor: this.vendor,
+        bugname: this.bugname
+      })
         .catch((err) => this.logger.error({err}, 'Error stopping transcription'));
     }
 

lib/tasks/listen.js

@@ -123,8 +123,6 @@ class TaskListen extends Task {
       ci,
       this.metadata);
     if (this.hook.auth) {
-      this.logger.debug({username: this.hook.auth.username, password: this.hook.auth.password},
-        'TaskListen:_startListening basic auth');
       await this.ep.set({
         'MOD_AUDIO_BASIC_AUTH_USERNAME': this.hook.auth.username,
         'MOD_AUDIO_BASIC_AUTH_PASSWORD': this.hook.auth.password

lib/tasks/rest_dial.js

@@ -16,6 +16,7 @@ class TaskRestDial extends Task {
     this.to = this.data.to;
     this.call_hook = this.data.call_hook;
     this.timeout = this.data.timeout || 60;
+    this.sipRequestWithinDialogHook = this.data.sipRequestWithinDialogHook;
 
     this.on('connect', this._onConnect.bind(this));
     this.on('callStatus', this._onCallStatus.bind(this));
@@ -64,7 +65,7 @@ class TaskRestDial extends Task {
     const cs = this.callSession;
     cs.setDialog(dlg);
     this.logger.debug('TaskRestDial:_onConnect - call connected');
-
+    if (this.sipRequestWithinDialogHook) this._initSipRequestWithinDialogHandler(cs, dlg);
     try {
       const b3 = this.getTracingPropagation();
       const httpHeaders = b3 && {b3};
@@ -142,6 +143,16 @@ class TaskRestDial extends Task {
         this.logger.error({err}, 'Rest:dial:_onAmdEvent - error calling actionHook');
       });
   }
+
+  _initSipRequestWithinDialogHandler(cs, dlg) {
+    cs.sipRequestWithinDialogHook = this.sipRequestWithinDialogHook;
+    dlg.on('info', this._onRequestWithinDialog.bind(this, cs));
+    dlg.on('message', this._onRequestWithinDialog.bind(this, cs));
+  }
+
+  async _onRequestWithinDialog(cs, req, res) {
+    cs._onRequestWithinDialog(req, res);
+  }
 }
 
 module.exports = TaskRestDial;

lib/tasks/say.js

@@ -23,6 +23,12 @@ const breakLengthyTextIfNeeded = (logger,  text) => {
   }
 };
 
+const parseTextFromSayString = (text) => {
+  const closingBraceIndex = text.indexOf('}');
+  if (closingBraceIndex === -1) return text;
+  return text.slice(closingBraceIndex + 1);
+};
+
 class TaskSay extends Task {
   constructor(logger, opts, parentTask) {
     super(logger, opts);
@@ -59,8 +65,8 @@ class TaskSay extends Task {
     }
   }
 
-  async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label}) {
+  async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label, preCache = false}) {
-    const {srf} = cs;
+    const {srf, accountSid:account_sid} = cs;
     const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, srf);
     const {writeAlerts, AlertType, stats} = srf.locals;
     const {synthAudio} = srf.locals.dbHelpers;
@@ -76,6 +82,8 @@ class TaskSay extends Task {
         voice = arr[1];
         model = arr[2];
       }
+    } else if (vendor === 'deepgram') {
+      model = voice;
     }
 
     /* allow for microsoft custom region voice and api_key to be specified as an override */
@@ -86,13 +94,26 @@ class TaskSay extends Task {
       credentials.api_key = this.options.apiKey || credentials.apiKey;
       credentials.region = this.options.region || credentials.region;
       voice = this.options.voice || voice;
+    } else if (vendor === 'elevenlabs') {
+      credentials = credentials || {};
+      credentials.model_id = this.options.model_id || credentials.model_id;
+      credentials.voice_settings = this.options.voice_settings || {};
+      credentials.optimize_streaming_latency = this.options.optimize_streaming_latency
+      || credentials.optimize_streaming_latency;
+      voice = this.options.voice_id || voice;
     }
 
-    this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
+    this.ep.set({
+      tts_engine: vendor,
+      tts_voice: voice,
+      cache_speech_handles: 1,
+    }).catch((err) => this.logger.info({err}, 'Error setting tts_engine on endpoint'));
+
+    if (!preCache) this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
     try {
       if (!credentials) {
         writeAlerts({
-          account_sid: cs.accountSid,
+          account_sid,
           alert_type: AlertType.TTS_NOT_PROVISIONED,
           vendor
         }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
@@ -111,14 +132,17 @@ class TaskSay extends Task {
         if (text.startsWith('silence_stream://')) return text;
 
         /* otel: trace time for tts */
-        const {span} = this.startChildSpan('tts-generation', {
+        if (!preCache)  {
-          'tts.vendor': vendor,
+          const {span} = this.startChildSpan('tts-generation', {
-          'tts.language': language,
+            'tts.vendor': vendor,
-          'tts.voice': voice
+            'tts.language': language,
-        });
+            'tts.voice': voice
+          });
+          this.otelSpan = span;
+        }
         try {
           const {filePath, servedFromCache, rtt} = await synthAudio(stats, {
-            account_sid: cs.accountSid,
+            account_sid,
             text,
             vendor,
             language,
@@ -127,30 +151,41 @@ class TaskSay extends Task {
             model,
             salt,
             credentials,
-            disableTtsCache : this.disableTtsCache
+            options: this.options,
+            disableTtsCache : this.disableTtsCache,
+            preCache
           });
-          this.logger.debug(`file ${filePath}, served from cache ${servedFromCache}`);
+          if (!filePath.startsWith('say:')) {
-          if (filePath) cs.trackTmpFile(filePath);
+            this.logger.debug(`file ${filePath}, served from cache ${servedFromCache}`);
-          if (!servedFromCache && !lastUpdated) {
+            if (filePath) cs.trackTmpFile(filePath);
-            lastUpdated = true;
+            if (this.otelSpan) {
-            updateSpeechCredentialLastUsed(credentials.speech_credential_sid)
+              this.otelSpan.setAttributes({'tts.cached': servedFromCache});
-              .catch(() => {/*already logged error */});
+              this.otelSpan.end();
+              this.otelSpan = null;
+            }
+            if (!servedFromCache && !lastUpdated) {
+              lastUpdated = true;
+              updateSpeechCredentialLastUsed(credentials.speech_credential_sid).catch(() => {/* logged error */});
+            }
+            if (!servedFromCache && rtt && !preCache) {
+              this.notifyStatus({
+                event: 'synthesized-audio',
+                vendor,
+                language,
+                characters: text.length,
+                elapsedTime: rtt
+              });
+            }
           }
-          span.setAttributes({'tts.cached': servedFromCache});
+          else {
-          span.end();
+            this.logger.debug('a streaming tts api will be used');
-          if (!servedFromCache && rtt) {
+            const modifiedPath = filePath.replace('say:{', `say:{session-uuid=${this.ep.uuid},`);
-            this.notifyStatus({
+            return modifiedPath;
-              event: 'synthesized-audio',
-              vendor,
-              language,
-              characters: text.length,
-              elapsedTime: rtt
-            });
           }
           return filePath;
         } catch (err) {
           this.logger.info({err}, 'Error synthesizing tts');
-          span.end();
+          if (this.otelSpan) this.otelSpan.end();
           writeAlerts({
             account_sid: cs.accountSid,
             alert_type: AlertType.TTS_FAILURE,
@@ -172,6 +207,11 @@ class TaskSay extends Task {
   }
 
   async exec(cs, {ep}) {
+    const {srf, accountSid:account_sid} = cs;
+    const {writeAlerts, AlertType} = srf.locals;
+    const {addFileToCache} = srf.locals.dbHelpers;
+    const engine = this.synthesizer.engine || 'standard';
+
     await super.exec(cs);
     this.ep = ep;
 
@@ -229,8 +269,48 @@ class TaskSay extends Task {
           await this.playToConfMember(this.ep, memberId, confName, confUuid, filepath[segment]);
         }
         else {
-          this.logger.debug(`Say:exec sending command to play file ${filepath[segment]}`);
+          if (filepath[segment].startsWith('say:{')) {
+            const arr = /^say:\{.*\}\s*(.*)$/.exec(filepath[segment]);
+            if (arr) this.logger.debug(`Say:exec sending streaming tts request: ${arr[1].substring(0, 64)}..`);
+          }
+          else this.logger.debug(`Say:exec sending  ${filepath[segment].substring(0, 64)}`);
+          this.ep.once('playback-start', (evt) => {
+            this.logger.debug({evt}, 'got playback-start');
+            if (this.otelSpan) {
+              this.logger.debug({evt}, 'got playback-start');
+              this._addStreamingTtsAttributes(this.otelSpan, evt);
+              this.otelSpan.end();
+              this.otelSpan = null;
+              if (evt.variable_tts_cache_filename) cs.trackTmpFile(evt.variable_tts_cache_filename);
+            }
+          });
+          this.ep.once('playback-stop', (evt) => {
+            this.logger.debug({evt}, 'got playback-stop');
+            if (evt.variable_tts_error) {
+              writeAlerts({
+                account_sid,
+                alert_type: AlertType.TTS_FAILURE,
+                vendor,
+                detail: evt.variable_tts_error
+              }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
+            }
+            if (evt.variable_tts_cache_filename) {
+              const text = parseTextFromSayString(this.text[segment]);
+              addFileToCache(evt.variable_tts_cache_filename, {
+                account_sid,
+                vendor,
+                language,
+                voice,
+                engine,
+                text
+              }).catch((err) => this.logger.info({err}, 'Error adding file to cache'));
+            }
+          });
           await ep.play(filepath[segment]);
+          if (filepath[segment].startsWith('say:{')) {
+            const arr = /^say:\{.*\}\s*(.*)$/.exec(filepath[segment]);
+            if (arr) this.logger.debug(`Say:exec complete playing streaming tts request: ${arr[1].substring(0, 64)}..`);
+          }
           this.logger.debug(`Say:exec completed play file ${filepath[segment]}`);
         }
         segment++;
@@ -251,8 +331,30 @@ class TaskSay extends Task {
         this.notifyStatus({event: 'kill-playback'});
         this.ep.api('uuid_break', this.ep.uuid);
       }
+      this.ep.removeAllListeners('playback-start');
+      this.ep.removeAllListeners('playback-stop');
     }
   }
+
+  _addStreamingTtsAttributes(span, evt) {
+    const attrs = {'tts.cached': false};
+    for (const [key, value] of Object.entries(evt)) {
+      if (key.startsWith('variable_tts_')) {
+        let newKey = key.substring('variable_tts_'.length)
+          .replace('elevenlabs_', 'elevenlabs.');
+        if (spanMapping[newKey]) newKey = spanMapping[newKey];
+        attrs[newKey] = value;
+      }
+    }
+    span.setAttributes(attrs);
+  }
 }
 
+const spanMapping = {
+  'elevenlabs.reported_latency_ms': 'elevenlabs.latency_ms',
+  'elevenlabs.request_id': 'elevenlabs.req_id',
+  'elevenlabs.history_item_id': 'elevenlabs.item_id',
+  'elevenlabs.optimize_streaming_latency': 'elevenlabs.optimization',
+};
+
 module.exports = TaskSay;

lib/tasks/stt-task.js

@@ -14,17 +14,15 @@ class SttTask extends Task {
     const {
       setChannelVarsForStt,
       normalizeTranscription,
-      removeSpeechListeners,
       setSpeechCredentialsAtRuntime,
       compileSonioxTranscripts,
       consolidateTranscripts
     } = require('../utils/transcription-utils')(logger);
     this.setChannelVarsForStt = setChannelVarsForStt;
     this.normalizeTranscription = normalizeTranscription;
-    this.removeSpeechListeners = removeSpeechListeners;
     this.compileSonioxTranscripts = compileSonioxTranscripts;
     this.consolidateTranscripts = consolidateTranscripts;
-
+    this.eventHandlers = [];
     this.isHandledByPrimaryProvider = true;
     if (this.data.recognizer) {
       const recognizer = this.data.recognizer;
@@ -49,11 +47,113 @@ class SttTask extends Task {
 
     /* buffer for soniox transcripts */
     this._sonioxTranscripts = [];
+    /*bug name prefix */
+    this.bugname_prefix = '';
 
   }
 
+  async exec(cs, {ep, ep2}) {
+    super.exec(cs);
+    this.ep = ep;
+    this.ep2 = ep2;
+    // copy all value from config verb to this object.
+    if (cs.recognizer) {
+      for (const k in cs.recognizer) {
+        if (Array.isArray(this.data.recognizer[k]) ||
+        Array.isArray(cs.recognizer[k])) {
+          this.data.recognizer[k] = [
+            ...this.data.recognizer[k],
+            ...cs.recognizer[k]
+          ];
+        } else if (typeof this.data.recognizer[k] === 'object' ||
+        typeof cs.recognizer[k] === 'object'
+        ) {
+          this.data.recognizer[k] = {
+            ...this.data.recognizer[k],
+            ...cs.recognizer[k]
+          };
+        } else {
+          this.data.recognizer[k] = cs.recognizer[k] || this.data.recognizer[k];
+        }
+      }
+    }
+    if ('default' === this.vendor || !this.vendor) {
+      this.vendor = cs.speechRecognizerVendor;
+      if (this.data.recognizer) this.data.recognizer.vendor = this.vendor;
+    }
+    if ('default' === this.language || !this.language) {
+      this.language = cs.speechRecognizerLanguage;
+      if (this.data.recognizer) this.data.recognizer.language = this.language;
+    }
+    if ('default' === this.label || !this.label) {
+      this.label = cs.speechRecognizerLabel;
+      if (this.data.recognizer) this.data.recognizer.label = this.label;
+    }
+    // Fallback options
+    if ('default' === this.fallbackVendor || !this.fallbackVendor) {
+      this.fallbackVendor = cs.fallbackSpeechRecognizerVendor;
+      if (this.data.recognizer) this.data.recognizer.fallbackVendor = this.fallbackVendor;
+    }
+    if ('default' === this.fallbackLanguage || !this.fallbackLanguage) {
+      this.fallbackLanguage = cs.fallbackSpeechRecognizerLanguage;
+      if (this.data.recognizer) this.data.recognizer.fallbackLanguage = this.fallbackLanguage;
+    }
+    if ('default' === this.fallbackLabel || !this.fallbackLabel) {
+      this.fallbackLabel = cs.fallbackSpeechRecognizerLabel;
+      if (this.data.recognizer) this.data.recognizer.fallbackLabel = this.fallbackLabel;
+    }
+    if (!this.data.recognizer.vendor) {
+      this.data.recognizer.vendor = this.vendor;
+    }
+    if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
+      // By default, application saves cobalt model in language
+      this.data.recognizer.model = cs.speechRecognizerLanguage;
+    }
+
+    if (
+      // not gather task, such as transcribe
+      (!this.input ||
+      // gather task with speech
+        this.input.includes('speech')) &&
+      !this.sttCredentials) {
+      try {
+        this.sttCredentials = await this._initSpeechCredentials(this.cs, this.vendor, this.label);
+      } catch (error) {
+        if (this.fallbackVendor && this.isHandledByPrimaryProvider) {
+          await this._fallback();
+        } else {
+          throw error;
+        }
+      }
+    }
+
+    /* when using cobalt model is required */
+    if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
+      this.notifyError({ msg: 'ASR error', details:'Cobalt requires a model to be specified'});
+      throw new Error('Cobalt requires a model to be specified');
+    }
+
+    if (cs.hasAltLanguages) {
+      this.data.recognizer.altLanguages = this.data.recognizer.altLanguages.concat(cs.altLanguages);
+      this.logger.debug({altLanguages: this.altLanguages},
+        'STT:exec - applying altLanguages');
+    }
+    if (cs.hasGlobalSttPunctuation && !this.data.recognizer.punctuation) {
+      this.data.recognizer.punctuation = cs.globalSttPunctuation;
+    }
+  }
+
+  addCustomEventListener(ep, event, handler) {
+    this.eventHandlers.push({ep, event, handler});
+    ep.addCustomEventListener(event, handler);
+  }
+
+  removeCustomEventListeners() {
+    this.eventHandlers.forEach((h) => h.ep.removeCustomEventListener(h.event, h.handler));
+  }
+
   async _initSpeechCredentials(cs, vendor, label) {
-    const {getNuanceAccessToken, getIbmAccessToken} = this.cs.srf.locals.dbHelpers;
+    const {getNuanceAccessToken, getIbmAccessToken} = cs.srf.locals.dbHelpers;
     let credentials = cs.getSpeechCredentials(vendor, 'stt', label);
 
     if (!credentials) {

lib/tasks/tag.js

@@ -12,7 +12,7 @@ class TaskTag extends Task {
   async exec(cs) {
     super.exec(cs);
     cs.callInfo.customerData = this.data;
-    //this.logger.debug({callInfo: cs.callInfo.toJSON()}, 'TaskTag:exec set customer data in callInfo');
+    this.logger.debug({customerData: cs.callInfo.customerData}, 'TaskTag:exec set customer data in callInfo');
   }
 }
 

lib/tasks/task.js

@@ -160,14 +160,24 @@ class Task extends Emitter {
       const httpHeaders = b3 && {b3};
       span.setAttributes({'http.body': JSON.stringify(params)});
       try {
+        if (this.id) params.verb_id = this.id;
         const json = await this.cs.requestor.request(type, this.actionHook, params, httpHeaders);
         span.setAttributes({'http.statusCode': 200});
-        span.end();
+        const isWsConnection = this.cs.requestor instanceof WsRequestor;
+        if (!isWsConnection || (expectResponse && json && Array.isArray(json) && json.length)) {
+          span.end();
+        } else {
+          /** we use this span to measure application response latency,
+           * and with websocket connections we generally get the application's response
+           * in a subsequent message from the far end, so we terminate the span when the
+           * first new set of verbs arrive after sending a transcript
+           * */
+          this.emit('VerbHookSpanWaitForEnd', {span});
+        }
         if (expectResponse && json && Array.isArray(json)) {
           const makeTask = require('./make_task');
           const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
           if (tasks && tasks.length > 0) {
-            this.logger.info({tasks: tasks}, `${this.name} replacing application with ${tasks.length} tasks`);
             this.callSession.replaceApplication(tasks);
           }
         }

lib/tasks/transcribe.js

@@ -33,19 +33,30 @@ class TaskTranscribe extends SttTask {
 
     this.childSpan = [null, null];
 
-    // Continuos asr timeout
+    // Continuous asr timeout
     this.asrTimeout = typeof this.data.recognizer.asrTimeout === 'number' ? this.data.recognizer.asrTimeout * 1000 : 0;
     if (this.asrTimeout > 0) {
       this.isContinuousAsr = true;
     }
     /* buffer speech for continuous asr */
     this._bufferedTranscripts = [];
+    this.bugname_prefix = 'transcribe_';
+    this.paused = false;
   }
 
   get name() { return TaskName.Transcribe; }
 
   async exec(cs, {ep, ep2}) {
-    super.exec(cs);
+    await super.exec(cs, {ep, ep2});
+
+    if (this.data.recognizer.vendor === 'nuance') {
+      this.data.recognizer.nuanceOptions = {
+        // by default, nuance STT will recognize only 1st utterance.
+        // enable multiple allow nuance detact all utterances
+        utteranceDetectionMode: 'multiple',
+        ...this.data.recognizer.nuanceOptions
+      };
+    }
     const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
 
     if (cs.hasGlobalSttHints) {
@@ -55,67 +66,6 @@ class TaskTranscribe extends SttTask {
       this.logger.debug({hints: this.data.recognizer.hints, hintsBoost: this.data.recognizer.hintsBoost},
         'Transcribe:exec - applying global sttHints');
     }
-    if (cs.hasAltLanguages) {
-      this.data.recognizer.altLanguages = this.data.recognizer.altLanguages.concat(cs.altLanguages);
-      this.logger.debug({altLanguages: this.altLanguages},
-        'Transcribe:exec - applying altLanguages');
-    }
-    if (cs.hasGlobalSttPunctuation && !this.data.recognizer.punctuation) {
-      this.data.recognizer.punctuation = cs.globalSttPunctuation;
-    }
-
-    this.ep = ep;
-    this.ep2 = ep2;
-    if ('default' === this.vendor || !this.vendor) {
-      this.vendor = cs.speechRecognizerVendor;
-      if (this.data.recognizer) this.data.recognizer.vendor = this.vendor;
-    }
-    if ('default' === this.language || !this.language) {
-      this.language = cs.speechRecognizerLanguage;
-      if (this.data.recognizer) this.data.recognizer.language = this.language;
-    }
-    if ('default' === this.label || !this.label) {
-      this.label = cs.speechRecognizerLabel;
-      if (this.data.recognizer) this.data.recognizer.label = this.label;
-    }
-    // Fallback options
-    if ('default' === this.fallbackVendor || !this.fallbackVendor) {
-      this.fallbackVendor = cs.fallbackSpeechRecognizerVendor;
-      if (this.data.recognizer) this.data.recognizer.fallbackVendor = this.fallbackVendor;
-    }
-    if ('default' === this.fallbackLanguage || !this.fallbackLanguage) {
-      this.fallbackLanguage = cs.fallbackSpeechRecognizerLanguage;
-      if (this.data.recognizer) this.data.recognizer.fallbackLanguage = this.fallbackLanguage;
-    }
-    if ('default' === this.fallbackLabel || !this.fallbackLabel) {
-      this.fallbackLabel = cs.fallbackSpeechRecognizerLabel;
-      if (this.data.recognizer) this.data.recognizer.fallbackLabel = this.fallbackLabel;
-    }
-    if (!this.data.recognizer.vendor) {
-      this.data.recognizer.vendor = this.vendor;
-    }
-    if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
-      // By default, application saves cobalt model in language
-      this.data.recognizer.model = cs.speechRecognizerLanguage;
-    }
-
-    if (!this.sttCredentials) {
-      try {
-        this.sttCredentials = await this._initSpeechCredentials(cs, this.vendor, this.label);
-      } catch (error) {
-        if (this.fallbackVendor && this.isHandledByPrimaryProvider) {
-          await this._fallback();
-        } else {
-          throw error;
-        }
-      }
-    }
-
-    /* when using cobalt model is required */
-    if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
-      this.notifyError({ msg: 'ASR error', details:'Cobalt requires a model to be specified'});
-      throw new Error('Cobalt requires a model to be specified');
-    }
 
     try {
       await this._startTranscribing(cs, ep, 1);
@@ -131,19 +81,22 @@ class TaskTranscribe extends SttTask {
       this.logger.info(err, 'TaskTranscribe:exec - error');
       this.parentTask && this.parentTask.emit('error', err);
     }
-    this.removeSpeechListeners(ep);
+    this.removeCustomEventListeners();
   }
 
   async _stopTranscription() {
     let stopTranscription = false;
     if (this.ep?.connected) {
       stopTranscription = true;
-      this.ep.stopTranscription({vendor: this.vendor})
+      this.ep.stopTranscription({
+        vendor: this.vendor,
+        bugname: this.bugname
+      })
         .catch((err) => this.logger.info(err, 'Error TaskTranscribe:kill'));
     }
     if (this.separateRecognitionPerChannel && this.ep2 && this.ep2.connected) {
       stopTranscription = true;
-      this.ep2.stopTranscription({vendor: this.vendor})
+      this.ep2.stopTranscription({vendor: this.vendor, bugname: this.bugname})
         .catch((err) => this.logger.info(err, 'Error TaskTranscribe:kill'));
     }
 
@@ -165,9 +118,11 @@ class TaskTranscribe extends SttTask {
       this.logger.info(`TaskTranscribe:updateTranscribe status ${status}`);
       switch (status) {
         case TranscribeStatus.Pause:
+          this.paused = true;
           await this._stopTranscription();
           break;
         case TranscribeStatus.Resume:
+          this.paused = false;
           await this._startTranscribing(this.cs, this.ep, 1);
           if (this.separateRecognitionPerChannel && this.ep2) {
             await this._startTranscribing(this.cs, this.ep2, 2);
@@ -186,51 +141,47 @@ class TaskTranscribe extends SttTask {
       if (this.isContinuousAsr) this._doContinuousAsrWithDeepgram(this.asrTimeout);
     }
 
-    const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.data.recognizer);
+    const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.language, this.data.recognizer);
     switch (this.vendor) {
       case 'google':
-        this.bugname = 'google_transcribe';
+        this.bugname = `${this.bugname_prefix}google_transcribe`;
-        ep.addCustomEventListener(GoogleTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, GoogleTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(GoogleTranscriptionEvents.NoAudioDetected,
+        this.addCustomEventListener(ep, GoogleTranscriptionEvents.NoAudioDetected,
           this._onNoAudio.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(GoogleTranscriptionEvents.MaxDurationExceeded,
+        this.addCustomEventListener(ep, GoogleTranscriptionEvents.MaxDurationExceeded,
           this._onMaxDurationExceeded.bind(this, cs, ep, channel));
         break;
 
       case 'aws':
       case 'polly':
-        this.bugname = 'aws_transcribe';
+        this.bugname = `${this.bugname_prefix}aws_transcribe`;
-        ep.addCustomEventListener(AwsTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, AwsTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(AwsTranscriptionEvents.NoAudioDetected,
+        this.addCustomEventListener(ep, AwsTranscriptionEvents.NoAudioDetected,
           this._onNoAudio.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(AwsTranscriptionEvents.MaxDurationExceeded,
+        this.addCustomEventListener(ep, AwsTranscriptionEvents.MaxDurationExceeded,
           this._onMaxDurationExceeded.bind(this, cs, ep, channel));
         break;
       case 'microsoft':
-        this.bugname = 'azure_transcribe';
+        this.bugname = `${this.bugname_prefix}azure_transcribe`;
-        ep.addCustomEventListener(AzureTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, AzureTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(AzureTranscriptionEvents.NoSpeechDetected,
+        this.addCustomEventListener(ep, AzureTranscriptionEvents.NoSpeechDetected,
           this._onNoAudio.bind(this, cs, ep, channel));
         break;
       case 'nuance':
-        this.bugname = 'nuance_transcribe';
+        this.bugname = `${this.bugname_prefix}nuance_transcribe`;
-        ep.addCustomEventListener(NuanceTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, NuanceTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(NuanceTranscriptionEvents.StartOfSpeech,
-          this._onStartOfSpeech.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(NuanceTranscriptionEvents.TranscriptionComplete,
-          this._onTranscriptionComplete.bind(this, cs, ep, channel));
         break;
       case 'deepgram':
-        this.bugname = 'deepgram_transcribe';
+        this.bugname = `${this.bugname_prefix}deepgram_transcribe`;
-        ep.addCustomEventListener(DeepgramTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(DeepgramTranscriptionEvents.Connect,
+        this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Connect,
           this._onVendorConnect.bind(this, cs, ep));
-        ep.addCustomEventListener(DeepgramTranscriptionEvents.ConnectFailure,
+        this.addCustomEventListener(ep, DeepgramTranscriptionEvents.ConnectFailure,
           this._onVendorConnectFailure.bind(this, cs, ep, channel));
 
         /* if app sets deepgramOptions.utteranceEndMs they essentially want continuous asr */
@@ -238,13 +189,13 @@ class TaskTranscribe extends SttTask {
 
         break;
       case 'soniox':
-        this.bugname = 'soniox_transcribe';
+        this.bugname = `${this.bugname_prefix}soniox_transcribe`;
-        ep.addCustomEventListener(SonioxTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, SonioxTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
         break;
       case 'cobalt':
-        this.bugname = 'cobalt_transcribe';
+        this.bugname = `${this.bugname_prefix}cobalt_transcribe`;
-        ep.addCustomEventListener(CobaltTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, CobaltTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
 
         /* cobalt doesnt have language, it has model, which is required */
@@ -273,45 +224,40 @@ class TaskTranscribe extends SttTask {
         break;
 
       case 'ibm':
-        this.bugname = 'ibm_transcribe';
+        this.bugname = `${this.bugname_prefix}ibm_transcribe`;
-        ep.addCustomEventListener(IbmTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, IbmTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(IbmTranscriptionEvents.Connect,
+        this.addCustomEventListener(ep, IbmTranscriptionEvents.Connect,
           this._onVendorConnect.bind(this, cs, ep));
-        ep.addCustomEventListener(IbmTranscriptionEvents.ConnectFailure,
+        this.addCustomEventListener(ep, IbmTranscriptionEvents.ConnectFailure,
           this._onVendorConnectFailure.bind(this, cs, ep, channel));
         break;
 
       case 'nvidia':
-        this.bugname = 'nvidia_transcribe';
+        this.bugname = `${this.bugname_prefix}nvidia_transcribe`;
-        ep.addCustomEventListener(NvidiaTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, NvidiaTranscriptionEvents.Transcription,
-          this._onTranscription.bind(this, cs, ep));
+          this._onTranscription.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(NvidiaTranscriptionEvents.StartOfSpeech,
-          this._onStartOfSpeech.bind(this, cs, ep));
-        ep.addCustomEventListener(NvidiaTranscriptionEvents.TranscriptionComplete,
-          this._onTranscriptionComplete.bind(this, cs, ep));
-        ep.addCustomEventListener(NvidiaTranscriptionEvents.VadDetected,
-          this._onVadDetected.bind(this, cs, ep));
         break;
 
       case 'assemblyai':
-        this.bugname = 'assemblyai_transcribe';
+        this.bugname = `${this.bugname_prefix}assemblyai_transcribe`;
-        ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Transcription,
+        this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.Transcription,
           this._onTranscription.bind(this, cs, ep, channel));
-        ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
+        this.addCustomEventListener(ep,
-        ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
+          AssemblyAiTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
-        ep.addCustomEventListener(AssemblyAiTranscriptionEvents.ConnectFailure,
+        this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
+        this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.ConnectFailure,
           this._onVendorConnectFailure.bind(this, cs, ep, channel));
         break;
 
       default:
         if (this.vendor.startsWith('custom:')) {
-          this.bugname = `${this.vendor}_transcribe`;
+          this.bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
-          ep.addCustomEventListener(JambonzTranscriptionEvents.Transcription,
+          this.addCustomEventListener(ep, JambonzTranscriptionEvents.Transcription,
             this._onTranscription.bind(this, cs, ep, channel));
-          ep.addCustomEventListener(JambonzTranscriptionEvents.Connect, this._onJambonzConnect.bind(this, cs, ep));
+          this.addCustomEventListener(ep, JambonzTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
-          ep.addCustomEventListener(JambonzTranscriptionEvents.ConnectFailure,
+          this.addCustomEventListener(ep, JambonzTranscriptionEvents.ConnectFailure,
-            this._onJambonzConnectFailure.bind(this, cs, ep));
+            this._onVendorConnectFailure.bind(this, cs, ep));
           break;
         }
         else {
@@ -322,7 +268,7 @@ class TaskTranscribe extends SttTask {
     }
 
     /* common handler for all stt engine errors */
-    ep.addCustomEventListener(JambonzTranscriptionEvents.Error, this._onJambonzError.bind(this, cs, ep));
+    this.addCustomEventListener(ep, JambonzTranscriptionEvents.Error, this._onJambonzError.bind(this, cs, ep));
     await ep.set(opts)
       .catch((err) => this.logger.info(err, 'Error setting channel variables'));
   }
@@ -337,6 +283,8 @@ class TaskTranscribe extends SttTask {
   }
 
   async _transcribe(ep) {
+    this.logger.debug(
+      `TaskTranscribe:_transcribe - starting transcription vendor ${this.vendor} bugname ${this.bugname}`);
     await ep.startTranscription({
       vendor: this.vendor,
       interim: this.interim ? true : false,
@@ -351,6 +299,10 @@ class TaskTranscribe extends SttTask {
     // make sure this is not a transcript from answering machine detection
     const bugname = fsEvent.getHeader('media-bugname');
     if (bugname && this.bugname !== bugname) return;
+    if (this.paused) {
+      this.logger.debug({evt}, 'TaskTranscribe:_onTranscription - paused, ignoring transcript');
+    }
+
 
     if (this.vendor === 'ibm' && evt?.state === 'listening') return;
 
@@ -369,7 +321,8 @@ class TaskTranscribe extends SttTask {
     }
     this.logger.debug({evt}, 'TaskTranscribe:_onTranscription - before normalization');
 
-    evt = this.normalizeTranscription(evt, this.vendor, channel, this.language);
+    evt = this.normalizeTranscription(evt, this.vendor, channel, this.language, undefined,
+      this.data.recognizer.punctuation);
     this.logger.debug({evt}, 'TaskTranscribe:_onTranscription');
     if (evt.alternatives.length === 0) {
       this.logger.info({evt}, 'TaskTranscribe:_onTranscription - got empty transcript, continue listening');
@@ -453,7 +406,8 @@ class TaskTranscribe extends SttTask {
   }
 
   _onNoAudio(cs, ep, channel) {
-    this.logger.debug(`TaskTranscribe:_onNoAudio restarting transcription on channel ${channel}`);
+    this.logger.debug(`TaskTranscribe:_onNoAudio on channel ${channel}`);
+    if (this.paused) return;
     if (this.childSpan[channel - 1] && this.childSpan[channel - 1].span) {
       this.childSpan[channel - 1].span.setAttributes({
         channel,
@@ -469,7 +423,8 @@ class TaskTranscribe extends SttTask {
   }
 
   _onMaxDurationExceeded(cs, ep, channel) {
-    this.logger.debug(`TaskTranscribe:_onMaxDurationExceeded restarting transcription on channel ${channel}`);
+    this.logger.debug(`TaskTranscribe:_onMaxDurationExceeded on channel ${channel}`);
+    if (this.paused) return;
     if (this.childSpan[channel - 1] && this.childSpan[channel - 1].span) {
       this.childSpan[channel - 1].span.setAttributes({
         channel,
@@ -494,8 +449,12 @@ class TaskTranscribe extends SttTask {
 
   async _onJambonzError(cs, _ep, evt) {
     this.logger.info({evt}, 'TaskTranscribe:_onJambonzError');
+    if (this.paused) return;
     if (this.isHandledByPrimaryProvider && this.fallbackVendor) {
-      _ep.stopTranscription({vendor: this.vendor})
+      _ep.stopTranscription({
+        vendor: this.vendor,
+        bugname: this.bugname
+      })
         .catch((err) => this.logger.error({err}, `Error stopping transcription for primary vendor ${this.vendor}`));
       const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
       try {

lib/utils/amd-utils.js

@@ -266,7 +266,7 @@ module.exports = (logger) => {
 
     /* set stt options */
     logger.info(`starting amd for vendor ${vendor} and language ${language}`);
-    const sttOpts = amd.setChannelVarsForStt({name: TaskName.Gather}, sttCredentials, {
+    const sttOpts = amd.setChannelVarsForStt({name: TaskName.Gather}, sttCredentials, language, {
       vendor,
       hints,
       enhancedModel: true,

lib/utils/background-task-manager.js

@@ -0,0 +1,196 @@
+const { normalizeJambones } = require('@jambonz/verb-specifications');
+const makeTask = require('../tasks/make_task');
+const { JAMBONZ_RECORD_WS_BASE_URL, JAMBONZ_RECORD_WS_USERNAME, JAMBONZ_RECORD_WS_PASSWORD } = require('../config');
+const Emitter = require('events');
+
+class BackgroundTaskManager extends Emitter {
+  constructor({cs, logger, rootSpan}) {
+    super();
+    this.tasks = new Map();
+    this.cs = cs;
+    this.logger = logger;
+    this.rootSpan = rootSpan;
+  }
+
+  isTaskRunning(type) {
+    return this.tasks.has(type);
+  }
+
+  getTask(type) {
+    if (this.tasks.has(type)) {
+      return this.tasks.get(type);
+    }
+  }
+
+  count() {
+    return this.tasks.size;
+  }
+
+  async newTask(type, taskOpts) {
+    this.logger.info({taskOpts}, `initiating Background task ${type}`);
+    if (this.tasks.has(type)) {
+      this.logger.info(`Background task ${type} is running, skiped`);
+      return;
+    }
+    let task;
+    switch (type) {
+      case 'listen':
+        task = await this._initListen(taskOpts);
+        break;
+      case 'bargeIn':
+        task = await this._initBargeIn(taskOpts);
+        break;
+      case 'record':
+        task = await this._initRecord();
+        break;
+      case 'transcribe':
+        task = await this._initTranscribe(taskOpts);
+        break;
+      default:
+        break;
+    }
+    if (task) {
+      this.tasks.set(type, task);
+    }
+    return task;
+  }
+
+  stop(type) {
+    const task = this.getTask(type);
+    if (task) {
+      this.logger.info(`stopping background task: ${type}`);
+      task.removeAllListeners();
+      task.span.end();
+      task.kill();
+      // Remove task from managed List
+      this.tasks.delete(type);
+    } else {
+      this.logger.debug(`stopping background task, ${type} is not running, skipped`);
+    }
+  }
+
+  stopAll() {
+    this.logger.debug('BackgroundTaskManager:stopAll');
+    for (const key of this.tasks.keys()) {
+      this.stop(key);
+    }
+  }
+
+  // Initiate Listen
+  async _initListen(opts, bugname = 'jambonz-background-listen', ignoreCustomerData = false, type = 'listen') {
+    let task;
+    try {
+      const t = normalizeJambones(this.logger, [opts]);
+      task = makeTask(this.logger, t[0]);
+      task.bugname = bugname;
+      task.ignoreCustomerData = ignoreCustomerData;
+      const resources = await this.cs._evaluatePreconditions(task);
+      const {span, ctx} = this.rootSpan.startChildSpan(`background-${type}:${task.summary}`);
+      task.span = span;
+      task.ctx = ctx;
+      task.exec(this.cs, resources)
+        .then(this._taskCompleted.bind(this, type, task))
+        .catch(this._taskError.bind(this, type, task));
+    } catch (err) {
+      this.logger.info({err, opts}, `BackgroundTaskManager:_initListen - Error creating ${bugname} task`);
+    }
+    return task;
+  }
+
+  // Initiate Gather
+  async _initBargeIn(opts) {
+    let task;
+    try {
+      const t = normalizeJambones(this.logger, [opts]);
+      task = makeTask(this.logger, t[0]);
+      task
+        .once('dtmf', this._bargeInTaskCompleted.bind(this))
+        .once('vad', this._bargeInTaskCompleted.bind(this))
+        .once('transcription', this._bargeInTaskCompleted.bind(this))
+        .once('timeout', this._bargeInTaskCompleted.bind(this));
+      const resources = await this.cs._evaluatePreconditions(task);
+      const {span, ctx} = this.rootSpan.startChildSpan(`background-bargeIn:${task.summary}`);
+      task.span = span;
+      task.ctx = ctx;
+      task.bugname_prefix = 'background_bargeIn_';
+      task.exec(this.cs, resources)
+        .then(() => {
+          this._taskCompleted('bargeIn', task);
+          if (task.sticky && !this.cs.callGone && !this.cs._stopping) {
+            this.logger.info('BackgroundTaskManager:_initBargeIn: restarting background bargeIn');
+            this.newTask('bargeIn', opts);
+          }
+          return;
+        })
+        .catch(this._taskError.bind(this, 'bargeIn', task));
+    } catch (err) {
+      this.logger.info(err, 'BackgroundTaskManager:_initGather - Error creating bargeIn task');
+    }
+    return task;
+  }
+
+  // Initiate Record
+  async _initRecord() {
+    if (this.cs.accountInfo.account.record_all_calls || this.cs.application.record_all_calls) {
+      if (!JAMBONZ_RECORD_WS_BASE_URL || !this.cs.accountInfo.account.bucket_credential) {
+        this.logger.error(`_initRecord: invalid configuration,
+ missing JAMBONZ_RECORD_WS_BASE_URL or bucket configuration`);
+        return undefined;
+      }
+      const listenOpts = {
+        url: `${JAMBONZ_RECORD_WS_BASE_URL}/record/${this.cs.accountInfo.account.bucket_credential.vendor}`,
+        disableBidirectionalAudio: true,
+        mixType : 'stereo',
+        passDtmf: true
+      };
+      if (JAMBONZ_RECORD_WS_USERNAME && JAMBONZ_RECORD_WS_PASSWORD) {
+        listenOpts.wsAuth = {
+          username: JAMBONZ_RECORD_WS_USERNAME,
+          password: JAMBONZ_RECORD_WS_PASSWORD
+        };
+      }
+      this.logger.debug({listenOpts}, '_initRecord: enabling listen');
+      return await this._initListen({verb: 'listen', ...listenOpts}, 'jambonz-session-record', true, 'record');
+    }
+  }
+
+  // Initiate Transcribe
+  async _initTranscribe(opts) {
+    let task;
+    try {
+      const t = normalizeJambones(this.logger, [opts]);
+      task = makeTask(this.logger, t[0]);
+      const resources = await this.cs._evaluatePreconditions(task);
+      const {span, ctx} = this.rootSpan.startChildSpan(`background-transcribe:${task.summary}`);
+      task.span = span;
+      task.ctx = ctx;
+      task.bugname_prefix = 'background_transcribe_';
+      task.exec(this.cs, resources)
+        .then(this._taskCompleted.bind(this, 'transcribe', task))
+        .catch(this._taskError.bind(this, 'transcribe', task));
+    } catch (err) {
+      this.logger.info(err, 'BackgroundTaskManager:_initTranscribe - Error creating transcribe task');
+    }
+    return task;
+  }
+
+  _taskCompleted(type, task) {
+    this.logger.debug({type, task}, 'BackgroundTaskManager:_taskCompleted: task completed');
+    task.removeAllListeners();
+    task.span.end();
+    this.tasks.delete(type);
+  }
+  _taskError(type, task, error) {
+    this.logger.info({type, task, error}, 'BackgroundTaskManager:_taskError: task Error');
+    task.removeAllListeners();
+    task.span.end();
+    this.tasks.delete(type);
+  }
+
+  _bargeInTaskCompleted(evt) {
+    this.logger.debug({evt}, 'BackgroundTaskManager:_bargeInTaskCompleted on event from background bargeIn');
+    this.emit('bargeIn-done', evt);
+  }
+}
+
+module.exports = BackgroundTaskManager;

lib/utils/db-utils.js

@@ -75,6 +75,8 @@ const speechMapper = (cred) => {
     else if ('deepgram' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
+      obj.deepgram_stt_uri = o.deepgram_stt_uri;
+      obj.deepgram_stt_use_tls = o.deepgram_stt_use_tls;
     }
     else if ('soniox' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
@@ -91,9 +93,14 @@ const speechMapper = (cred) => {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
       obj.model_id = o.model_id;
+      obj.options = o.options;
     } else if ('assemblyai' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.api_key = o.api_key;
+    } else if ('whisper' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = o.api_key;
+      obj.model_id = o.model_id;
     } else if (obj.vendor.startsWith('custom:')) {
       const o = JSON.parse(decrypt(credential));
       obj.auth_token = o.auth_token;

lib/utils/http-requestor.js

@@ -14,6 +14,7 @@ const {
   HTTP_PROXY_PORT,
   HTTP_PROXY_PROTOCOL,
   NODE_ENV,
+  HTTP_USER_AGENT_HEADER,
 } = require('../config');
 
 const toBase64 = (str) => Buffer.from(str || '', 'utf8').toString('base64');
@@ -116,6 +117,10 @@ class HttpRequestor extends BaseRequestor {
     const url = hook.url || hook;
     const method = hook.method || 'POST';
     let buf = '';
+    httpHeaders = {
+      ...httpHeaders,
+      ...(HTTP_USER_AGENT_HEADER && {'user-agent' : HTTP_USER_AGENT_HEADER})
+    };
 
     assert.ok(url, 'HttpRequestor:request url was not provided');
     assert.ok, (['GET', 'POST'].includes(method), `HttpRequestor:request method must be 'GET' or 'POST' not ${method}`);

lib/utils/install-srf-locals.js

@@ -8,9 +8,6 @@ const {
   JAMBONES_MYSQL_CONNECTION_LIMIT,
   JAMBONES_MYSQL_PORT,
   JAMBONES_FREESWITCH,
-  JAMBONES_REDIS_HOST,
-  JAMBONES_REDIS_PORT,
-  JAMBONES_REDIS_SENTINELS,
   SMPP_URL,
   JAMBONES_TIME_SERIES_HOST,
   JAMBONES_ESL_LISTEN_ADDRESS,
@@ -140,7 +137,8 @@ function installSrfLocals(srf, logger) {
     lookupTeamsByAccount,
     lookupAccountBySid,
     lookupAccountCapacitiesBySid,
-    lookupSmppGateways
+    lookupSmppGateways,
+    lookupClientByAccountAndUsername
   } = require('@jambonz/db-helpers')({
     host: JAMBONES_MYSQL_HOST,
     user: JAMBONES_MYSQL_USER,
@@ -174,19 +172,14 @@ function installSrfLocals(srf, logger) {
     retrieveByPatternSortedSet,
     sortedSetLength,
     sortedSetPositionByPattern
-  } = require('@jambonz/realtimedb-helpers')(JAMBONES_REDIS_SENTINELS || {
+  } = require('@jambonz/realtimedb-helpers')({}, logger, tracer);
-    host: JAMBONES_REDIS_HOST,
-    port: JAMBONES_REDIS_PORT || 6379
-  }, logger, tracer);
   const registrar = new Registrar(logger, client);
   const {
     synthAudio,
+    addFileToCache,
     getNuanceAccessToken,
     getIbmAccessToken,
-  } = require('@jambonz/speech-utils')(JAMBONES_REDIS_SENTINELS || {
+  } = require('@jambonz/speech-utils')({}, logger);
-    host: JAMBONES_REDIS_HOST,
-    port: JAMBONES_REDIS_PORT || 6379
-  }, logger, tracer);
   const {
     writeAlerts,
     AlertType
@@ -217,11 +210,13 @@ function installSrfLocals(srf, logger) {
       lookupAccountBySid,
       lookupAccountCapacitiesBySid,
       lookupSmppGateways,
+      lookupClientByAccountAndUsername,
       updateCallStatus,
       retrieveCall,
       listCalls,
       deleteCall,
       synthAudio,
+      addFileToCache,
       createHash,
       retrieveHash,
       deleteKey,

lib/utils/place-outdial.js

@@ -13,9 +13,17 @@ const moment = require('moment');
 const stripCodecs = require('./strip-ancillary-codecs');
 const RootSpan = require('./call-tracer');
 const uuidv4 = require('uuid-random');
+const HttpRequestor = require('./http-requestor');
+const WsRequestor = require('./ws-requestor');
+const {makeOpusFirst} = require('./sdp-utils');
+const listTaskNames = require('./summarize-tasks');
+const {
+  JAMBONES_USE_FREESWITCH_TIMER_FD
+} = require('../config');
 
 class SingleDialer extends Emitter {
-  constructor({logger, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask}) {
+  constructor({logger, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask,
+    onHoldMusic}) {
     super();
     assert(target.type);
 
@@ -38,6 +46,7 @@ class SingleDialer extends Emitter {
 
     this.callSid = uuidv4();
     this.dialTask = dialTask;
+    this.onHoldMusic = onHoldMusic;
 
     this.on('callStatusChange', this._notifyCallStatusChange.bind(this));
   }
@@ -76,7 +85,8 @@ class SingleDialer extends Emitter {
       ...(this.from.host && {'X-Preferred-From-Host': this.from.host}),
       'X-Jambonz-Routing': this.target.type,
       'X-Call-Sid': this.callSid,
-      ...(this.applicationSid && {'X-Application-Sid': this.applicationSid})
+      ...(this.applicationSid && {'X-Application-Sid': this.applicationSid}),
+      ...(this.target.proxy && {'X-SIP-Proxy': this.target.proxy})
     };
     if (srf.locals.fsUUID) {
       opts.headers = {
@@ -127,6 +137,7 @@ class SingleDialer extends Emitter {
       this.serviceUrl = srf.locals.serviceUrl;
 
       this.ep = await ms.createEndpoint();
+      this._configMsEndpoint();
       this.logger.debug(`SingleDialer:exec - created endpoint ${this.ep.uuid}`);
 
       /**
@@ -148,7 +159,7 @@ class SingleDialer extends Emitter {
 
       Object.assign(opts, {
         proxy: `sip:${this.sbcAddress}`,
-        localSdp: this.ep.local.sdp
+        localSdp: opts.opusFirst ? makeOpusFirst(this.ep.local.sdp) : this.ep.local.sdp
       });
       if (this.target.auth) opts.auth = this.target.auth;
       inviteSpan = this.startSpan('invite', {
@@ -176,6 +187,7 @@ class SingleDialer extends Emitter {
            *  (a) create a logger for this call
            */
           req.srf = srf;
+          this.req = req;
           this.callInfo = new CallInfo({
             direction: CallDirection.Outbound,
             parentCallInfo: this.parentCallInfo,
@@ -184,6 +196,10 @@ class SingleDialer extends Emitter {
             callSid: this.callSid,
             traceId: this.rootSpan.traceId
           });
+          if (this.dialTask && this.dialTask.tag !== null &&
+            typeof this.dialTask.tag === 'object' && !Array.isArray(this.dialTask.tag)) {
+            this.callInfo.customerData = this.dialTask.tag;
+          }
           this.logger = srf.locals.parentLogger.child({
             callSid: this.callSid,
             parentCallSid: this.parentCallInfo.callSid,
@@ -248,7 +264,7 @@ class SingleDialer extends Emitter {
         .on('modify', async(req, res) => {
           try {
             if (this.ep) {
-              if (this.dialTask && this.dialTask.isOnHold) {
+              if (this.dialTask && this.dialTask.isOnHoldEnabled) {
                 this.logger.info('dial is onhold, emit event');
                 this.emit('reinvite', req, res);
               } else {
@@ -315,6 +331,16 @@ class SingleDialer extends Emitter {
     }
   }
 
+  _configMsEndpoint() {
+    const opts = {
+      ...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
+      ...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'})
+    };
+    if (Object.keys(opts).length > 0) {
+      this.ep.set(opts);
+    }
+  }
+
   /**
    * Run an application on the call after answer, e.g. call screening.
    * Once the application completes in some fashion, emit an 'accepted' event
@@ -334,6 +360,7 @@ class SingleDialer extends Emitter {
       // verify it contains only allowed verbs
       const allowedTasks = tasks.filter((task) => {
         return [
+          TaskPreconditions.None,
           TaskPreconditions.StableCall,
           TaskPreconditions.Endpoint
         ].includes(task.preconditions);
@@ -365,7 +392,7 @@ class SingleDialer extends Emitter {
     }
   }
 
-  async doAdulting({logger, tasks, application}) {
+  async doAdulting({logger, tasks, application, call_hook_url}) {
     this.adulting = true;
     this.emit('adulting');
     if (this.ep) {
@@ -381,15 +408,50 @@ class SingleDialer extends Emitter {
     this.dlg.linkedSpanId = this.rootSpan.traceId;
     const rootSpan = new RootSpan('outbound-call', this.dlg);
     const newLogger = logger.child({traceId: rootSpan.traceId});
+    //clone application from parent call with new requestor
+    //parrent application will be closed in case the parent hangup
+    const app = {...application};
+    if (call_hook_url) {
+      app.call_hook.url = call_hook_url;
+    }
+    if ('WS' === app.call_hook?.method ||
+      app.call_hook?.url.startsWith('ws://') || app.call_hook?.url.startsWith('wss://')) {
+      const requestor = new WsRequestor(logger, this.accountInfo.account.account_sid,
+        app.call_hook, this.accountInfo.account.webhook_secret);
+      app.requestor = requestor;
+      app.notifier = requestor;
+      app.call_hook.method = 'WS';
+    }
+    else {
+      app.requestor = new HttpRequestor(logger, this.accountInfo.account.account_sid,
+        app.call_hook, this.accountInfo.account.webhook_secret);
+      if (app.call_status_hook) app.notifier = new HttpRequestor(logger,
+        this.accountInfo.account.account_sid, app.call_status_hook,
+        this.accountInfo.account.webhook_secret);
+      else app.notifier = {request: () => {}, close: () => {}};
+    }
+    // Time to open session:new for adulting call
+    // If ws is used, open session:new to control the call later.
+    if (!tasks || tasks.length === 0 || app.requestor instanceof WsRequestor) {
+      const b3 = rootSpan?.getTracingPropagation();
+      const httpHeaders = b3 && {b3};
+      const newTask = await app.requestor.request(
+        'session:new', call_hook_url, this.callInfo.toJSON(), httpHeaders);
+      tasks = normalizeJambones(newLogger, newTask).map((tdata) => makeTask(newLogger, tdata));
+      newLogger.info({tasks: listTaskNames(tasks)}, 'SingleDialer:doAdulting new task list for adulting call');
+    }
+    // Replace old application with new application.
+    this.application = app;
     const cs = new AdultingCallSession({
       logger: newLogger,
       singleDialer: this,
-      application,
+      application: app,
       callInfo: this.callInfo,
       accountInfo: this.accountInfo,
       tasks,
       rootSpan
     });
+    cs.req = this.req;
     cs.exec().catch((err) => newLogger.error({err}, 'doAdulting: error executing session'));
     return cs;
   }
@@ -410,6 +472,7 @@ class SingleDialer extends Emitter {
   async reAnchorMedia() {
     assert(this.dlg && this.dlg.connected && !this.ep);
     this.ep = await this.ms.createEndpoint({remoteSdp: this.dlg.remote.sdp});
+    this._configMsEndpoint();
     await this.dlg.modify(this.ep.local.sdp, {
       headers: {
         'X-Reason': 'anchor-media'
@@ -440,11 +503,12 @@ class SingleDialer extends Emitter {
 }
 
 function placeOutdial({
-  logger, srf, ms, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask
+  logger, srf, ms, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask,
+  onHoldMusic
 }) {
   const myOpts = deepcopy(opts);
   const sd = new SingleDialer({
-    logger, sbcAddress, target, myOpts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask
+    logger, sbcAddress, target, myOpts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask, onHoldMusic
   });
   sd.exec(srf, ms, myOpts);
   return sd;

lib/utils/sdp-utils.js

@@ -12,6 +12,30 @@ const mergeSdpMedia = (sdp1, sdp2) => {
   return sdpTransform.write(parsedSdp1);
 };
 
+const getCodecPlacement = (parsedSdp, codec) => parsedSdp?.media[0]?.rtp?.findIndex((e) => e.codec === codec);
+
+const isOpusFirst = (sdp) => {
+  return getCodecPlacement(sdpTransform.parse(sdp), 'opus') === 0;
+};
+
+const makeOpusFirst = (sdp) => {
+  const parsedSdp = sdpTransform.parse(sdp);
+  // Find the index of the OPUS codec
+  const opusIndex = getCodecPlacement(parsedSdp, 'opus');
+
+  // Move OPUS codec to the beginning
+  if (opusIndex > 0) {
+    const opusEntry = parsedSdp.media[0].rtp.splice(opusIndex, 1)[0];
+    parsedSdp.media[0].rtp.unshift(opusEntry);
+
+    // Also move the corresponding payload type in the "m" line
+    const opusPayloadType = parsedSdp.media[0].payloads.split(' ')[opusIndex];
+    const otherPayloadTypes = parsedSdp.media[0].payloads.split(' ').filter((pt) => pt != opusPayloadType);
+    parsedSdp.media[0].payloads = [opusPayloadType, ...otherPayloadTypes].join(' ');
+  }
+  return sdpTransform.write(parsedSdp);
+};
+
 const extractSdpMedia = (sdp) => {
   const parsedSdp1 = sdpTransform.parse(sdp);
   if (parsedSdp1.media.length > 1) {
@@ -28,5 +52,7 @@ const extractSdpMedia = (sdp) => {
 module.exports = {
   isOnhold,
   mergeSdpMedia,
-  extractSdpMedia
+  extractSdpMedia,
+  isOpusFirst,
+  makeOpusFirst
 };

lib/utils/siprec-utils.js

@@ -97,8 +97,12 @@ const parseSiprecPayload = (req, logger) => {
             obj[`${prefix}participantstreamassoc`].forEach((ps) => {
               const part = participants[ps.$.participant_id];
               if (part) {
-                part.send = ps[`${prefix}send`][0];
+                if (ps.hasOwnProperty(`${prefix}send`)) {
-                part.recv = ps[`${prefix}recv`][0];
+                  part.send = ps[`${prefix}send`][0];
+                }
+                if (ps.hasOwnProperty(`${prefix}recv`)) {
+                  part.recv = ps[`${prefix}recv`][0];
+                }
               }
             });
           }
@@ -109,9 +113,9 @@ const parseSiprecPayload = (req, logger) => {
           obj[`${prefix}stream`].forEach((s) => {
             const streamId = s.$.stream_id;
             let sender;
-            for (const [k, v] of Object.entries(participants)) {
+            for (const v of Object.values(participants)) {
               if (v.send === streamId) {
-                sender = k;
+                sender = v;
                 break;
               }
             }
@@ -121,9 +125,15 @@ const parseSiprecPayload = (req, logger) => {
 
             sender.label = s[`${prefix}label`][0];
 
-            if (-1 !== ['1', 'a_leg', 'inbound'].indexOf(sender.label)) {
+            if (-1 !== ['1', 'a_leg', 'inbound', '10'].indexOf(sender.label)) {
-              opts.caller.aor = sender.aor ;
+              opts.caller.aor = sender.aor;
               if (sender.name) opts.caller.name = sender.name;
+              // Remap the sdp stream base on sender label
+              if (!opts.sdp1.includes(`a=label:${sender.label}`)) {
+                const tmp = opts.sdp1;
+                opts.sdp1 = opts.sdp2;
+                opts.sdp2 = tmp;
+              }
             }
             else {
               opts.callee.aor = sender.aor ;

lib/utils/transcription-utils.js

@@ -1,15 +1,5 @@
 const {
   TaskName,
-  AzureTranscriptionEvents,
-  GoogleTranscriptionEvents,
-  AwsTranscriptionEvents,
-  NuanceTranscriptionEvents,
-  DeepgramTranscriptionEvents,
-  SonioxTranscriptionEvents,
-  NvidiaTranscriptionEvents,
-  CobaltTranscriptionEvents,
-  JambonzTranscriptionEvents,
-  AssemblyAiTranscriptionEvents
 } = require('./constants.json');
 
 const stickyVars = {
@@ -112,6 +102,54 @@ const stickyVars = {
   ]
 };
 
+/**
+ * @see https://developers.deepgram.com/docs/models-languages-overview
+ */
+const optimalDeepramModels = {
+  zh: ['base', 'base'],
+  'zh-CN':['base', 'base'],
+  'zh-TW': ['base', 'base'],
+  da: ['enhanced', 'enhanced'],
+  en: ['nova-2-phonecall', 'nova-2'],
+  'en-US': ['nova-2-phonecall', 'nova-2'],
+  'en-AU': ['nova-2', 'nova-2'],
+  'en-GB': ['nova-2', 'nova-2'],
+  'en-IN': ['nova-2', 'nova-2'],
+  'en-NZ': ['nova-2', 'nova-2'],
+  nl: ['nova-2', 'nova-2'],
+  fr: ['nova-2', 'nova-2'],
+  'fr-CA': ['nova-2', 'nova-2'],
+  de: ['nova-2', 'nova-2'],
+  hi: ['nova-2', 'nova-2'],
+  'hi-Latn': ['nova-2', 'nova-2'],
+  id: ['base', 'base'],
+  it: ['nova-2', 'nova-2'],
+  ja: ['enhanced', 'enhanced'],
+  ko: ['nova-2', 'nova-2'],
+  no: ['nova-2', 'nova-2'],
+  pl: ['nova-2', 'nova-2'],
+  pt: ['nova-2', 'nova-2'],
+  'pt-BR': ['nova-2', 'nova-2'],
+  'pt-PT': ['nova-2', 'nova-2'],
+  ru: ['nova-2', 'nova-2'],
+  es: ['nova-2', 'nova-2'],
+  'es-419': ['nova-2', 'nova-2'],
+  'es-LATAM': ['enhanced', 'enhanced'],
+  sv: ['nova-2', 'nova-2'],
+  ta: ['enhanced', 'enhanced'],
+  taq: ['enhanced', 'enhanced'],
+  tr: ['nova-2', 'nova-2'],
+  uk: ['nova-2', 'nova-2']
+};
+
+const selectDefaultDeepgramModel = (task, language) => {
+  if (language in optimalDeepramModels) {
+    const [gather, transcribe] = optimalDeepramModels[language];
+    return task.name === TaskName.Gather ? gather : transcribe;
+  }
+  return 'base';
+};
+
 const consolidateTranscripts = (bufferedTranscripts, channel, language) => {
   if (bufferedTranscripts.length === 1) return bufferedTranscripts[0];
   let totalConfidence = 0;
@@ -338,19 +376,20 @@ const normalizeNuance = (evt, channel, language) => {
   };
 };
 
-const normalizeMicrosoft = (evt, channel, language) => {
+const normalizeMicrosoft = (evt, channel, language, punctuation = true) => {
   const copy = JSON.parse(JSON.stringify(evt));
   const nbest = evt.NBest;
   const language_code = evt.PrimaryLanguage?.Language || language;
   const alternatives = nbest ? nbest.map((n) => {
     return {
       confidence: n.Confidence,
-      transcript: n.Display
+      // remove all puntuation if needed
+      transcript: punctuation ? n.Display : n.Display.replace(/\p{P}/gu, '')
     };
   }) :
     [
       {
-        transcript: evt.DisplayText || evt.Text
+        transcript: punctuation ? evt.DisplayText || evt.Text : (evt.DisplayText || evt.Text).replace(/\p{P}/gu, '')
       }
     ];
 
@@ -400,14 +439,14 @@ const normalizeAssemblyAi = (evt, channel, language) => {
 };
 
 module.exports = (logger) => {
-  const normalizeTranscription = (evt, vendor, channel, language, shortUtterance) => {
+  const normalizeTranscription = (evt, vendor, channel, language, shortUtterance, punctuation) => {
 
     //logger.debug({ evt, vendor, channel, language }, 'normalizeTranscription');
     switch (vendor) {
       case 'deepgram':
         return normalizeDeepgram(evt, channel, language, shortUtterance);
       case 'microsoft':
-        return normalizeMicrosoft(evt, channel, language);
+        return normalizeMicrosoft(evt, channel, language, punctuation);
       case 'google':
         return normalizeGoogle(evt, channel, language);
       case 'aws':
@@ -433,7 +472,7 @@ module.exports = (logger) => {
     }
   };
 
-  const setChannelVarsForStt = (task, sttCredentials, rOpts = {}) => {
+  const setChannelVarsForStt = (task, sttCredentials, language, rOpts = {}) => {
     let opts = {};
     const {enable, voiceMs = 0, mode = -1} = rOpts.vad || {};
     const vad = {enable, voiceMs, mode};
@@ -473,7 +512,8 @@ module.exports = (logger) => {
         ...(rOpts.hints?.length > 0 && typeof rOpts.hints[0] === 'object' &&
           {GOOGLE_SPEECH_HINTS: JSON.stringify(rOpts.hints)}),
         ...(typeof rOpts.hintsBoost === 'number' && {GOOGLE_SPEECH_HINTS_BOOST: rOpts.hintsBoost}),
-        ...(rOpts.altLanguages?.length > 0 &&
+        // When altLanguages is emptylist, we have to send value to freeswitch to clear the previous settings
+        ...(rOpts.altLanguages &&
           {GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES: [...new Set(rOpts.altLanguages)].join(',')}),
         ...(rOpts.interactionType &&
           {GOOGLE_SPEECH_METADATA_INTERACTION_TYPE: rOpts.interactionType}),
@@ -503,7 +543,8 @@ module.exports = (logger) => {
           {AZURE_SPEECH_HINTS: rOpts.hints.map((h) => h.trim()).join(',')}),
         ...(rOpts.hints?.length > 0 && typeof rOpts.hints[0] === 'object' &&
           {AZURE_SPEECH_HINTS: rOpts.hints.map((h) => h.phrase).join(',')}),
-        ...(rOpts.altLanguages && rOpts.altLanguages.length > 0 &&
+        // When altLanguages is emptylist, we have to send value to freeswitch to clear the previous settings
+        ...(rOpts.altLanguages &&
           {AZURE_SPEECH_ALTERNATIVE_LANGUAGE_CODES: [...new Set(rOpts.altLanguages)].join(',')}),
         ...(rOpts.requestSnr && {AZURE_REQUEST_SNR: 1}),
         ...(rOpts.profanityOption && {AZURE_PROFANITY_OPTION: rOpts.profanityOption}),
@@ -523,6 +564,9 @@ module.exports = (logger) => {
         }),
         ...(sttCredentials.use_custom_stt && sttCredentials.custom_stt_endpoint &&
           {AZURE_SERVICE_ENDPOINT_ID: sttCredentials.custom_stt_endpoint}),
+        //azureSttEndpointId overrides sttCredentials.custom_stt_endpoint
+        ...(rOpts.azureSttEndpointId &&
+          {AZURE_SERVICE_ENDPOINT_ID: rOpts.azureSttEndpointId}),
       };
     }
     else if ('nuance' === vendor) {
@@ -574,15 +618,24 @@ module.exports = (logger) => {
       };
     }
     else if ('deepgram' === vendor) {
+      let {model} = rOpts;
       const {deepgramOptions = {}} = rOpts;
+      const deepgramUri = deepgramOptions.deepgramSttUri || sttCredentials.deepgram_stt_uri;
+      const useTls = deepgramOptions.deepgramSttUseTls || sttCredentials.deepgram_stt_use_tls;
+
+      /* default to a sensible model if not supplied */
+      if (!model) {
+        model = selectDefaultDeepgramModel(task, language);
+      }
       opts = {
         ...opts,
+        DEEPGRAM_SPEECH_MODEL: model,
+        ...(deepgramUri && {DEEPGRAM_URI: deepgramUri}),
+        ...(deepgramUri && useTls && {DEEPGRAM_USE_TLS: 1}),
         ...(sttCredentials.api_key) &&
           {DEEPGRAM_API_KEY: sttCredentials.api_key},
         ...(deepgramOptions.tier) &&
           {DEEPGRAM_SPEECH_TIER: deepgramOptions.tier},
-        ...(deepgramOptions.model) &&
-          {DEEPGRAM_SPEECH_MODEL: deepgramOptions.model},
         ...(deepgramOptions.punctuate) &&
           {DEEPGRAM_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION: 1},
         ...(deepgramOptions.smartFormatting) &&
@@ -612,7 +665,7 @@ module.exports = (logger) => {
         ...(deepgramOptions.keywords) &&
           {DEEPGRAM_SPEECH_KEYWORDS: deepgramOptions.keywords.join(',')},
         ...('endpointing' in deepgramOptions) &&
-          {DEEPGRAM_SPEECH_ENDPOINTING: deepgramOptions.endpointing},
+          {DEEPGRAM_SPEECH_ENDPOINTING: deepgramOptions.endpointing === false ? 'false' : deepgramOptions.endpointing},
         ...(deepgramOptions.utteranceEndMs) &&
           {DEEPGRAM_SPEECH_UTTERANCE_END_MS: deepgramOptions.utteranceEndMs},
         ...(deepgramOptions.vadTurnoff) &&
@@ -740,7 +793,7 @@ module.exports = (logger) => {
 
       opts = {
         ...opts,
-        JAMBONZ_STT_API_KEY: auth_token,
+        ...(auth_token && {JAMBONZ_STT_API_KEY: auth_token}),
         JAMBONZ_STT_URL: custom_stt_url,
         ...(Object.keys(options).length > 0 && {JAMBONZ_STT_OPTIONS: JSON.stringify(options)}),
       };
@@ -752,48 +805,6 @@ module.exports = (logger) => {
     return opts;
   };
 
-  const removeSpeechListeners = (ep) => {
-    ep.removeCustomEventListener(GoogleTranscriptionEvents.Transcription);
-    ep.removeCustomEventListener(GoogleTranscriptionEvents.EndOfUtterance);
-    ep.removeCustomEventListener(GoogleTranscriptionEvents.VadDetected);
-
-    ep.removeCustomEventListener(AwsTranscriptionEvents.Transcription);
-    ep.removeCustomEventListener(AwsTranscriptionEvents.VadDetected);
-
-    ep.removeCustomEventListener(AzureTranscriptionEvents.Transcription);
-    ep.removeCustomEventListener(AzureTranscriptionEvents.NoSpeechDetected);
-    ep.removeCustomEventListener(AzureTranscriptionEvents.VadDetected);
-
-    ep.removeCustomEventListener(NuanceTranscriptionEvents.Transcription);
-    ep.removeCustomEventListener(NuanceTranscriptionEvents.TranscriptionComplete);
-    ep.removeCustomEventListener(NuanceTranscriptionEvents.StartOfSpeech);
-    ep.removeCustomEventListener(NuanceTranscriptionEvents.VadDetected);
-
-    ep.removeCustomEventListener(DeepgramTranscriptionEvents.Transcription);
-    ep.removeCustomEventListener(DeepgramTranscriptionEvents.Connect);
-    ep.removeCustomEventListener(DeepgramTranscriptionEvents.ConnectFailure);
-
-    ep.removeCustomEventListener(SonioxTranscriptionEvents.Transcription);
-
-    ep.removeCustomEventListener(CobaltTranscriptionEvents.Transcription);
-    ep.removeCustomEventListener(CobaltTranscriptionEvents.CompileContext);
-
-    ep.removeCustomEventListener(NvidiaTranscriptionEvents.Transcription);
-    ep.removeCustomEventListener(NvidiaTranscriptionEvents.TranscriptionComplete);
-    ep.removeCustomEventListener(NvidiaTranscriptionEvents.StartOfSpeech);
-    ep.removeCustomEventListener(NvidiaTranscriptionEvents.VadDetected);
-
-    ep.removeCustomEventListener(JambonzTranscriptionEvents.Transcription);
-    ep.removeCustomEventListener(JambonzTranscriptionEvents.Connect);
-    ep.removeCustomEventListener(JambonzTranscriptionEvents.ConnectFailure);
-
-    ep.removeCustomEventListener(JambonzTranscriptionEvents.Error);
-
-    ep.removeCustomEventListener(AssemblyAiTranscriptionEvents.Transcription);
-    ep.removeCustomEventListener(AssemblyAiTranscriptionEvents.Connect);
-    ep.removeCustomEventListener(AssemblyAiTranscriptionEvents.ConnectFailure);
-  };
-
   const setSpeechCredentialsAtRuntime = (recognizer) => {
     if (!recognizer) return;
     if (recognizer.vendor === 'nuance') {
@@ -832,7 +843,6 @@ module.exports = (logger) => {
   return {
     normalizeTranscription,
     setChannelVarsForStt,
-    removeSpeechListeners,
     setSpeechCredentialsAtRuntime,
     compileSonioxTranscripts,
     consolidateTranscripts

lib/utils/ws-requestor.js

@@ -9,7 +9,8 @@ const {
   JAMBONES_WS_PING_INTERVAL_MS,
   MAX_RECONNECTS,
   JAMBONES_WS_HANDSHAKE_TIMEOUT_MS,
-  JAMBONES_WS_MAX_PAYLOAD
+  JAMBONES_WS_MAX_PAYLOAD,
+  HTTP_USER_AGENT_HEADER
 } = require('../config');
 
 class WsRequestor extends BaseRequestor {
@@ -228,6 +229,9 @@ class WsRequestor extends BaseRequestor {
         maxRedirects: 2,
         handshakeTimeout,
         maxPayload: JAMBONES_WS_MAX_PAYLOAD ? parseInt(JAMBONES_WS_MAX_PAYLOAD) : 24 * 1024,
+        headers: {
+          ...(HTTP_USER_AGENT_HEADER && {'user-agent' : HTTP_USER_AGENT_HEADER})
+        }
       };
       if (this.username && this.password) opts = {...opts, auth: `${this.username}:${this.password}`};
 
@@ -322,7 +326,9 @@ class WsRequestor extends BaseRequestor {
           'WsRequestor:_onSocketClosed time to reconnect');
         if (!this.ws && !this.connectInProgress) {
           this.connectInProgress = true;
-          this._connect().catch((err) => this.connectInProgress = false);
+          return this._connect()
+            .catch((err) => this.logger.error('WsRequestor:_onSocketClosed There is error while reconnect', err))
+            .finally(() => this.connectInProgress = false);
         }
       }, this.backoffMs);
       this.backoffMs = this.backoffMs < 2000 ? this.backoffMs * 2 : (this.backoffMs + 2000);

package.json

@@ -1,9 +1,9 @@
 {
   "name": "jambonz-feature-server",
-  "version": "0.8.4",
+  "version": "0.8.6",
   "main": "app.js",
   "engines": {
-    "node": ">= 10.16.0"
+    "node": ">= 18.x"
   },
   "keywords": [
     "sip",
@@ -27,14 +27,14 @@
   "dependencies": {
     "@aws-sdk/client-auto-scaling": "^3.360.0",
     "@aws-sdk/client-sns": "^3.360.0",
-    "@jambonz/db-helpers": "^0.9.1",
+    "@jambonz/db-helpers": "^0.9.3",
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/mw-registrar": "^0.2.4",
-    "@jambonz/realtimedb-helpers": "^0.8.6",
+    "@jambonz/realtimedb-helpers": "^0.8.7",
-    "@jambonz/speech-utils": "^0.0.24",
+    "@jambonz/speech-utils": "^0.0.41",
     "@jambonz/stats-collector": "^0.1.9",
     "@jambonz/time-series": "^0.2.8",
-    "@jambonz/verb-specifications": "^0.0.45",
+    "@jambonz/verb-specifications": "^0.0.53",
     "@opentelemetry/api": "^1.4.0",
     "@opentelemetry/exporter-jaeger": "^1.9.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.35.0",
@@ -47,8 +47,8 @@
     "bent": "^7.3.12",
     "debug": "^4.3.4",
     "deepcopy": "^2.1.0",
-    "drachtio-fsmrf": "^3.0.27",
+    "drachtio-fsmrf": "^3.0.37",
-    "drachtio-srf": "^4.5.29",
+    "drachtio-srf": "^4.5.31",
     "express": "^4.18.2",
     "express-validator": "^7.0.1",
     "ip": "^1.1.8",

test/create-call-test.js

@@ -99,6 +99,8 @@ test('test create-call call-hook basic authentication', async(t) => {
     let obj = await getJSON(`http:127.0.0.1:3100/lastRequest/${from}`)
     t.ok(obj.headers.Authorization = 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=',
       'create-call: call-hook contains basic authentication header');
+    t.ok(obj.headers['user-agent'] = 'jambonz',
+    'create-call: call-hook contains user-agent header');
     disconnect();
   } catch (err) {
     console.log(`error received: ${err}`);

test/docker-compose-testbed.yaml

@@ -42,7 +42,7 @@ services:
         ipv4_address: 172.38.0.7
 
   drachtio:
-    image: drachtio/drachtio-server:0.8.22
+    image: drachtio/drachtio-server:0.8.24
     restart: always
     command: drachtio --contact "sip:*;transport=udp" --mtu 4096 --address 0.0.0.0 --port 9022
     ports:
@@ -57,7 +57,7 @@ services:
         condition: service_healthy
 
   freeswitch:
-    image: drachtio/drachtio-freeswitch-mrf:0.4.33
+    image: drachtio/drachtio-freeswitch-mrf:0.6.1
     restart: always
     command: freeswitch --rtp-range-start 20000 --rtp-range-end 20100
     environment:

test/hangup-test.js

@@ -0,0 +1,53 @@
+const test = require('tape');
+const { sippUac } = require('./sipp')('test_fs');
+const clearModule = require('clear-module');
+const {provisionCallHook, provisionCustomHook} = require('./utils')
+const bent = require('bent');
+const getJSON = bent('json')
+
+process.on('unhandledRejection', (reason, p) => {
+  console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+});
+
+function connect(connectable) {
+  return new Promise((resolve, reject) => {
+    connectable.on('connect', () => {
+      return resolve();
+    });
+  });
+}
+
+test('\'hangup\' custom headers', async(t) => {
+  clearModule.all();
+  const {srf, disconnect} = require('../app');
+
+  try {
+    await connect(srf);
+
+    // GIVEN
+    const verbs = [
+      {
+        verb: 'play',
+        url: 'https://example.com/example.mp3'
+      },
+      {
+        "verb": "hangup",
+        "headers": {
+          "X-Reason" : "maximum call duration exceeded"
+        }
+      }
+    ];
+
+    const from = 'hangup_custom_headers';
+    await provisionCallHook(from, verbs)
+
+    // THEN
+    await sippUac('uac-success-received-bye.xml', '172.38.0.10', from);
+    t.pass('play: succeeds when using single link');
+    disconnect();
+  } catch (err) {
+    console.log(`error received: ${err}`);
+    disconnect();
+    t.error(err);
+  }
+});

test/index.js

@@ -16,6 +16,8 @@ require('./listen-tests');
 require('./config-test');
 require('./queue-test');
 require('./in-dialog-test');
+require('./hangup-test');
+require('./sdp-utils-test');
 require('./http-proxy-test');
 require('./remove-test-db');
 require('./docker_stop');

test/play-tests.js

@@ -188,7 +188,7 @@ test('\'play\' tests with seekOffset and actionHook', async(t) => {
     const seconds = parseInt(obj.body.playback_seconds);
     const milliseconds = parseInt(obj.body.playback_milliseconds);
     const lastOffsetPos = parseInt(obj.body.playback_last_offset_pos);
-    //console.log({obj}, 'lastRequest');
+    console.log({obj}, 'lastRequest');
     t.ok(obj.body.reason === "playCompleted", "play: actionHook success received");
     t.ok(seconds === 2, "playback_seconds: actionHook success received");
     t.ok(milliseconds === 2048, "playback_milliseconds: actionHook success received");

test/sdp-utils-test.js

@@ -0,0 +1,26 @@
+const test = require('tape');
+const {makeOpusFirst, isOpusFirst} = require('../lib/utils/sdp-utils');
+const sdpTransform = require('sdp-transform');
+
+test('test opus first', (t) => {
+    const sdp = 'v=0\r\no=- 3348584794228993675 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0\r\na=extmap-allow-mixed\r\na=msid-semantic: WMS caca8b77-5ae5-4e73-a4d5-de1fce930335\r\nm=audio 57088 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126\r\nc=IN IP4 14.238.89.50\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:1401281302 1 udp 2122260223 10.231.36.146 57088 typ host generation 0 network-id 1 network-cost 10\r\na=candidate:2173263513 1 udp 1686052607 14.238.89.50 57088 typ srflx raddr 10.231.36.146 rport 57088 generation 0 network-id 1 network-cost 10\r\na=ice-ufrag:k5nc\r\na=ice-pwd:J0qwMs6HrIcFNZbDG5m8Kqpk\r\na=ice-options:trickle\r\na=fingerprint:sha-256 66:DE:9A:76:CE:11:2D:65:C4:08:C7:87:B4:90:7E:F1:8D:07:B9:F4:FF:E3:81:D7:E7:7D:C6:56:47:01:6E:55\r\na=setup:actpass\r\na=mid:0\r\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=sendrecv\r\na=msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\na=rtcp-mux\r\na=rtcp-fb:111 transport-cc\r\na=fmtp:111 minptime=10;useinbandfec=1\r\na=rtpmap:63 red/48000/2\r\na=fmtp:63 111/111\r\na=rtpmap:9 G722/8000\r\na=rtpmap:0 PCMU/8000\r\na=rtpmap:8 PCMA/8000\r\na=rtpmap:13 CN/8000\r\na=rtpmap:111 opus/48000/2\r\na=rtpmap:110 telephone-event/48000\r\na=rtpmap:126 telephone-event/8000\r\na=ssrc:3207459321 cname:4nyPJ6KXvseBUIhu\r\na=ssrc:3207459321 msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\n';
+    const opusSdp = makeOpusFirst(sdp);
+    const parsedSdp = sdpTransform.parse(opusSdp);
+    const opusIndex = parsedSdp.media[0].rtp.findIndex((entry) => entry.codec === 'opus');
+    t.ok(opusIndex === 0, 'succesffuly move opus to be first offer')
+    t.end();
+});
+
+
+test('test is opus first', (t) => {
+
+    const sdp = 'v=0\r\no=- 3348584794228993675 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0\r\na=extmap-allow-mixed\r\na=msid-semantic: WMS caca8b77-5ae5-4e73-a4d5-de1fce930335\r\nm=audio 57088 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126\r\nc=IN IP4 14.238.89.50\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:1401281302 1 udp 2122260223 10.231.36.146 57088 typ host generation 0 network-id 1 network-cost 10\r\na=candidate:2173263513 1 udp 1686052607 14.238.89.50 57088 typ srflx raddr 10.231.36.146 rport 57088 generation 0 network-id 1 network-cost 10\r\na=ice-ufrag:k5nc\r\na=ice-pwd:J0qwMs6HrIcFNZbDG5m8Kqpk\r\na=ice-options:trickle\r\na=fingerprint:sha-256 66:DE:9A:76:CE:11:2D:65:C4:08:C7:87:B4:90:7E:F1:8D:07:B9:F4:FF:E3:81:D7:E7:7D:C6:56:47:01:6E:55\r\na=setup:actpass\r\na=mid:0\r\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=sendrecv\r\na=msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\na=rtcp-mux\r\na=rtpmap:111 opus/48000/2\r\na=rtcp-fb:111 transport-cc\r\na=fmtp:111 minptime=10;useinbandfec=1\r\na=rtpmap:63 red/48000/2\r\na=fmtp:63 111/111\r\na=rtpmap:9 G722/8000\r\na=rtpmap:0 PCMU/8000\r\na=rtpmap:8 PCMA/8000\r\na=rtpmap:13 CN/8000\r\na=rtpmap:110 telephone-event/48000\r\na=rtpmap:126 telephone-event/8000\r\na=ssrc:3207459321 cname:4nyPJ6KXvseBUIhu\r\na=ssrc:3207459321 msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\n';
+    t.ok(isOpusFirst(sdp), "opus is first offer");
+
+    const sdp2 = 'v=0\r\no=- 3348584794228993675 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0\r\na=extmap-allow-mixed\r\na=msid-semantic: WMS caca8b77-5ae5-4e73-a4d5-de1fce930335\r\nm=audio 57088 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126\r\nc=IN IP4 14.238.89.50\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:1401281302 1 udp 2122260223 10.231.36.146 57088 typ host generation 0 network-id 1 network-cost 10\r\na=candidate:2173263513 1 udp 1686052607 14.238.89.50 57088 typ srflx raddr 10.231.36.146 rport 57088 generation 0 network-id 1 network-cost 10\r\na=ice-ufrag:k5nc\r\na=ice-pwd:J0qwMs6HrIcFNZbDG5m8Kqpk\r\na=ice-options:trickle\r\na=fingerprint:sha-256 66:DE:9A:76:CE:11:2D:65:C4:08:C7:87:B4:90:7E:F1:8D:07:B9:F4:FF:E3:81:D7:E7:7D:C6:56:47:01:6E:55\r\na=setup:actpass\r\na=mid:0\r\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=sendrecv\r\na=msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\na=rtcp-mux\r\na=rtcp-fb:111 transport-cc\r\na=fmtp:111 minptime=10;useinbandfec=1\r\na=rtpmap:63 red/48000/2\r\na=fmtp:63 111/111\r\na=rtpmap:9 G722/8000\r\na=rtpmap:0 PCMU/8000\r\na=rtpmap:8 PCMA/8000\r\na=rtpmap:13 CN/8000\r\na=rtpmap:111 opus/48000/2\r\na=rtpmap:110 telephone-event/48000\r\na=rtpmap:126 telephone-event/8000\r\na=ssrc:3207459321 cname:4nyPJ6KXvseBUIhu\r\na=ssrc:3207459321 msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\n';
+    t.ok(!isOpusFirst(sdp2), "opus is not first offer")
+
+    const sdp3 = 'v=0\r\no=xhoaluu2 1314 1504 IN IP4 192.168.1.4\r\ns=Talk\r\nc=IN IP4 192.168.1.4\r\nt=0 0\r\na=ice-pwd:397d063ea23fdc05164e3ee4\r\na=ice-ufrag:16c449a3\r\na=rtcp-xr:rcvr-rtt=all:10000 stat-summary=loss,dup,jitt,TTL voip-metrics\r\na=group:BUNDLE as\r\na=record:off\r\nm=audio 56542 RTP/AVPF 0 8\r\nc=IN IP4 14.226.233.151\r\na=rtcp-mux\r\na=mid:as\r\na=extmap:1 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=rtcp:63076 IN IP4 192.168.1.4\r\na=candidate:1 1 UDP 2130706303 192.168.1.4 56542 typ host\r\na=candidate:1 2 UDP 2130706302 192.168.1.4 63076 typ host\r\na=candidate:2 1 UDP 2130706431 2001:ee0:d744:dcf0:c1d3:d73f:7a93:dc9f 56542 typ host\r\na=candidate:2 2 UDP 2130706430 2001:ee0:d744:dcf0:c1d3:d73f:7a93:dc9f 63076 typ host\r\na=candidate:3 1 UDP 2130706431 2001:ee0:d744:dcf0:15:6be3:8e6b:b736 56542 typ host\r\na=candidate:3 2 UDP 2130706430 2001:ee0:d744:dcf0:15:6be3:8e6b:b736 63076 typ host\r\na=candidate:4 1 UDP 1694498687 14.226.233.151 56542 typ srflx raddr 192.168.1.4 rport 56542\r\na=rtcp-fb:* trr-int 1000\r\na=rtcp-fb:* ccm tmmbr';
+    t.ok(!isOpusFirst(sdp2), "opus is not first offer")
+    t.end();
+});

test/transcribe-tests.js

@@ -52,6 +52,7 @@ test('\'transcribe\' test - google', async(t) => {
     // THEN
     await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
     let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
+    //console.log(JSON.stringify(obj));
     t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().startsWith('i\'d like to speak to customer support'),
       'transcribe: succeeds when using google credentials');
 
@@ -89,6 +90,7 @@ test('\'transcribe\' test - microsoft', async(t) => {
     // THEN
     await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
     let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
+    //console.log(JSON.stringify(obj));
     t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().startsWith('i\'d like to speak to customer support'),
       'transcribe: succeeds when using  microsoft credentials');
 
@@ -126,6 +128,7 @@ test('\'transcribe\' test - aws', async(t) => {
     // THEN
     await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
     let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
+    //console.log(JSON.stringify(obj));
     t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().startsWith('i\'d like to speak to customer support'),
       'transcribe: succeeds when using aws credentials');
 
@@ -137,6 +140,71 @@ test('\'transcribe\' test - aws', async(t) => {
   }
 });
 
+
+test('\'transcribe\' test - deepgram config options', async(t) => {
+  if (!DEEPGRAM_API_KEY ) {
+    t.pass('skipping deepgram tests');
+    return t.end();
+  }
+  clearModule.all();
+  const {srf, disconnect} = require('../app');
+
+  try {
+    await connect(srf);
+    // GIVEN
+    let verbs = [
+      {
+        "verb": "config",
+        "recognizer": {
+          "vendor": "deepgram",
+          "language": "en-US",
+          "altLanguages": [
+            "en-US"
+          ],
+          "deepgramOptions": {
+            "model": "2-ea",
+            "tier": "nova",
+            "numerals": true,
+            "ner": true,
+            "vadTurnoff": 10,
+            "keywords": [
+              "CPT"
+            ]
+          }
+        }
+      },
+      {
+        "verb": "transcribe",
+        "transcriptionHook": "/transcriptionHook",
+        "recognizer": {
+          "vendor": "deepgram",
+          "altLanguages": [
+            "en-AU"
+          ],
+          "hints": ["customer support", "sales", "human resources", "HR"],
+          "deepgramOptions": {
+            "apiKey": DEEPGRAM_API_KEY,
+          }
+        }
+      }
+    ];
+    let from = "gather_success";
+    await provisionCallHook(from, verbs);
+    // THEN
+    await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
+    let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
+    //console.log(JSON.stringify(obj));
+    t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().includes('like to speak to customer support'),
+      'transcribe: succeeds when using deepgram credentials');
+
+    disconnect();
+  } catch (err) {
+    console.log(`error received: ${err}`);
+    disconnect();
+    t.error(err);
+  }
+});
+
 test('\'transcribe\' test - deepgram', async(t) => {
   if (!DEEPGRAM_API_KEY ) {
     t.pass('skipping deepgram tests');
@@ -166,6 +234,7 @@ test('\'transcribe\' test - deepgram', async(t) => {
     // THEN
     await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
     let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
+    //console.log(JSON.stringify(obj));
     t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().includes('like to speak to customer support'),
       'transcribe: succeeds when using deepgram credentials');
 
@@ -245,9 +314,131 @@ test('\'transcribe\' test - google with asrTimeout', async(t) => {
     // THEN
     await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
     let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
+    //console.log(JSON.stringify(obj));
     t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().startsWith('i\'d like to speak to customer support'),
       'transcribe: succeeds when using google credentials');
 
+    disconnect();
+  } catch (err) {
+    console.log(`error received: ${err}`);
+    disconnect();
+    t.error(err);
+  }
+});
+
+test('\'transcribe\' test - deepgram config options altLanguages', async(t) => {
+  if (!DEEPGRAM_API_KEY ) {
+    t.pass('skipping deepgram tests');
+    return t.end();
+  }
+  clearModule.all();
+  const {srf, disconnect} = require('../app');
+
+  try {
+    await connect(srf);
+    // GIVEN
+    let verbs = [
+      {
+        "verb": "config",
+        "recognizer": {
+          "vendor": "deepgram",
+          "language": "en-US",
+          "altLanguages": [
+            "en-US"
+          ],
+          "deepgramOptions": {
+            "model": "nova-2",
+            "numerals": true,
+            "ner": true,
+            "vadTurnoff": 10,
+            "keywords": [
+              "CPT"
+            ]
+          }
+        }
+      },
+      {
+        "verb": "transcribe",
+        "transcriptionHook": "/transcriptionHook",
+        "recognizer": {
+          "vendor": "deepgram",
+          "hints": ["customer support", "sales", "human resources", "HR"],
+          "deepgramOptions": {
+            "apiKey": DEEPGRAM_API_KEY,
+          }
+        }
+      }
+    ];
+    let from = "gather_success_no_altLanguages";
+    await provisionCallHook(from, verbs);
+    // THEN
+    await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
+    let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
+    //console.log(JSON.stringify(obj));
+    t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().includes('like to speak to customer support'),
+      'transcribe: succeeds when using deepgram credentials');
+
+    disconnect();
+  } catch (err) {
+    console.log(`error received: ${err}`);
+    disconnect();
+    t.error(err);
+  }
+});
+
+test('\'transcribe\' test - deepgram config options altLanguages', async(t) => {
+  if (!DEEPGRAM_API_KEY ) {
+    t.pass('skipping deepgram tests');
+    return t.end();
+  }
+  clearModule.all();
+  const {srf, disconnect} = require('../app');
+
+  try {
+    await connect(srf);
+    // GIVEN
+    let verbs = [
+      {
+        "verb": "config",
+        "recognizer": {
+          "vendor": "deepgram",
+          "language": "en-US",
+          "altLanguages": [
+            "en-US"
+          ],
+          "deepgramOptions": {
+            "model": "nova-2",
+            "numerals": true,
+            "ner": true,
+            "vadTurnoff": 10,
+            "keywords": [
+              "CPT"
+            ]
+          }
+        }
+      },
+      {
+        "verb": "transcribe",
+        "transcriptionHook": "/transcriptionHook",
+        "recognizer": {
+          "vendor": "deepgram",
+          "hints": ["customer support", "sales", "human resources", "HR"],
+          "altLanguages": [],
+          "deepgramOptions": {
+            "apiKey": DEEPGRAM_API_KEY,
+          }
+        }
+      }
+    ];
+    let from = "gather_success_has_altLanguages";
+    await provisionCallHook(from, verbs);
+    // THEN
+    await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
+    let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
+    //console.log(JSON.stringify(obj));
+    t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().includes('like to speak to customer support'),
+      'transcribe: succeeds when using deepgram credentials');
+
     disconnect();
   } catch (err) {
     console.log(`error received: ${err}`);