Compare commits

...

34 Commits

Author SHA1 Message Date
Sam Machin
f85b5f9a01 Update call-session.js 2025-01-10 13:56:43 +00:00
Dave Horton
6bad1a22f3 fix #1025 (#1026)
* fix #1025

* redirect verb should be able to redirect to a new websocket endpoint
2025-01-09 15:45:20 -05:00
Hoan Luu Huu
fcefa1ff31 fix inband dtmf does not work in dial verb (#1018) 2025-01-08 18:29:43 -05:00
Hoan Luu Huu
67cd53c930 rest:dial support timeLimit (#1024)
* rest:dial support timeLimit

* wip

* wip

* clear maxCallDuration timer
2025-01-07 12:21:09 -05:00
Dave Horton
a59784b8ab update base image to node:20-alpine (#1022) 2025-01-04 16:38:25 -05:00
Dave Horton
a2581eaeb4 tts throttling and send user_interruption event (#1019)
* tts throttling and send user_interruption event

* tts streaming: if we get a flush with tokens pending, send the flush after the tokens

* wip
2025-01-04 16:34:01 -05:00
Dave Horton
3706aa4d98 #1020 - fix for sticky bargein (#1021) 2025-01-03 10:41:35 -05:00
Dave Horton
25f1e65f63 feed TTS in sentence chunks when streaming (#1013)
* feed TTS in sentence chunks when streaming

* tts streaming: treat a paragraph as a chunk of text, even it not ending with a line end character

* wip
2024-12-31 15:16:25 -05:00
rammohan-y
c9f0481ca6 feat/1009, sending reason in X-Reason header when AHD processor giveup (#1014)
* feat/1009, sending reason in X-Reason header when AHD processor giveup is executed

* fixed jslint error

* added an alert
2024-12-31 15:09:23 -05:00
Hoan Luu Huu
564f6c9e55 support kill dial if sd ep is media timeout (#1001)
* support kill dial if sd ep is media timeout

* support kill dial if sd ep is media timeout

* support kill dial if sd ep is media timeout

* add media timeout reason header to bye message

* wip

* wip

* make configuration for freeswitch media timeout

* make configuration for freeswitch media timeout

* wip
2024-12-23 07:19:41 -05:00
Dave Horton
02f25f8343 fix cartesia channel vars for streaming (#1012) 2024-12-20 16:48:20 -05:00
Hoan Luu Huu
13ef89d605 support elevenlabs tts stream (#1011)
* support elevenlabs tts stream

* wip

* wip
2024-12-20 09:50:13 -05:00
Dave Horton
d05e470867 remove hardcoding of openai model 2024-12-19 18:42:57 -05:00
Hoan Luu Huu
17250f8386 support cartesia tts (#1008)
* support cartesia tts

* update speech util version

* update speech utils version
2024-12-19 07:35:47 -05:00
Dave Horton
ba3f46df64 Feat/tts streaming (#994)
* wip

* add TtsStreamingBuffer class to abstract handling of streaming tokens

* wip

* add throttling support

* support background ttsStream (#995)

* wip

* add TtsStreamingBuffer class to abstract handling of streaming tokens

* wip

* support background ttsStream

* wip

---------

Co-authored-by: Dave Horton <daveh@beachdognet.com>

* wip

* dont send if we have nothing to send

* initial testing with cartesia

* wip

---------

Co-authored-by: Hoan Luu Huu <110280845+xquanluu@users.noreply.github.com>
2024-12-18 14:44:37 -05:00
RJ Burnham
f37e1540ee Make voicemail hints case insensitive (#1007) 2024-12-13 13:42:29 -05:00
Dave Horton
5e04db82bf Feat/deepgram voice agent (#1006)
* wip

* wip

* wip
2024-12-13 10:05:23 -05:00
Dave Horton
0aa37a83ae Feat/handle 3pcc invite (#1005)
* wip

* wip

* linting
2024-12-12 18:39:15 -05:00
Hoan Luu Huu
c29ab0d858 support referBy display name (#1000)
* support referBy display name

* wip

* update verb specification
2024-12-11 12:46:29 -05:00
Sam Machin
71d4c90cbc catch error (#1002)
* catch error

* remove notifyTaskDone
2024-12-11 12:34:44 -05:00
Hoan Luu Huu
a929a649f9 fix ConfirmCallSession cannot be played (#993)
* fix ConfirmCallSession cannot be played

* fix review comments

* fix review comments
2024-12-10 19:36:42 -05:00
Dave Horton
3bb4f1a29f fix #998 incorrectly sending final transcript with is_final=false (#999) 2024-12-10 18:48:02 -05:00
Hoan Luu Huu
54cc76606b fix cannot replace endpoint for adulting session (#992)
* fix cannot replace endpoint for adulting session

* fix cannot replace endpoint for adulting session
2024-12-06 07:51:24 -05:00
rammohan-y
0458bb7d6c Feat/884: Capture system_alert when feature-server is online or offline (#950)
* writing alerts during startup and shutdown of feature-server

* feat/884: created constants for system component name and state

* feat/88: added 0.2.11 version of time-series

* feat/884: renamed constant, and added GracefulShutdownInProgress system alert
2024-12-05 09:23:03 -05:00
Sam Machin
dce4fe1f82 Fix/986 (#990)
* throw new NonFatalTask error on play file not found

* linting

* make SpeechCredentialError subclass of NonFatalTask error

* cleanup

* Update action-hook-delay.js

* bump fsmrf version

* linting and package-lock

* Update package-lock.json

* update error

* only throw on fs error "File not found"

* add alert

* update time-series dep

* Update package-lock.json

* linting

* Update play.js

* remove stack trace from error message

* fix error formatting
2024-12-04 05:47:49 -05:00
Hoan Luu Huu
e96c35d571 fixed iamrole from sessionToken to securityToken (#988)
* fixed iamrole from sessionToken to securityToken

* wip

* support get aws credential from instance profile
2024-11-29 21:58:42 -05:00
Hoan Luu Huu
070671a3fb support send refer custom header to referhook (#981) 2024-11-28 08:34:34 -05:00
rammohan-y
efdb56f0a0 feat/971 - fix to allow hints objects array (#987) 2024-11-28 07:25:10 -05:00
Hoan Luu Huu
e2edbb4a5b support enable dtmf tone (#976)
* support enable dtmf tone

* wip

* wip
2024-11-26 20:25:48 -05:00
Markus Frindt
3a6d63a1c6 Fix the issue for outbound calls that always the None credentials wer… (#984)
* Fix the issue for outbound calls that always the None credentials were used. session:new for rest dial did not contain recognizer.label and synthesizer.label

* update comment

---------

Co-authored-by: mfrindt <m.frindt@cognigy.com>
2024-11-26 10:26:20 -05:00
rammohan-y
c874ab8100 feat/975: fixed continuous asr not stopping when asrDtmfTerminationDi… (#977)
* feat/975: fixed continuous asr not stopping when asrDtmfTerminationDigit is configured

* feat/975: giving first preference to asrDtmfTerminationDigit if there is already ASR happened
2024-11-26 08:23:11 -05:00
Dave Horton
24a66fed64 wip (#979) 2024-11-19 09:37:00 -05:00
Hoan Luu Huu
c8c3738ae8 custom stt vendor ws connection should not be closed in asrTimeout (#973) 2024-11-18 10:17:31 -05:00
Dave Horton
c1330d4651 fix transcribe fixes for speechmatics (#978)
* fix transcribe fixes for speechmatics

* update to verb-specs with fixes for speechmatics

* add support for speechmatics translation

* add handlers for receiving translations

* call translation hookd

* gather: no need to restart speechmatics after a final transcript during continuous asr

* graceful shutdown

* wip

* wip

* wip

* wip

* wip
2024-11-16 10:21:04 -05:00
41 changed files with 1972 additions and 209 deletions

3
.gitignore vendored
View File

@@ -42,4 +42,5 @@ ecosystem.config.js
test/credentials/*.json
run-tests.sh
run-coverage.sh
.vscode
.vscode
.env

View File

@@ -1,4 +1,4 @@
FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
FROM --platform=linux/amd64 node:20-alpine as base
RUN apk --update --no-cache add --virtual .builds-deps build-base python3

47
app.js
View File

@@ -25,10 +25,22 @@ const opts = {
};
const pino = require('pino');
const logger = pino(opts, pino.destination({sync: false}));
const {LifeCycleEvents, FS_UUID_SET_NAME} = require('./lib/utils/constants');
const {LifeCycleEvents, FS_UUID_SET_NAME, SystemState, FEATURE_SERVER} = require('./lib/utils/constants');
const installSrfLocals = require('./lib/utils/install-srf-locals');
installSrfLocals(srf, logger);
const writeSystemAlerts = srf.locals?.writeSystemAlerts;
if (writeSystemAlerts) {
writeSystemAlerts({
system_component: FEATURE_SERVER,
state : SystemState.Online,
fields : {
detail: `feature-server with process_id ${process.pid} started`,
host: srf.locals?.ipv4
}
});
}
const {
initLocals,
createRootSpan,
@@ -100,12 +112,19 @@ createHttpListener(logger, srf)
});
setInterval(async() => {
const monInterval = setInterval(async() => {
srf.locals.stats.gauge('fs.sip.calls.count', sessionTracker.count);
// Checking system log level
const systemInformation = await srf.locals.dbHelpers.lookupSystemInformation();
if (systemInformation && systemInformation.log_level) {
logger.level = systemInformation.log_level;
try {
const systemInformation = await srf.locals.dbHelpers.lookupSystemInformation();
if (systemInformation && systemInformation.log_level) {
logger.level = systemInformation.log_level;
}
} catch (err) {
if (process.env.NODE_ENV === 'test') {
clearInterval(monInterval);
logger.error('all tests complete');
}
else logger.error({err}, 'Error checking system log level in database');
}
}, 20000);
@@ -117,13 +136,25 @@ const disconnect = () => {
srf.locals.mediaservers?.forEach((ms) => ms.disconnect());
});
};
process.on('SIGTERM', handle);
process.on('SIGINT', handle);
function handle(signal) {
async function handle(signal) {
const {removeFromSet} = srf.locals.dbHelpers;
srf.locals.disabled = true;
logger.info(`got signal ${signal}`);
const writeSystemAlerts = srf.locals?.writeSystemAlerts;
if (writeSystemAlerts) {
// it has to be synchronous call, or else by the time system saves the app terminates
await writeSystemAlerts({
system_component: FEATURE_SERVER,
state : SystemState.Offline,
fields : {
detail: `feature-server with process_id ${process.pid} stopped, signal ${signal}`,
host: srf.locals?.ipv4
}
});
}
const setName = `${(JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
const fsServiceUrlSetName = `${(JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
if (setName && srf.locals.localSipAddress) {

View File

@@ -137,6 +137,8 @@ const JAMBONES_EAGERLY_PRE_CACHE_AUDIO = parseInt(process.env.JAMBONES_EAGERLY_P
const JAMBONES_USE_FREESWITCH_TIMER_FD = process.env.JAMBONES_USE_FREESWITCH_TIMER_FD;
const JAMBONES_DIAL_SBC_FOR_REGISTERED_USER = process.env.JAMBONES_DIAL_SBC_FOR_REGISTERED_USER || false;
const JAMBONES_MEDIA_TIMEOUT_MS = process.env.JAMBONES_MEDIA_TIMEOUT_MS || 0;
const JAMBONES_MEDIA_HOLD_TIMEOUT_MS = process.env.JAMBONES_MEDIA_HOLD_TIMEOUT_MS || 0;
module.exports = {
JAMBONES_MYSQL_HOST,
@@ -223,5 +225,7 @@ module.exports = {
JAMBONZ_DISABLE_DIAL_PAI_HEADER,
JAMBONES_DISABLE_DIRECT_P2P_CALL,
JAMBONES_USE_FREESWITCH_TIMER_FD,
JAMBONES_DIAL_SBC_FOR_REGISTERED_USER
JAMBONES_DIAL_SBC_FOR_REGISTERED_USER,
JAMBONES_MEDIA_TIMEOUT_MS,
JAMBONES_MEDIA_HOLD_TIMEOUT_MS
};

View File

@@ -208,10 +208,13 @@ router.post('/',
/**
* create our application object -
* not from the database as per an inbound call,
* but from the provided params in the request
* we merge the inbound call application,
* with the provided app params from the request body
*/
const app = req.body;
const app = {
...application,
...req.body
};
/**
* attach our requestor and notifier objects

View File

@@ -45,8 +45,10 @@ class AdultingCallSession extends CallSession {
return this.sd.ep;
}
/* see note above */
set ep(newEp) {}
// When adulting session kicked from conference, replaceEndpoint is a must
set ep(newEp) {
this.sd.ep = newEp;
}
get callSid() {
return this.callInfo.callSid;

View File

@@ -2,13 +2,15 @@ const Emitter = require('events');
const fs = require('fs');
const {
CallDirection,
MediaPath,
TaskPreconditions,
CallStatus,
TaskName,
KillReason,
RecordState,
AllowedSipRecVerbs,
AllowedConfirmSessionVerbs
AllowedConfirmSessionVerbs,
TtsStreamingEvents
} = require('../utils/constants');
const moment = require('moment');
const assert = require('assert');
@@ -20,18 +22,22 @@ const listTaskNames = require('../utils/summarize-tasks');
const HttpRequestor = require('../utils/http-requestor');
const WsRequestor = require('../utils/ws-requestor');
const ActionHookDelayProcessor = require('../utils/action-hook-delay');
const TtsStreamingBuffer = require('../utils/tts-streaming-buffer');
const {parseUri} = require('drachtio-srf');
const {
JAMBONES_INJECT_CONTENT,
JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
AWS_REGION,
JAMBONES_USE_FREESWITCH_TIMER_FD
JAMBONES_USE_FREESWITCH_TIMER_FD,
JAMBONES_MEDIA_TIMEOUT_MS,
JAMBONES_MEDIA_HOLD_TIMEOUT_MS
} = require('../config');
const bent = require('bent');
const BackgroundTaskManager = require('../utils/background-task-manager');
const dbUtils = require('../utils/db-utils');
const BADPRECONDITIONS = 'preconditions not met';
const CALLER_CANCELLED_ERR_MSG = 'Response not sent due to unknown transaction';
const { NonFatalTaskError} = require('../utils/error');
const sqlRetrieveQueueEventHook = `SELECT * FROM webhooks
WHERE webhook_sid =
(
@@ -398,33 +404,41 @@ class CallSession extends Emitter {
return this.application.transferredCall === true;
}
/**
* returns true if this session is an inbound call session
*/
get isInboundCallSession() {
return this.constructor.name === 'InboundCallSession';
}
/**
* returns true if this session is a ConfirmCallSession
*/
get isAdultingCallSession() {
return this.constructor.name === 'AdultingCallSession';
}
/**
* returns true if this session is a ConfirmCallSession
*/
get isConfirmCallSession() {
return this.constructor.name === 'ConfirmCallSession';
}
/**
* returns true if this session is a SipRecCallSession
*/
get isSipRecCallSession() {
return this.constructor.name === 'SipRecCallSession';
}
/**
* returns true if this session is a SmsCallSession
*/
get isSmsCallSession() {
return this.constructor.name === 'SmsCallSession';
}
get isRestCallSession() {
return this.constructor.name === 'RestCallSession';
}
get InboundCallSession() {
return this.constructor.name === 'InboundCallSession';
}
get isNormalCallSession() {
return this.constructor.name === 'InboundCallSession' || this.constructor.name === 'RestCallSession';
}
get is3pccInvite() {
return this.isInboundCallSession && this.req?.body?.length === 0;
}
get webhook_secret() {
return this.accountInfo?.account?.webhook_secret;
@@ -438,6 +452,10 @@ class CallSession extends Emitter {
return this.backgroundTaskManager.isTaskRunning('bargeIn');
}
get isTtsStreamEnabled() {
return this.backgroundTaskManager.isTaskRunning('ttsStream');
}
get isListenEnabled() {
return this.backgroundTaskManager.isTaskRunning('listen');
}
@@ -500,6 +518,10 @@ class CallSession extends Emitter {
this._sipRequestWithinDialogHook = url;
}
get isTtsStreamOpen() {
return this.currentTask?.isStreamingTts ||
this.backgroundTaskManager.getTask('ttsStream')?.isStreamingTts;
}
// Bot Delay (actionHook delayed)
get actionHookDelayEnabled() {
return this._actionHookDelayEnabled;
@@ -550,7 +572,18 @@ class CallSession extends Emitter {
this._actionHookDelayProcessor = new ActionHookDelayProcessor(this.logger, opts, this, this.ep);
this._actionHookDelayProcessor.on('giveup', () => {
this.logger.info('CallSession: ActionHookDelayProcessor: giveup event - hanging up call');
this._jambonzHangup();
const {writeAlerts} = this.srf.locals;
try {
writeAlerts({
alert_type: 'bot-action-delay-giveup',
account_sid: this.accountSid,
message: 'Call terminated due to bot action delay timeout',
target_sid: this.callSid
});
} catch (err) {
this.logger.error({err}, 'Error writing bot-action-delay-giveup alert');
}
this._jambonzHangup('bot-action-delay-giveup');
if (this.wakeupResolver) {
this.logger.debug('CallSession: Giveup timer expired - waking up');
this.wakeupResolver({reason: 'noResponseGiveUp'});
@@ -574,6 +607,25 @@ class CallSession extends Emitter {
}
}
getTsStreamingVendor() {
let v;
if (this.currentTask?.isStreamingTts) {
const {vendor} = this.currentTask.getTtsVendorData(this);
v = vendor;
}
else if (this.backgroundTaskManager.getTask('ttsStream')?.isStreamingTts) {
const {vendor} = this.backgroundTaskManager.getTask('ttsStream').getTtsVendorData(this);
v = vendor;
}
return v;
}
get appIsUsingWebsockets() {
return this.requestor instanceof WsRequestor;
}
/* end of getters and setters */
async clearOrRestoreActionHookDelayProcessor() {
if (this._actionHookDelayProcessor) {
await this._actionHookDelayProcessor.stop();
@@ -791,6 +843,38 @@ class CallSession extends Emitter {
}
}
async enableBackgroundTtsStream(say) {
try {
if (this.isTtsStreamEnabled) {
this.logger.debug('CallSession:enableBackgroundTtsStream - ttsStream currently enabled, ignoring request');
} else if (this.appIsUsingWebsockets && this.isNormalCallSession) {
await this.backgroundTaskManager.newTask('ttsStream', say);
this.logger.debug('CallSession:enableBackgroundTtsStream - ttsStream enabled');
} else {
this.logger.debug(
'CallSession:enableBackgroundTtsStream - ignoring request as call does not have required conditions');
}
} catch (err) {
this.logger.info({err, say}, 'CallSession:enableBackgroundTtsStream - Error creating background tts stream task');
}
}
disableTtsStream() {
if (this.isTtsStreamEnabled) {
this.backgroundTaskManager.stop('ttsStream');
this.logger.debug('CallSession:disableTtsStream - ttsStream disabled');
}
}
clearTtsStream() {
this.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'user_interruption'})
.catch((err) => this.logger.info({err}, 'CallSession:clearTtsStream - Error sending user_interruption'));
this.ttsStreamingBuffer?.clear();
}
startTtsStream() {
this.ttsStreamingBuffer?.start();
}
async enableBotMode(gather, autoEnable) {
try {
let task;
@@ -979,6 +1063,14 @@ class CallSession extends Emitter {
options: credential.options
};
}
else if ('cartesia' === vendor) {
return {
api_key: credential.api_key,
model_id: credential.model_id,
embedding: credential.embedding,
options: credential.options
};
}
else if ('rimelabs' === vendor) {
return {
api_key: credential.api_key,
@@ -1040,6 +1132,27 @@ class CallSession extends Emitter {
async exec() {
this.logger.info({tasks: listTaskNames(this.tasks)}, `CallSession:exec starting ${this.tasks.length} tasks`);
// calculate if inbandDTMF tone is used
const voip_carrier_sid = this.req?.has('X-Voip-Carrier-Sid') ? this.req.get('X-Voip-Carrier-Sid') :
this.req?.has('X-Requested-Carrier-Sid') ? this.req.get('X-Requested-Carrier-Sid') : null;
if (voip_carrier_sid) {
const {lookupVoipCarrierBySid} = dbUtils(this.logger, this.srf);
const [voipCarrier] = await lookupVoipCarrierBySid(voip_carrier_sid);
this.inbandDtmfEnabled = voipCarrier?.dtmf_type === 'tones';
}
if (this.isNormalCallSession) {
this.ttsStreamingBuffer = new TtsStreamingBuffer(this);
this.ttsStreamingBuffer.on(TtsStreamingEvents.Empty, this._onTtsStreamingEmpty.bind(this));
this.ttsStreamingBuffer.on(TtsStreamingEvents.Pause, this._onTtsStreamingPause.bind(this));
this.ttsStreamingBuffer.on(TtsStreamingEvents.Resume, this._onTtsStreamingResume.bind(this));
this.ttsStreamingBuffer.on(TtsStreamingEvents.ConnectFailure, this._onTtsStreamingConnectFailure.bind(this));
}
else {
this.logger.info(`CallSession:exec - not a normal call session: ${this.constructor.name}`);
}
while (this.tasks.length && !this.callGone) {
const taskNum = ++this.taskIdx;
const stackNum = this.stackIdx;
@@ -1080,6 +1193,8 @@ class CallSession extends Emitter {
this.currentTask = null;
if (err.message?.includes(BADPRECONDITIONS)) {
this.logger.info(`CallSession:exec task #${stackNum}:${taskNum}: ${task.name}: ${err.message}`);
} else if (err instanceof NonFatalTaskError) {
this.logger.error(err, `Error executing task #${stackNum}:${taskNum}: ${task.name}`);
}
else {
this.logger.error(err, `Error executing task #${stackNum}:${taskNum}: ${task.name}`);
@@ -1172,6 +1287,11 @@ class CallSession extends Emitter {
this.wakeupResolver({reason: 'session ended'});
this.wakeupResolver = null;
}
if (this._maxCallDurationTimer) {
clearTimeout(this._maxCallDurationTimer);
this._maxCallDurationTimer = null;
}
}
/**
@@ -1555,7 +1675,7 @@ Duration=${duration} `
const t = normalizeJambones(this.logger, [
{
verb: 'dub',
...opts
...opts.dub
}
])
.map((tdata) => makeTask(this.logger, tdata));
@@ -1589,6 +1709,15 @@ Duration=${duration} `
this.logger.info({response}, '_lccBoostAudioSignal: response from freeswitch');
}
async _lccMediaPath(desiredPath) {
const task = this.currentTask;
if (!task || task.name !== TaskName.Dial) {
return this.logger.info('CallSession:_lccMediaPath - invalid command since we are not in a dial verb');
}
task.updateMediaPath(desiredPath)
.catch((err) => this.logger.error(err, 'CallSession:_lccMediaPath'));
}
_lccToolOutput(tool_call_id, opts, callSid) {
// only valid if we are in an LLM verb
const task = this.currentTask;
@@ -1612,6 +1741,39 @@ Duration=${duration} `
.catch((err) => this.logger.error(err, 'CallSession:_lccLlmUpdate'));
}
async _lccTtsTokens(opts) {
const {id, tokens} = opts;
if (id === undefined) {
this.logger.info({opts}, 'CallSession:_lccTtsTokens - invalid command since id is missing');
return;
}
else if (tokens === undefined) {
this.logger.info({opts}, 'CallSession:_lccTtsTokens - invalid command since id is missing');
return this.requestor.request('tts:tokens-result', '/tokens-result', {
id,
status: 'failed',
reason: 'missing tokens'
}).catch((err) => this.logger.debug({err}, 'CallSession:_notifyTaskStatus - Error sending'));
}
let res;
try {
res = await this.ttsStreamingBuffer?.bufferTokens(tokens);
this.logger.info({id, res}, 'CallSession:_lccTtsTokens - tts:tokens-result');
} catch (err) {
this.logger.info(err, 'CallSession:_lccTtsTokens');
}
this.requestor.request('tts:tokens-result', '/tokens-result', {id, ...res})
.catch((err) => this.logger.debug({err}, 'CallSession:_notifyTaskStatus - Error sending'));
}
_lccTtsFlush(opts) {
this.ttsStreamingBuffer?.flush(opts);
}
_lccTtsClear(opts) {
this.ttsStreamingBuffer?.clear(opts);
}
/**
* perform call hangup by jambonz
@@ -1672,6 +1834,9 @@ Duration=${duration} `
else if (opts.boostAudioSignal) {
return this._lccBoostAudioSignal(opts, callSid);
}
else if (opts.media_path) {
return this._lccMediaPath(opts.media_path, callSid);
}
else if (opts.llm_tool_output) {
return this._lccToolOutput(opts.tool_call_id, opts.llm_tool_output, callSid);
}
@@ -1975,6 +2140,13 @@ Duration=${duration} `
});
break;
case 'media:path':
this._lccMediaPath(data, call_sid)
.catch((err) => {
this.logger.info({err, data}, 'CallSession:_onCommand - error setting media path');
});
break;
case 'llm:tool-output':
this._lccToolOutput(tool_call_id, data, call_sid);
break;
@@ -1983,6 +2155,18 @@ Duration=${duration} `
this._lccLlmUpdate(data, call_sid);
break;
case 'tts:tokens':
this._lccTtsTokens(data);
break;
case 'tts:flush':
this._lccTtsFlush(data);
break;
case 'tts:clear':
this._lccTtsClear(data);
break;
default:
this.logger.info(`CallSession:_onCommand - invalid command ${command}`);
}
@@ -2040,6 +2224,8 @@ Duration=${duration} `
};
}
this._enableInbandDtmfIfRequired(this.ep);
// we are going from an early media connection to answer
if (this.direction === CallDirection.Inbound) {
// only do this for inbound call.
@@ -2063,7 +2249,6 @@ Duration=${duration} `
//ep.cs = this;
this.ep = ep;
this.logger.info(`allocated endpoint ${ep.uuid}`);
this._configMsEndpoint();
this.ep.on('destroy', () => {
@@ -2176,6 +2361,8 @@ Duration=${duration} `
// close all background tasks
this.backgroundTaskManager.stopAll();
this.clearOrRestoreActionHookDelayProcessor().catch((err) => {});
this.ttsStreamingBuffer?.stop();
}
/**
@@ -2225,22 +2412,43 @@ Duration=${duration} `
},
localSdp: this.ep.local.sdp
});
this.logger.debug('answered call');
this.dlg.on('destroy', this._callerHungup.bind(this));
this.wrapDialog(this.dlg);
this.dlg.callSid = this.callSid;
this.emit('callStatusChange', {sipStatus: 200, sipReason: 'OK', callStatus: CallStatus.InProgress});
if (this.recordOptions && this.recordState === RecordState.RecordingOff) {
this.startRecording();
const tidyUp = () => {
this.dlg.on('destroy', this._callerHungup.bind(this));
this.wrapDialog(this.dlg);
this.dlg.callSid = this.callSid;
this.emit('callStatusChange', {sipStatus: 200, sipReason: 'OK', callStatus: CallStatus.InProgress});
if (this.recordOptions && this.recordState === RecordState.RecordingOff) {
this.startRecording();
}
this.dlg.on('modify', this._onReinvite.bind(this));
this.dlg.on('refer', this._onRefer.bind(this));
if (this.sipRequestWithinDialogHook) {
this.dlg.on('info', this._onRequestWithinDialog.bind(this));
this.dlg.on('message', this._onRequestWithinDialog.bind(this));
}
this.logger.debug(`CallSession:propagateAnswer - answered callSid ${this.callSid}`);
};
if (this.is3pccInvite) {
return new Promise((resolve, reject) => {
this.dlg.once('ack', (ackRequest) => {
this.logger.debug({body: ackRequest.body}, 'received ACK for 3pcc invite');
this.ep.modify(ackRequest.body)
.then(() => {
tidyUp();
return resolve();
})
.catch((err) => {
this.logger.info({err}, 'Error modifying endpoint with ACK');
reject(err);
});
});
this.logger.debug('this is a 3pcc invite');
});
}
this.dlg.on('modify', this._onReinvite.bind(this));
this.dlg.on('refer', this._onRefer.bind(this));
if (this.sipRequestWithinDialogHook) {
this.dlg.on('info', this._onRequestWithinDialog.bind(this));
this.dlg.on('message', this._onRequestWithinDialog.bind(this));
}
this.logger.debug(`CallSession:propagateAnswer - answered callSid ${this.callSid}`);
tidyUp();
}
else {
this.logger.debug('CallSession:propagateAnswer - call already answered - re-anchor media with a reinvite');
@@ -2318,6 +2526,12 @@ Duration=${duration} `
try {
const to = parseUri(req.getParsedHeader('Refer-To').uri);
const by = parseUri(req.getParsedHeader('Referred-By').uri);
const customHeaders = Object.keys(req.headers)
.filter((h) => h.toLowerCase().startsWith('x-'))
.reduce((acc, h) => {
acc[h] = req.get(h);
return acc;
}, {});
const b3 = this.b3;
const httpHeaders = b3 && {b3};
const json = await this.requestor.request('verb:hook', this._referHook, {
@@ -2330,6 +2544,7 @@ Duration=${duration} `
referred_by_user: by.scheme === 'tel' ? by.number : by.user,
referring_call_sid: this.callSid,
referred_call_sid: null,
...customHeaders
}
}, httpHeaders);
@@ -2505,27 +2720,35 @@ Duration=${duration} `
};
}
async releaseMediaToSBC(remoteSdp) {
async releaseMediaToSBC(remoteSdp, releaseMediaEntirely) {
assert(this.dlg && this.dlg.connected && this.ep && typeof remoteSdp === 'string');
await this.dlg.modify(remoteSdp, {
headers: {
'X-Reason': 'release-media'
'X-Reason': releaseMediaEntirely ? 'release-media-entirely' : 'release-media'
}
});
this.ep.destroy()
.then(() => this.ep = null)
.catch((err) => this.logger.error({err}, 'CallSession:releaseMediaToSBC: Error destroying endpoint'));
try {
await this.ep.destroy();
} catch (err) {
this.logger.error({err}, 'CallSession:releaseMediaToSBC: Error destroying endpoint');
}
this.ep = null;
}
async reAnchorMedia() {
async reAnchorMedia(currentMediaRoute = MediaPath.PartialMedia) {
assert(this.dlg && this.dlg.connected && !this.ep);
this.ep = await this.ms.createEndpoint({remoteSdp: this.dlg.remote.sdp});
this._configMsEndpoint();
await this.dlg.modify(this.ep.local.sdp, {
headers: {
'X-Reason': 'anchor-media'
}
});
this._configMsEndpoint();
if (currentMediaRoute === MediaPath.NoMedia) {
await this.ep.modify(this.dlg.remote.sdp);
}
}
async handleReinviteAfterMediaReleased(req, res) {
@@ -2592,15 +2815,40 @@ Duration=${duration} `
}
_configMsEndpoint() {
this._enableInbandDtmfIfRequired(this.ep);
this.ep.once('destroy', this._handleMediaTimeout.bind(this));
const opts = {
...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'})
...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'}),
...(JAMBONES_MEDIA_TIMEOUT_MS && {media_timeout: JAMBONES_MEDIA_TIMEOUT_MS}),
...(JAMBONES_MEDIA_HOLD_TIMEOUT_MS && {media_hold_timeout: JAMBONES_MEDIA_HOLD_TIMEOUT_MS})
};
if (Object.keys(opts).length > 0) {
this.ep.set(opts);
}
}
async _handleMediaTimeout(evt) {
if (evt.reason === 'MEDIA_TIMEOUT' && !this.callGone) {
this.logger.info('CallSession:_handleMediaTimeout: received MEDIA_TIMEOUT, hangup the call');
this._jambonzHangup('Media Timeout');
}
}
async _enableInbandDtmfIfRequired(ep) {
if (ep.inbandDtmfEnabled) return;
// only enable inband dtmf detection if voip carrier dtmf_type === tones
if (this.inbandDtmfEnabled) {
// https://developer.signalwire.com/freeswitch/FreeSWITCH-Explained/Modules/mod-dptools/6587132/#0-about
try {
ep.execute('start_dtmf');
ep.inbandDtmfEnabled = true;
} catch (err) {
this.logger.info(err, 'CallSession:_enableInbandDtmf - error enable inband DTMF');
}
}
}
/**
* notifyTaskError - only used when websocket connection is used instead of webhooks
*/
@@ -2680,6 +2928,58 @@ Duration=${duration} `
this.verbHookSpan = null;
}
}
_onTtsStreamingEmpty() {
const task = this.currentTask;
if (task && TaskName.Say === task.name) {
task.notifyTtsStreamIsEmpty();
}
}
_onTtsStreamingPause() {
this.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_paused'})
.catch((err) => this.logger.info({err}, 'CallSession:_onTtsStreamingPause - Error sending'));
}
_onTtsStreamingResume() {
this.requestor?.request('tts:streaming-event', 'streaming-event', {event_type: 'stream_resumed'})
.catch((err) => this.logger.info({err}, 'CallSession:_onTtsStreamingResume - Error sending'));
}
async _onTtsStreamingConnectFailure(vendor) {
const {writeAlerts, AlertType} = this.srf.locals;
try {
await writeAlerts({
alert_type: AlertType.TTS_STREAMING_CONNECTION_FAILURE,
account_sid: this.accountSid,
vendor
});
} catch (error) {
this.logger.error({error}, 'Error writing WEBHOOK_CONNECTION_FAILURE alert');
}
this.logger.info({vendor}, 'CallSession:_onTtsStreamingConnectFailure - tts streaming connect failure');
}
async startMaxCallDurationTimer(timeLimit) {
if (!this._maxCallDurationTimer && timeLimit > 0) {
this.timeLimit = timeLimit;
this._maxCallDurationTimer = setTimeout(this._onMaxCallDuration.bind(this), timeLimit * 1000);
this.logger.debug(`CallSession:startMaxCallDurationTimer - started max call duration timer for ${timeLimit}s`);
}
}
/**
* _onMaxCallDuration - called when the call has reached the maximum duration
*/
_onMaxCallDuration() {
this.logger.info(`callSession:_onMaxCallDuration tearing down call as it has reached ${this.timeLimit}s`);
if (!this.dlg) {
this.logger.debug('CallSession:_onMaxCallDuration - no dialog, call already gone');
return;
}
this._jambonzHangup('Max Call Duration');
this._maxCallDurationTimer = null;
}
}
module.exports = CallSession;

View File

@@ -8,7 +8,7 @@ const CallSession = require('./call-session');
*/
class ConfirmCallSession extends CallSession {
constructor({logger, application, dlg, ep, tasks, callInfo, accountInfo, memberId, confName, rootSpan}) {
constructor({logger, application, dlg, ep, tasks, callInfo, accountInfo, memberId, confName, rootSpan, req}) {
super({
logger,
application,
@@ -23,6 +23,7 @@ class ConfirmCallSession extends CallSession {
});
this.dlg = dlg;
this.ep = ep;
this.req = req;
}
/**

View File

@@ -70,8 +70,12 @@ class InboundCallSession extends CallSession {
this._hangup('caller');
}
_jambonzHangup() {
this.dlg?.destroy();
_jambonzHangup(reason) {
this.dlg?.destroy({
headers: {
...(reason && {'X-Reason': reason})
}
});
// kill current task or wakeup the call session.
this._callReleased();
}

View File

@@ -653,7 +653,8 @@ class Conference extends Task {
memberId: this.memberId,
confName: this.confName,
tasks,
rootSpan: cs.rootSpan
rootSpan: cs.rootSpan,
req: cs.req
});
await this._playSession.exec();
this._playSession = null;

View File

@@ -16,7 +16,8 @@ class TaskConfig extends Task {
'fillerNoise',
'actionHookDelayAction',
'boostAudioSignal',
'vad'
'vad',
'ttsStream'
].forEach((k) => this[k] = this.data[k] || {});
if ('notifyEvents' in this.data) {
@@ -45,6 +46,12 @@ class TaskConfig extends Task {
};
delete this.transcribeOpts.enable;
}
if (this.ttsStream.enable) {
this.sayOpts = {
verb: 'say',
stream: true
};
}
if (this.data.reset) {
if (typeof this.data.reset === 'string') this.data.reset = [this.data.reset];
@@ -75,6 +82,7 @@ class TaskConfig extends Task {
get hasVad() { return Object.keys(this.vad).length; }
get hasFillerNoise() { return Object.keys(this.fillerNoise).length; }
get hasReferHook() { return Object.keys(this.data).includes('referHook'); }
get hasTtsStream() { return Object.keys(this.ttsStream).length; }
get summary() {
const phrase = [];
@@ -106,6 +114,9 @@ class TaskConfig extends Task {
if (this.onHoldMusic) phrase.push(`onHoldMusic: ${this.onHoldMusic}`);
if ('boostAudioSignal' in this.data) phrase.push(`setGain ${this.data.boostAudioSignal}`);
if (this.hasReferHook) phrase.push('set referHook');
if (this.hasTtsStream) {
phrase.push(`${this.ttsStream.enable ? 'enable' : 'disable'} ttsStream`);
}
return `${this.name}{${phrase.join(',')}}`;
}
@@ -305,6 +316,22 @@ class TaskConfig extends Task {
if (this.hasReferHook) {
cs.referHook = this.data.referHook;
}
if (this.ttsStream.enable && this.sayOpts) {
this.sayOpts.synthesizer = this.hasSynthesizer ? this.synthesizer : {
vendor: cs.speechSynthesisVendor,
language: cs.speechSynthesisLanguage,
voice: cs.speechSynthesisVoice,
...(cs.speechSynthesisLabel && {
label: cs.speechSynthesisLabel
})
};
this.logger.info({opts: this.gatherOpts}, 'Config: enabling ttsStream');
cs.enableBackgroundTtsStream(this.sayOpts);
} else if (!this.ttsStream.enable) {
this.logger.info('Config: disabling ttsStream');
cs.disableTtsStream();
}
}
async kill(cs) {

View File

@@ -6,6 +6,7 @@ const {
TaskName,
TaskPreconditions,
MAX_SIMRINGS,
MediaPath,
KillReason
} = require('../utils/constants');
const assert = require('assert');
@@ -108,6 +109,7 @@ class TaskDial extends Task {
this.proxy = this.data.proxy;
this.tag = this.data.tag;
this.boostAudioSignal = this.data.boostAudioSignal;
this._mediaPath = MediaPath.FullMedia;
if (this.dtmfHook) {
const {parentDtmfCollector, childDtmfCollector} = parseDtmfOptions(logger, this.data.dtmfCapture || {});
@@ -155,17 +157,21 @@ class TaskDial extends Task {
get canReleaseMedia() {
const keepAnchor = this.data.anchorMedia ||
this.cs.isBackGroundListen ||
this.cs.onHoldMusic ||
ANCHOR_MEDIA_ALWAYS ||
this.listenTask ||
this.dubTasks ||
this.transcribeTask ||
this.startAmd;
this.cs.isBackGroundListen ||
this.cs.onHoldMusic ||
ANCHOR_MEDIA_ALWAYS ||
this.listenTask ||
this.dubTasks ||
this.transcribeTask ||
this.startAmd;
return !keepAnchor;
}
get shouldExitMediaPathEntirely() {
return this.data.exitMediaPath;
}
get summary() {
if (this.target.length === 1) {
const target = this.target[0];
@@ -186,6 +192,16 @@ class TaskDial extends Task {
async exec(cs) {
await super.exec(cs);
if (this.data.anchorMedia && this.data.exitMediaPath) {
this.logger.info('Dial:exec - incompatible anchorMedia and exitMediaPath are both set, will obey anchorMedia');
delete this.data.exitMediaPath;
}
if (!this.canReleaseMedia && this.data.exitMediaPath) {
this.logger.info(
'Dial:exec - exitMediaPath is set so features such as transcribe and record will not work on this call');
}
try {
if (this.listenTask) {
const {span, ctx} = this.startChildSpan(`nested:${this.listenTask.summary}`);
@@ -257,7 +273,9 @@ class TaskDial extends Task {
this._removeDtmfDetection(this.dlg);
await this._killOutdials();
if (this.sd) {
this.sd.kill();
const byeReasonHeader = this.killReason === KillReason.MediaTimeout ? 'Media Timeout' : undefined;
this.sd.kill(byeReasonHeader);
this.sd.ep?.removeListener('destroy', this._handleMediaTimeout.bind(this));
this.sd.removeAllListeners();
this.sd = null;
}
@@ -303,7 +321,7 @@ class TaskDial extends Task {
if (!cs.callGone && this.epOther) {
/* if we can release the media back to the SBC, do so now */
if (this.canReleaseMedia) this._releaseMedia(cs, this.sd);
if (this.canReleaseMedia) this._releaseMedia(cs, this.sd, this.shouldExitMediaPathEntirely);
else this.epOther.bridge(this.ep);
}
} catch (err) {
@@ -345,6 +363,12 @@ class TaskDial extends Task {
const by = parseUri(req.getParsedHeader('Referred-By').uri);
const referredBy = req.get('Referred-By');
const userAgent = req.get('User-Agent');
const customHeaders = Object.keys(req.headers)
.filter((h) => h.toLowerCase().startsWith('x-'))
.reduce((acc, h) => {
acc[h] = req.get(h);
return acc;
}, {});
this.logger.info({to}, 'refer to parsed');
const json = await cs.requestor.request('verb:hook', this.referHook, {
...(callInfo.toJSON()),
@@ -355,7 +379,8 @@ class TaskDial extends Task {
...(userAgent && {sip_user_agent: userAgent}),
...(by && {referred_by_user: by.scheme === 'tel' ? by.number : by.user}),
referring_call_sid,
referred_call_sid
referred_call_sid,
...customHeaders
}
}, httpHeaders);
if (json && Array.isArray(json)) {
@@ -496,7 +521,7 @@ class TaskDial extends Task {
const {req, callInfo, direction, srf} = cs;
const {getSBC} = srf.locals;
const {lookupTeamsByAccount, lookupAccountBySid} = srf.locals.dbHelpers;
const {lookupCarrier, lookupCarrierByPhoneNumber} = dbUtils(this.logger, cs.srf);
const {lookupCarrier, lookupCarrierByPhoneNumber, lookupVoipCarrierBySid} = dbUtils(this.logger, cs.srf);
let sbcAddress = this.proxy || getSBC();
const teamsInfo = {};
let fqdn;
@@ -515,6 +540,8 @@ class TaskDial extends Task {
...this.headers
};
// default to inband dtmf if not specified
this.inbandDtmfEnabled = cs.inbandDtmfEnabled;
// get calling user from From header
const parsedFrom = req.getParsedHeader('from');
const fromUri = parseUri(parsedFrom.uri);
@@ -592,10 +619,17 @@ class TaskDial extends Task {
const str = this.callerId || req.callingNumber || '';
const callingNumber = str.startsWith('+') ? str.substring(1) : str;
const voip_carrier_sid = await lookupCarrierByPhoneNumber(cs.accountSid, callingNumber);
const req_voip_carrier_sid = req.has('X-Voip-Carrier-Sid') ? req.get('X-Voip-Carrier-Sid') : null;
if (voip_carrier_sid) {
this.logger.info(
`Dial:_attemptCalls: selected voip_carrier_sid ${voip_carrier_sid} for callingNumber: ${callingNumber}`);
opts.headers['X-Requested-Carrier-Sid'] = voip_carrier_sid;
// Checking if outbound carrier is different from inbound carrier and has dtmf type tones
if (voip_carrier_sid !== req_voip_carrier_sid) {
const [voipCarrier] = await lookupVoipCarrierBySid(voip_carrier_sid);
this.inbandDtmfEnabled = voipCarrier?.dtmf_type === 'tones';
}
}
}
@@ -752,7 +786,7 @@ class TaskDial extends Task {
// Offhold, time to release media
const newSdp = await this.ep.modify(req.body);
await res.send(200, {body: newSdp});
await this._releaseMedia(this.cs, this.sd);
await this._releaseMedia(this.cs, this.sd, this.shouldExitMediaPathEntirely);
this.isOutgoingLegHold = false;
} else {
this.logger.debug('Dial: _onReinvite receive unhold Request, update media server');
@@ -861,7 +895,17 @@ class TaskDial extends Task {
}
/* if we can release the media back to the SBC, do so now */
if (this.canReleaseMedia) setTimeout(this._releaseMedia.bind(this, cs, sd), 200);
if (this.canReleaseMedia || this.shouldExitMediaPathEntirely) {
setTimeout(this._releaseMedia.bind(this, cs, sd, this.shouldExitMediaPathEntirely), 200);
}
this.sd.ep.once('destroy', this._handleMediaTimeout.bind(this));
}
_handleMediaTimeout(evt) {
if (evt.reason === 'MEDIA_TIMEOUT' && this.sd && this.bridged) {
this.kill(this.cs, KillReason.MediaTimeout);
}
}
_bridgeEarlyMedia(sd) {
@@ -873,22 +917,57 @@ class TaskDial extends Task {
}
}
/* public api */
async updateMediaPath(desiredPath) {
this.logger.info(`Dial:updateMediaPath - ${this._mediaPath} => ${desiredPath}`);
switch (desiredPath) {
case MediaPath.NoMedia:
assert(this._mediaPath !== MediaPath.NoMedia, 'updateMediaPath: already no-media');
await this._releaseMedia(this.cs, this.sd, true);
break;
case MediaPath.PartialMedia:
assert(this._mediaPath !== MediaPath.PartialMedia, 'updateMediaPath: already partial-media');
if (this._mediaPath === MediaPath.FullMedia) {
await this._releaseMedia(this.cs, this.sd, false);
}
else {
// to go from no-media to partial-media we need to go through full-media first
await this.reAnchorMedia(this.cs, this.sd);
await this._releaseMedia(this.cs, this.sd, false);
}
assert(!this.epOther, 'updateMediaPath: epOther should be null');
assert(!this.ep, 'updateMediaPath: ep should be null');
break;
case MediaPath.FullMedia:
assert(this._mediaPath !== MediaPath.FullMedia, 'updateMediaPath: already full-media');
await this.reAnchorMedia(this.cs, this.sd);
break;
default:
assert(false, `updateMediaPath: invalid path request ${desiredPath}`);
}
}
/**
* Release the media from freeswitch
* @param {*} cs
* @param {*} sd
*/
async _releaseMedia(cs, sd) {
async _releaseMedia(cs, sd, releaseEntirely = false) {
assert(cs.ep && sd.ep);
try {
// Wait until we got new SDP from B leg to ofter to A Leg
const aLegSdp = cs.ep.remote.sdp;
await sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp);
await sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp, releaseEntirely);
const bLegSdp = sd.dlg.remote.sdp;
await cs.releaseMediaToSBC(bLegSdp);
await cs.releaseMediaToSBC(bLegSdp, releaseEntirely);
this.epOther = null;
this.logger.info('Dial:_releaseMedia - successfully released media from freewitch');
this._mediaPath = releaseEntirely ? MediaPath.NoMedia : MediaPath.PartialMedia;
this.logger.info(
`Dial:_releaseMedia - successfully released media from freewitch, media path is now ${this._mediaPath}`);
} catch (err) {
this.logger.info({err}, 'Dial:_releaseMedia error');
}
@@ -898,8 +977,14 @@ class TaskDial extends Task {
if (cs.ep && sd.ep) return;
this.logger.info('Dial:reAnchorMedia - re-anchoring media to freewitch');
await Promise.all([sd.reAnchorMedia(), cs.reAnchorMedia()]);
await Promise.all([sd.reAnchorMedia(this._mediaPath), cs.reAnchorMedia(this._mediaPath)]);
this.epOther = cs.ep;
this.epOther.bridge(this.ep);
this._mediaPath = MediaPath.FullMedia;
this.logger.info(
`Dial:_releaseMedia - successfully re-anchored media to freewitch, media path is now ${this._mediaPath}`);
}
// Handle RE-INVITE hold from caller leg.
@@ -918,11 +1003,12 @@ class TaskDial extends Task {
}
this._onHoldHook(req);
} else if (!isOnhold(req.body)) {
if (this.epOther && this.ep && this.isIncomingLegHold && this.canReleaseMedia) {
if (this.epOther && this.ep && this.isIncomingLegHold &&
(this.canReleaseMedia || this.shouldExitMediaPathEntirely)) {
// Offhold, time to release media
const newSdp = await this.epOther.modify(req.body);
await res.send(200, {body: newSdp});
await this._releaseMedia(this.cs, this.sd);
await this._releaseMedia(this.cs, this.sd, this.shouldExitMediaPathEntirely);
isHandled = true;
}
this.isIncomingLegHold = false;
@@ -981,7 +1067,8 @@ class TaskDial extends Task {
callInfo: this.cs.callInfo,
accountInfo: this.cs.accountInfo,
tasks,
rootSpan: this.cs.rootSpan
rootSpan: this.cs.rootSpan,
req: this.cs.req
});
await this._onHoldSession.exec();
this._onHoldSession = null;

View File

@@ -369,7 +369,8 @@ class TaskEnqueue extends Task {
callInfo: cs.callInfo,
accountInfo: cs.accountInfo,
tasks: tasksToRun,
rootSpan: cs.rootSpan
rootSpan: cs.rootSpan,
req: cs.req
});
await this._playSession.exec();
this._playSession = null;

View File

@@ -155,7 +155,8 @@ class TaskGather extends SttTask {
const {hints, hintsBoost} = cs.globalSttHints;
const setOfHints = new Set((this.data.recognizer.hints || [])
.concat(hints)
.filter((h) => typeof h === 'string' && h.length > 0));
// allow for hints to be an array of object
.filter((h) => (typeof h === 'string' && h.length > 0) || (typeof h === 'object')));
this.data.recognizer.hints = [...setOfHints];
if (!this.data.recognizer.hintsBoost && hintsBoost) this.data.recognizer.hintsBoost = hintsBoost;
this.logger.debug({hints: this.data.recognizer.hints, hintsBoost: this.data.recognizer.hintsBoost},
@@ -342,6 +343,13 @@ class TaskGather extends SttTask {
this._killAudio(cs);
this.emit('dtmf', evt);
}
if (this.isContinuousAsr && evt.dtmf === this.asrDtmfTerminationDigit && this._bufferedTranscripts.length > 0) {
this.logger.info(`continuousAsr triggered with dtmf ${this.asrDtmfTerminationDigit}`);
this._clearAsrTimer();
this._clearTimer();
this._startFinalAsrTimer();
return;
}
if (evt.dtmf === this.finishOnKey && this.input.includes('digits')) {
resolved = true;
this._resolve('dtmf-terminator-key');
@@ -364,13 +372,6 @@ class TaskGather extends SttTask {
this._resolve('dtmf-num-digits');
}
}
else if (this.isContinuousAsr && evt.dtmf === this.asrDtmfTerminationDigit) {
this.logger.info(`continuousAsr triggered with dtmf ${this.asrDtmfTerminationDigit}`);
this._clearAsrTimer();
this._clearTimer();
this._startFinalAsrTimer();
return;
}
if (!resolved && this.interDigitTimeout > 0 && this.digitBuffer.length >= this.minDigits) {
/* start interDigitTimer */
const ms = this.interDigitTimeout * 1000;
@@ -722,6 +723,7 @@ class TaskGather extends SttTask {
this._fillerNoiseOn = false; // in a race, if we just started audio it may sneak through here
this.ep.api('uuid_break', this.ep.uuid)
.catch((err) => this.logger.info(err, 'Error killing audio'));
cs.clearTtsStream();
}
return;
}
@@ -846,7 +848,10 @@ class TaskGather extends SttTask {
this._startAsrTimer();
/* some STT engines will keep listening after a final response, so no need to restart */
if (!['soniox', 'aws', 'microsoft', 'deepgram'].includes(this.vendor)) this._startTranscribing(ep);
if (!['soniox', 'aws', 'microsoft', 'deepgram', 'speechmatics'].includes(this.vendor) &&
!this.vendor.startsWith('custom')) {
this._startTranscribing(ep);
}
}
else {
/* this was removed to fix https://github.com/jambonz/jambonz-feature-server/issues/783 */
@@ -1166,7 +1171,6 @@ class TaskGather extends SttTask {
} catch (err) { /*already logged error*/ }
// Gather got response from hook, cancel actionHookDelay processing
this.logger.debug('TaskGather:_resolve - checking ahd');
if (this.cs.actionHookDelayProcessor) {
if (returnedVerbs) {
this.logger.debug('TaskGather:_resolve - got response from action hook, cancelling actionHookDelay');

View File

@@ -1,6 +1,7 @@
const Task = require('../task');
const {TaskPreconditions} = require('../../utils/constants');
const TaskLlmOpenAI_S2S = require('./llms/openai_s2s');
const TaskLlmVoiceAgent_S2S = require('./llms/voice_agent_s2s');
class TaskLlm extends Task {
constructor(logger, opts) {
@@ -40,10 +41,14 @@ class TaskLlm extends Task {
switch (this.vendor) {
case 'openai':
case 'microsoft':
if (this.model.startsWith('gpt-4o-realtime')) {
llm = new TaskLlmOpenAI_S2S(this.logger, this.data, this);
}
llm = new TaskLlmOpenAI_S2S(this.logger, this.data, this);
break;
case 'voiceagent':
case 'deepgram':
llm = new TaskLlmVoiceAgent_S2S(this.logger, this.data, this);
break;
default:
throw new Error(`Unsupported vendor ${this.vendor} for LLM`);
}

View File

@@ -59,7 +59,7 @@ class TaskLlmOpenAI_S2S extends Task {
this.parent = parentTask;
this.vendor = this.parent.vendor;
this.model = this.parent.model;
this.model = this.parent.model || 'gpt-4o-realtime-preview-2024-12-17';
this.auth = this.parent.auth;
this.connectionOptions = this.parent.connectOptions;
@@ -120,9 +120,9 @@ class TaskLlmOpenAI_S2S extends Task {
switch (this.vendor) {
case 'openai':
return 'v1/realtime?model=gpt-4o-realtime-preview-2024-10-01';
return 'v1/realtime?model=${this.model}';
case 'microsoft':
return 'openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-1001&';
return `openai/realtime?api-version=2024-10-01-preview&deployment=${this.model}`;
}
}

View File

@@ -0,0 +1,313 @@
const Task = require('../../task');
const TaskName = 'Llm_VoiceAgent_s2s';
const {LlmEvents_VoiceAgent} = require('../../../utils/constants');
const ClientEvent = 'client.event';
const SessionDelete = 'session.delete';
const va_server_events = [
'Error',
'Welcome',
'SettingsApplied',
'ConversationText',
'UserStartedSpeaking',
'EndOfThought',
'AgentThinking',
'FunctionCallRequest',
'FunctionCalling',
'AgentStartedSpeaking',
'AgentAudioDone',
];
const expandWildcards = (events) => {
// no-op for deepgram
return events;
};
class TaskLlmVoiceAgent_S2S extends Task {
constructor(logger, opts, parentTask) {
super(logger, opts, parentTask);
this.parent = parentTask;
this.vendor = this.parent.vendor;
this.model = this.parent.model;
this.auth = this.parent.auth;
this.connectionOptions = this.parent.connectOptions;
const {apiKey} = this.auth || {};
if (!apiKey) throw new Error('auth.apiKey is required for VoiceAgent S2S');
this.apiKey = apiKey;
this.authType = 'bearer';
this.actionHook = this.data.actionHook;
this.eventHook = this.data.eventHook;
this.toolHook = this.data.toolHook;
const {settingsConfiguration} = this.data.llmOptions;
if (typeof settingsConfiguration !== 'object') {
throw new Error('llmOptions with an initial settingsConfiguration is required for VoiceAgent S2S');
}
// eslint-disable-next-line no-unused-vars
const {audio, ...rest} = settingsConfiguration;
const cfg = this.settingsConfiguration = rest;
if (!cfg.agent) throw new Error('llmOptions.settingsConfiguration.agent is required for VoiceAgent S2S');
if (!cfg.agent.think) {
throw new Error('llmOptions.settingsConfiguration.agent.think is required for VoiceAgent S2S');
}
if (!cfg.agent.think.model) {
throw new Error('llmOptions.settingsConfiguration.agent.think.model is required for VoiceAgent S2S');
}
if (!cfg.agent.think.provider?.type) {
throw new Error('llmOptions.settingsConfiguration.agent.think.provider.type is required for VoiceAgent S2S');
}
this.results = {
completionReason: 'normal conversation end'
};
/**
* only one of these will have items,
* if includeEvents, then these are the events to include
* if excludeEvents, then these are the events to exclude
*/
this.includeEvents = [];
this.excludeEvents = [];
/* default to all events if user did not specify */
this._populateEvents(this.data.events || va_server_events);
this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
}
get name() { return TaskName; }
get host() {
const {host} = this.connectionOptions || {};
return host || 'agent.deepgram.com';
}
get path() {
const {path} = this.connectionOptions || {};
if (path) return path;
return '/agent';
}
async _api(ep, args) {
const res = await ep.api('uuid_voice_agent_s2s', `^^|${args.join('|')}`);
if (!res.body?.startsWith('+OK')) {
throw new Error(`Error calling uuid_voice_agent_s2s: ${JSON.stringify(res.body)}`);
}
}
async exec(cs, {ep}) {
await super.exec(cs);
await this._startListening(cs, ep);
await this.awaitTaskDone();
/* note: the parent llm verb started the span, which is why this is necessary */
await this.parent.performAction(this.results);
this._unregisterHandlers();
}
async kill(cs) {
super.kill(cs);
this._api(cs.ep, [cs.ep.uuid, SessionDelete])
.catch((err) => this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:kill - error deleting session'));
this.notifyTaskDone();
}
/**
* Send function call response to the VoiceAgent server
*/
async processToolOutput(ep, tool_call_id, data) {
try {
const {data:response} = data;
this.logger.debug({tool_call_id, response}, 'TaskLlmVoiceAgent_S2S:processToolOutput');
if (!response.type || response.type !== 'FunctionCallResponse') {
this.logger.info({response},
'TaskLlmVoiceAgent_S2S:processToolOutput - invalid tool output, must be FunctionCallResponse');
}
else {
await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(response)]);
}
} catch (err) {
this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:processToolOutput');
}
}
/**
* Send a session.update to the VoiceAgent server
* Note: creating and deleting conversation items also supported as well as interrupting the assistant
*/
async processLlmUpdate(ep, data, _callSid) {
try {
this.logger.debug({data, _callSid}, 'TaskLlmVoiceAgent_S2S:processLlmUpdate');
if (!data.type || ![
'UpdateInstructions',
'UpdateSpeak',
'InjectAgentMessage',
].includes(data.type)) {
this.logger.info({data}, 'TaskLlmVoiceAgent_S2S:processLlmUpdate - invalid mid-call request');
}
else {
await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
}
} catch (err) {
this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:processLlmUpdate');
}
}
async _startListening(cs, ep) {
this._registerHandlers(ep);
try {
const args = [ep.uuid, 'session.create', this.host, this.path, this.authType, this.apiKey];
await this._api(ep, args);
} catch (err) {
this.logger.error({err}, `TaskLlmVoiceAgent_S2S:_startListening: ${JSON.stringify(err)}`);
this.notifyTaskDone();
}
}
async _sendClientEvent(ep, obj) {
let ok = true;
this.logger.debug({obj}, 'TaskLlmVoiceAgent_S2S:_sendClientEvent');
try {
const args = [ep.uuid, ClientEvent, JSON.stringify(obj)];
await this._api(ep, args);
} catch (err) {
ok = false;
this.logger.error({err}, 'TaskLlmVoiceAgent_S2S:_sendClientEvent - Error');
}
return ok;
}
async _sendInitialMessage(ep) {
if (!await this._sendClientEvent(ep, this.settingsConfiguration)) {
this.notifyTaskDone();
}
}
_registerHandlers(ep) {
this.addCustomEventListener(ep, LlmEvents_VoiceAgent.Connect, this._onConnect.bind(this, ep));
this.addCustomEventListener(ep, LlmEvents_VoiceAgent.ConnectFailure, this._onConnectFailure.bind(this, ep));
this.addCustomEventListener(ep, LlmEvents_VoiceAgent.Disconnect, this._onDisconnect.bind(this, ep));
this.addCustomEventListener(ep, LlmEvents_VoiceAgent.ServerEvent, this._onServerEvent.bind(this, ep));
}
_unregisterHandlers() {
this.removeCustomEventListeners();
}
_onError(_ep, evt) {
this.logger.info({evt}, 'TaskLlmVoiceAgent_S2S:_onError');
this.notifyTaskDone();
}
_onConnect(ep) {
this.logger.debug('TaskLlmVoiceAgent_S2S:_onConnect');
this._sendInitialMessage(ep);
}
_onConnectFailure(_ep, evt) {
this.logger.info(evt, 'TaskLlmVoiceAgent_S2S:_onConnectFailure');
this.results = {completionReason: 'connection failure'};
this.notifyTaskDone();
}
_onDisconnect(_ep, evt) {
this.logger.info(evt, 'TaskLlmVoiceAgent_S2S:_onConnectFailure');
this.results = {completionReason: 'disconnect from remote end'};
this.notifyTaskDone();
}
async _onServerEvent(_ep, evt) {
let endConversation = false;
const type = evt.type;
this.logger.info({evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent');
/* check for failures, such as rate limit exceeded, that should terminate the conversation */
if (type === 'response.done' && evt.response.status === 'failed') {
endConversation = true;
this.results = {
completionReason: 'server failure',
error: evt.response.status_details?.error
};
}
/* server errors of some sort */
else if (type === 'error') {
endConversation = true;
this.results = {
completionReason: 'server error',
error: evt.error
};
}
/* tool calls */
else if (type === 'FunctionCallRequest') {
this.logger.debug({evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - function_call');
if (!this.toolHook) {
this.logger.warn({evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - no toolHook defined!');
}
else {
const {function_name:name, function_call_id:call_id} = evt;
const args = evt.input;
try {
await this.parent.sendToolHook(call_id, {name, args});
} catch (err) {
this.logger.info({err, evt}, 'TaskLlmVoiceAgent - error calling function');
this.results = {
completionReason: 'client error calling function',
error: err
};
endConversation = true;
}
}
}
/* check whether we should notify on this event */
if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
this.parent.sendEventHook(evt)
.catch((err) => this.logger.info({err}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - error sending event hook'));
}
if (endConversation) {
this.logger.info({results: this.results},
'TaskLlmVoiceAgent_S2S:_onServerEvent - ending conversation due to error');
this.notifyTaskDone();
}
}
_populateEvents(events) {
if (events.includes('all')) {
/* work by excluding specific events */
const exclude = events
.filter((evt) => evt.startsWith('-'))
.map((evt) => evt.slice(1));
if (exclude.length === 0) this.includeEvents = va_server_events;
else this.excludeEvents = expandWildcards(exclude);
}
else {
/* work by including specific events */
const include = events
.filter((evt) => !evt.startsWith('-'));
this.includeEvents = expandWildcards(include);
}
this.logger.debug({
includeEvents: this.includeEvents,
excludeEvents: this.excludeEvents
}, 'TaskLlmVoiceAgent_S2S:_populateEvents');
}
}
module.exports = TaskLlmVoiceAgent_S2S;

View File

@@ -1,5 +1,6 @@
const Task = require('./task');
const {TaskName, TaskPreconditions} = require('../utils/constants');
const { PlayFileNotFoundError } = require('../utils/error');
class TaskPlay extends Task {
constructor(logger, opts) {
@@ -66,8 +67,20 @@ class TaskPlay extends Task {
}
}
} catch (err) {
if (timeout) clearTimeout(timeout);
this.logger.info(err, `TaskPlay:exec - error playing ${this.url}`);
this.logger.info(`TaskPlay:exec - error playing ${this.url}: ${err.message}`);
this.playComplete = true;
if (err.message === 'File Not Found') {
const {writeAlerts, AlertType} = cs.srf.locals;
await this.performAction({status: 'fail', reason: 'playFailed'}, !(this.parentTask || cs.isConfirmCallSession));
this.emit('playDone');
writeAlerts({
account_sid: cs.accountSid,
alert_type: AlertType.PLAY_FILENOTFOUND,
url: this.url,
target_sid: cs.callSid
});
throw new PlayFileNotFoundError(this.url);
}
}
this.emit('playDone');
}

View File

@@ -1,5 +1,6 @@
const Task = require('./task');
const {TaskName} = require('../utils/constants');
const WsRequestor = require('../utils/ws-requestor');
/**
* Redirects to a new application
@@ -13,6 +14,17 @@ class TaskRedirect extends Task {
async exec(cs) {
await super.exec(cs);
if (cs.requestor instanceof WsRequestor && cs.application.requestor._isAbsoluteUrl(this.actionHook)) {
this.logger.info(`Task:performAction redirecting to ${this.actionHook}, requires new ws connection`);
try {
this.cs.requestor.close();
const requestor = new WsRequestor(this.logger, cs.accountSid, {url: this.actionHook}, this.webhook_secret) ;
this.cs.application.requestor = requestor;
} catch (err) {
this.logger.info(err, `Task:performAction error redirecting to ${this.actionHook}`);
}
}
await this.performAction();
}
}

View File

@@ -12,6 +12,7 @@ class TaskRestDial extends Task {
this.from = this.data.from;
this.callerName = this.data.callerName;
this.timeLimit = this.data.timeLimit;
this.fromHost = this.data.fromHost;
this.to = this.data.to;
this.call_hook = this.data.call_hook;
@@ -66,6 +67,9 @@ class TaskRestDial extends Task {
const cs = this.callSession;
cs.setDialog(dlg);
cs.referHook = this.referHook;
if (this.timeLimit) {
cs.startMaxCallDurationTimer(this.timeLimit);
}
this.logger.debug('TaskRestDial:_onConnect - call connected');
if (this.sipRequestWithinDialogHook) this._initSipRequestWithinDialogHandler(cs, dlg);
try {
@@ -77,11 +81,13 @@ class TaskRestDial extends Task {
synthesizer: {
vendor: cs.speechSynthesisVendor,
language: cs.speechSynthesisLanguage,
voice: cs.speechSynthesisVoice
voice: cs.speechSynthesisVoice,
label: cs.speechSynthesisLabel,
},
recognizer: {
vendor: cs.speechRecognizerVendor,
language: cs.speechRecognizerLanguage
language: cs.speechRecognizerLanguage,
label: cs.speechRecognizerLabel,
}
}
};

View File

@@ -1,3 +1,4 @@
const assert = require('assert');
const TtsTask = require('./tts-task');
const {TaskName, TaskPreconditions} = require('../utils/constants');
const pollySSMLSplit = require('polly-ssml-split');
@@ -35,24 +36,40 @@ class TaskSay extends TtsTask {
super(logger, opts, parentTask);
this.preconditions = TaskPreconditions.Endpoint;
this.text = (Array.isArray(this.data.text) ? this.data.text : [this.data.text])
.map((t) => breakLengthyTextIfNeeded(this.logger, t))
.flat();
assert.ok((typeof this.data.text === 'string' || Array.isArray(this.data.text)) || this.data.stream === true,
'Say: either text or stream:true is required');
this.loop = this.data.loop || 1;
this.isHandledByPrimaryProvider = true;
if (this.data.stream === true) {
this._isStreamingTts = true;
this.closeOnStreamEmpty = this.data.closeOnStreamEmpty !== false;
}
else {
this._isStreamingTts = false;
this.text = (Array.isArray(this.data.text) ? this.data.text : [this.data.text])
.map((t) => breakLengthyTextIfNeeded(this.logger, t))
.flat();
this.loop = this.data.loop || 1;
this.isHandledByPrimaryProvider = true;
}
}
get name() { return TaskName.Say; }
get summary() {
for (let i = 0; i < this.text.length; i++) {
if (this.text[i].startsWith('silence_stream')) continue;
return `${this.name}{text=${this.text[i].slice(0, 15)}${this.text[i].length > 15 ? '...' : ''}}`;
if (this.isStreamingTts) return `${this.name} streaming`;
else {
for (let i = 0; i < this.text.length; i++) {
if (this.text[i].startsWith('silence_stream')) continue;
return `${this.name}{text=${this.text[i].slice(0, 15)}${this.text[i].length > 15 ? '...' : ''}}`;
}
return `${this.name}{${this.text[0]}}`;
}
return `${this.name}{${this.text[0]}}`;
}
get isStreamingTts() { return this._isStreamingTts; }
_validateURL(urlString) {
try {
new URL(urlString);
@@ -63,14 +80,19 @@ class TaskSay extends TtsTask {
}
async exec(cs, obj) {
if (this.isStreamingTts && !cs.appIsUsingWebsockets) {
throw new Error('Say: streaming say verb requires applications to use the websocket API');
}
try {
await this.handling(cs, obj);
if (this.isStreamingTts) await this.handlingStreaming(cs, obj);
else await this.handling(cs, obj);
this.emit('playDone');
} catch (error) {
if (error instanceof SpeechCredentialError) {
// if say failed due to speech credentials, alarm is writtern and error notification is sent
// finished this say to move to next task.
this.logger.info('Say failed due to SpeechCredentialError, finished!');
this.logger.info({error}, 'Say failed due to SpeechCredentialError, finished!');
this.emit('playDone');
return;
}
@@ -78,6 +100,35 @@ class TaskSay extends TtsTask {
}
}
async handlingStreaming(cs, {ep}) {
const {vendor, language, voice, label} = this.getTtsVendorData(cs);
const credentials = cs.getSpeechCredentials(vendor, 'tts', label);
if (!credentials) {
throw new SpeechCredentialError(
`No text-to-speech service credentials for ${vendor} with labels: ${label} have been configured`);
}
try {
await this.setTtsStreamingChannelVars(vendor, language, voice, credentials, ep);
await cs.startTtsStream();
cs.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_open'})
.catch((err) => this.logger.info({err}, 'TaskSay:handlingStreaming - Error sending'));
} catch (err) {
this.logger.info({err}, 'TaskSay:handlingStreaming - Error setting channel vars');
cs.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_closed'})
.catch((err) => this.logger.info({err}, 'TaskSay:handlingStreaming - Error sending'));
//TODO: send tts:streaming-event with error?
this.notifyTaskDone();
}
await this.awaitTaskDone();
this.logger.info('TaskSay:handlingStreaming - done');
}
async handling(cs, {ep}) {
const {srf, accountSid:account_sid, callSid:target_sid} = cs;
const {writeAlerts, AlertType} = srf.locals;
@@ -96,7 +147,7 @@ class TaskSay extends TtsTask {
let voice = this.synthesizer.voice && this.synthesizer.voice !== 'default' ?
this.synthesizer.voice :
cs.speechSynthesisVoice;
let label = this.taskInlcudeSynthesizer ? this.synthesizer.label : cs.speechSynthesisLabel;
let label = this.taskIncludeSynthesizer ? this.synthesizer.label : cs.speechSynthesisLabel;
const fallbackVendor = this.synthesizer.fallbackVendor && this.synthesizer.fallbackVendor !== 'default' ?
this.synthesizer.fallbackVendor :
@@ -107,7 +158,7 @@ class TaskSay extends TtsTask {
const fallbackVoice = this.synthesizer.fallbackVoice && this.synthesizer.fallbackVoice !== 'default' ?
this.synthesizer.fallbackVoice :
cs.fallbackSpeechSynthesisVoice;
const fallbackLabel = this.taskInlcudeSynthesizer ?
const fallbackLabel = this.taskIncludeSynthesizer ?
this.synthesizer.fallbackLabel : cs.fallbackSpeechSynthesisLabel;
if (cs.hasFallbackTts) {
@@ -253,6 +304,7 @@ class TaskSay extends TtsTask {
this._playResolve = null;
}
}
this.notifyTaskDone();
}
_addStreamingTtsAttributes(span, evt) {
@@ -263,6 +315,7 @@ class TaskSay extends TtsTask {
.replace('whisper_', 'whisper.')
.replace('deepgram_', 'deepgram.')
.replace('playht_', 'playht.')
.replace('cartesia_', 'cartesia.')
.replace('rimelabs_', 'rimelabs.')
.replace('verbio_', 'verbio.')
.replace('elevenlabs_', 'elevenlabs.');
@@ -273,6 +326,13 @@ class TaskSay extends TtsTask {
delete attrs['cache_filename']; //no value in adding this to the span
span.setAttributes(attrs);
}
notifyTtsStreamIsEmpty() {
if (this.isStreamingTts && this.closeOnStreamEmpty) {
this.logger.info('TaskSay:notifyTtsStreamIsEmpty - stream is empty, killing task');
this.notifyTaskDone();
}
}
}
const spanMapping = {
@@ -309,6 +369,11 @@ const spanMapping = {
'playht.name_lookup_time_ms': 'name_lookup_ms',
'playht.connect_time_ms': 'connect_ms',
'playht.final_response_time_ms': 'final_response_ms',
// Cartesia
'cartesia.request_id': 'cartesia.req_id',
'cartesia.name_lookup_time_ms': 'name_lookup_ms',
'cartesia.connect_time_ms': 'connect_ms',
'cartesia.final_response_time_ms': 'final_response_ms',
// Rimelabs
'rimelabs.name_lookup_time_ms': 'name_lookup_ms',
'rimelabs.connect_time_ms': 'connect_ms',

View File

@@ -12,6 +12,7 @@ class TaskSipRefer extends Task {
this.referTo = this.data.referTo;
this.referredBy = this.data.referredBy;
this.referredByDisplayName = this.data.referredByDisplayName;
this.headers = this.data.headers || {};
this.eventHook = this.data.eventHook;
}
@@ -94,7 +95,10 @@ class TaskSipRefer extends Task {
}
if (status >= 200) {
this.referSpan.setAttributes({'refer.finalNotify': status});
await this.performAction({refer_status: 202, final_referred_call_status: status});
await this.performAction({refer_status: 202, final_referred_call_status: status})
.catch((err) => {
this.logger.error(err, 'TaskSipRefer:exec - error performing action finalNotify');
});
this.notifyTaskDone();
}
}
@@ -102,7 +106,7 @@ class TaskSipRefer extends Task {
}
_normalizeReferHeaders(cs, dlg) {
let {referTo, referredBy} = this;
let {referTo, referredBy, referredByDisplayName} = this;
/* get IP address of the SBC to use as hostname if needed */
const {host} = parseUri(dlg.remote.uri);
@@ -117,9 +121,12 @@ class TaskSipRefer extends Task {
referredBy = cs.req?.callingNumber || dlg.local.uri;
this.logger.info({referredBy}, 'setting referredby');
}
if (!referredByDisplayName) {
referredByDisplayName = cs.req?.callingName;
}
if (!referredBy.startsWith('<') && !referredBy.startsWith('sip') && !referredBy.startsWith('"')) {
/* they may have only provided a phone number/user */
referredBy = `sip:${referredBy}@${host}`;
referredBy = `${referredByDisplayName ? `"${referredByDisplayName}"` : ''}<sip:${referredBy}@${host}>`;
}
return {referTo, referredBy};
}

View File

@@ -219,7 +219,8 @@ class SttTask extends Task {
roleArn
});
this.logger.debug({roleArn}, `(roleArn) got aws access token ${servedFromCache ? 'from cache' : ''}`);
credentials = {...credentials, accessKeyId, secretAccessKey, sessionToken};
// from role ARN, we will get SessionToken, but feature server use it as securityToken.
credentials = {...credentials, accessKeyId, secretAccessKey, securityToken: sessionToken};
}
else if (vendor === 'verbio' && credentials.client_id && credentials.client_secret) {
const {access_token, servedFromCache} = await getVerbioAccessToken(credentials);
@@ -229,9 +230,13 @@ class SttTask extends Task {
}
else if (vendor == 'aws' && !JAMBONES_AWS_TRANSCRIBE_USE_GRPC) {
/* get AWS access token */
const {accessKeyId, secretAccessKey, securityToken, region } = credentials;
const {speech_credential_sid, accessKeyId, secretAccessKey, securityToken, region } = credentials;
if (!securityToken) {
const { servedFromCache, ...newCredentials} = await getAwsAuthToken({accessKeyId, secretAccessKey, region});
const { servedFromCache, ...newCredentials} = await getAwsAuthToken({
speech_credential_sid,
accessKeyId,
secretAccessKey,
region});
this.logger.debug({newCredentials}, `got aws security token ${servedFromCache ? 'from cache' : ''}`);
credentials = {...newCredentials, region};
}

View File

@@ -27,6 +27,7 @@ class TaskTranscribe extends SttTask {
super(logger, opts, parentTask);
this.transcriptionHook = this.data.transcriptionHook;
this.translationHook = this.data.translationHook;
this.earlyMedia = this.data.earlyMedia === true || (parentTask && parentTask.earlyMedia);
if (this.data.recognizer) {
@@ -302,7 +303,9 @@ class TaskTranscribe extends SttTask {
case 'speechmatics':
this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
this.addCustomEventListener(
ep, SpeechmaticsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
ep, SpeechmaticsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep, channel));
this.addCustomEventListener(
ep, SpeechmaticsTranscriptionEvents.Translation, this._onTranslation.bind(this, cs, ep, channel));
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Info,
this._onSpeechmaticsInfo.bind(this, cs, ep));
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.RecognitionStarted,
@@ -441,7 +444,7 @@ class TaskTranscribe extends SttTask {
this._startAsrTimer(channel);
/* some STT engines will keep listening after a final response, so no need to restart */
if (!['soniox', 'aws', 'microsoft', 'deepgram', 'google']
if (!['soniox', 'aws', 'microsoft', 'deepgram', 'google', 'speechmatics']
.includes(this.vendor)) this._startTranscribing(cs, ep, channel);
}
else {
@@ -466,7 +469,7 @@ class TaskTranscribe extends SttTask {
this._resolve(channel, evt);
/* some STT engines will keep listening after a final response, so no need to restart */
if (!['soniox', 'aws', 'microsoft', 'deepgram', 'google'].includes(this.vendor) &&
if (!['soniox', 'aws', 'microsoft', 'deepgram', 'google', 'speechmatics'].includes(this.vendor) &&
!this.vendor.startsWith('custom:')) {
this.logger.debug('TaskTranscribe:_onTranscription - restarting transcribe');
this._startTranscribing(cs, ep, channel);
@@ -492,6 +495,47 @@ class TaskTranscribe extends SttTask {
}
}
async _onTranslation(_cs, _ep, channel, evt, _fsEvent) {
this.logger.debug({evt}, 'TaskTranscribe:_onTranslation');
if (this.translationHook && evt.results?.length > 0) {
try {
const b3 = this.getTracingPropagation();
const httpHeaders = b3 && {b3};
const payload = {
...this.cs.callInfo,
...httpHeaders,
translation: {
channel,
language: evt.language,
translation: evt.results[0].content
}
};
this.logger.debug({payload}, 'sending translationHook');
const json = await this.cs.requestor.request('verb:hook', this.translationHook, payload);
this.logger.info({json}, 'completed translationHook');
if (json && Array.isArray(json) && !this.parentTask) {
const makeTask = require('./make_task');
const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
if (tasks && tasks.length > 0) {
this.logger.info({tasks: tasks}, `${this.name} replacing application with ${tasks.length} tasks`);
this.cs.replaceApplication(tasks);
}
}
} catch (err) {
this.logger.info(err, 'TranscribeTask:_onTranslation error');
}
if (this.parentTask) {
this.parentTask.emit('translation', evt);
}
}
if (this.killed) {
this.logger.debug('TaskTranscribe:_onTranslation exiting after receiving final transcription');
this._clearTimer();
this.notifyTaskDone();
}
}
async _resolve(channel, evt) {
if (evt.is_final) {
/* we've got a final transcript, so end the otel child span for this channel */
@@ -671,7 +715,7 @@ class TaskTranscribe extends SttTask {
this.logger.debug({evt}, 'TaskGather:_onSpeechmaticsInfo');
}
async _onSpeechmaticsErrror(cs, _ep, evt) {
async _onSpeechmaticsError(cs, _ep, evt) {
// eslint-disable-next-line no-unused-vars
const {message, ...e} = evt;
this._onVendorError(cs, _ep, {error: JSON.stringify(e)});

View File

@@ -13,11 +13,11 @@ class TtsTask extends Task {
this.earlyMedia = this.data.earlyMedia === true || (parentTask && parentTask.earlyMedia);
/**
* Task use taskInlcudeSynthesizer to identify
* if taskInlcudeSynthesizer === true, use label from verb.synthesizer, even it's empty
* if taskInlcudeSynthesizer === false, use label from application.synthesizer
* Task use taskIncludeSynthesizer to identify
* if taskIncludeSynthesizer === true, use label from verb.synthesizer, even it's empty
* if taskIncludeSynthesizer === false, use label from application.synthesizer
*/
this.taskInlcudeSynthesizer = !!this.data.synthesizer;
this.taskIncludeSynthesizer = !!this.data.synthesizer;
this.synthesizer = this.data.synthesizer || {};
this.disableTtsCache = this.data.disableTtsCache;
this.options = this.synthesizer.options || {};
@@ -44,6 +44,64 @@ class TtsTask extends Task {
}
}
getTtsVendorData(cs) {
const vendor = this.synthesizer.vendor && this.synthesizer.vendor !== 'default' ?
this.synthesizer.vendor :
cs.speechSynthesisVendor;
const language = this.synthesizer.language && this.synthesizer.language !== 'default' ?
this.synthesizer.language :
cs.speechSynthesisLanguage ;
const voice = this.synthesizer.voice && this.synthesizer.voice !== 'default' ?
this.synthesizer.voice :
cs.speechSynthesisVoice;
const label = this.taskIncludeSynthesizer ? this.synthesizer.label : cs.speechSynthesisLabel;
return {vendor, language, voice, label};
}
async setTtsStreamingChannelVars(vendor, language, voice, credentials, ep) {
const {api_key, model_id} = credentials;
const {stability, similarity_boost, use_speaker_boost, style} = this.options;
let obj;
this.logger.debug({credentials},
`setTtsStreamingChannelVars: vendor: ${vendor}, language: ${language}, voice: ${voice}`);
switch (vendor) {
case 'deepgram':
obj = {
DEEPGRAM_API_KEY: api_key,
DEEPGRAM_TTS_STREAMING_MODEL: voice
};
break;
case 'cartesia':
obj = {
CARTESIA_API_KEY: api_key,
CARTESIA_TTS_STREAMING_MODEL_ID: model_id,
CARTESIA_TTS_STREAMING_VOICE_ID: voice,
CARTESIA_TTS_STREAMING_LANGUAGE: language || 'en',
};
break;
case 'elevenlabs':
obj = {
ELEVENLABS_API_KEY: api_key,
ELEVENLABS_TTS_STREAMING_MODEL_ID: model_id,
ELEVENLABS_TTS_STREAMING_VOICE_ID: voice,
// 20/12/2024 - only eleven_turbo_v2_5 support multiple language
...(['eleven_turbo_v2_5'].includes(model_id) && {ELEVENLABS_TTS_STREAMING_LANGUAGE: language}),
...(stability && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_STABILITY: stability}),
...(similarity_boost && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_SIMILARITY_BOOST: similarity_boost}),
...(use_speaker_boost && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_USE_SPEAKER_BOOST: use_speaker_boost}),
...(style && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_STYLE: style})
};
break;
default:
throw new Error(`vendor ${vendor} is not supported for tts streaming yet`);
}
this.logger.info({vendor, credentials, obj}, 'setTtsStreamingChannelVars');
await ep.set(obj);
}
async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label, preCache = false}) {
const {srf, accountSid:account_sid} = cs;
const {writeAlerts, AlertType, stats} = srf.locals;

View File

@@ -126,7 +126,12 @@ class ActionHookDelayProcessor extends Emitter {
try {
this._taskInProgress = makeTask(this.logger, t[0]);
this._taskInProgress.disableTracing = true;
this._taskInProgress.exec(this.cs, {ep: this.ep});
this._taskInProgress.exec(this.cs, {ep: this.ep}).catch((err) => {
this.logger.info(`ActionHookDelayProcessor#_onNoResponseTimer: error playing file: ${err.message}`);
this._taskInProgress = null;
this.ep.removeAllListeners('playback-start');
this.ep.removeAllListeners('playback-stop');
});
} catch (err) {
this.logger.info(err, 'ActionHookDelayProcessor#_onNoResponseTimer: error starting action');
this._taskInProgress = null;

View File

@@ -153,7 +153,7 @@ class Amd extends Emitter {
const wordCount = t.alternatives[0].transcript.split(' ').length;
const final = t.is_final;
const foundHint = hints.find((h) => t.alternatives[0].transcript.includes(h));
const foundHint = hints.find((h) => t.alternatives[0].transcript.toLowerCase().includes(h.toLowerCase()));
if (foundHint) {
/* we detected a common voice mail greeting */
this.logger.debug(`Amd:evaluateTranscription: found hint ${foundHint}`);

View File

@@ -46,6 +46,9 @@ class BackgroundTaskManager extends Emitter {
case 'transcribe':
task = await this._initTranscribe(opts);
break;
case 'ttsStream':
task = await this._initTtsStream(opts);
break;
default:
break;
}
@@ -100,6 +103,7 @@ class BackgroundTaskManager extends Emitter {
async _initBargeIn(opts) {
let task;
try {
const copy = JSON.parse(JSON.stringify(opts));
const t = normalizeJambones(this.logger, [opts]);
task = makeTask(this.logger, t[0]);
task
@@ -118,7 +122,7 @@ class BackgroundTaskManager extends Emitter {
if (task.sticky && !this.cs.callGone && !this.cs._stopping) {
this.logger.info('BackgroundTaskManager:_initBargeIn: restarting background bargeIn');
this._bargeInHandled = false;
this.newTask('bargeIn', opts, true);
this.newTask('bargeIn', copy, true);
}
return;
})
@@ -173,6 +177,25 @@ class BackgroundTaskManager extends Emitter {
return task;
}
// Initiate Tts Stream
async _initTtsStream(opts) {
let task;
try {
const t = normalizeJambones(this.logger, [opts]);
task = makeTask(this.logger, t[0]);
const resources = await this.cs._evaluatePreconditions(task);
const {span, ctx} = this.rootSpan.startChildSpan(`background-ttsStream:${task.summary}`);
task.span = span;
task.ctx = ctx;
task.exec(this.cs, resources)
.then(this._taskCompleted.bind(this, 'ttsStream', task))
.catch(this._taskError.bind(this, 'ttsStream', task));
} catch (err) {
this.logger.info(err, 'BackgroundTaskManager:_initTtsStream - Error creating ttsStream task');
}
return task;
}
_taskCompleted(type, task) {
this.logger.debug({type, task}, `BackgroundTaskManager:_taskCompleted: task completed, sticky: ${task.sticky}`);
task.removeAllListeners();

View File

@@ -129,6 +129,7 @@
},
"SpeechmaticsTranscriptionEvents": {
"Transcription": "speechmatics_transcribe::transcription",
"Translation": "speechmatics_transcribe::translation",
"Info": "speechmatics_transcribe::info",
"RecognitionStarted": "speechmatics_transcribe::recognition_started",
"ConnectFailure": "speechmatics_transcribe::connect_failed",
@@ -174,6 +175,13 @@
"Disconnect": "openai_s2s::disconnect",
"ServerEvent": "openai_s2s::server_event"
},
"LlmEvents_VoiceAgent": {
"Error": "error",
"Connect": "voice_agent_s2s::connect",
"ConnectFailure": "voice_agent_s2s::connect_failed",
"Disconnect": "voice_agent_s2s::disconnect",
"ServerEvent": "voice_agent_s2s::server_event"
},
"QueueResults": {
"Bridged": "bridged",
"Error": "error",
@@ -188,7 +196,8 @@
},
"KillReason": {
"Hangup": "hangup",
"Replaced": "replaced"
"Replaced": "replaced",
"MediaTimeout": "media_timeout"
},
"HookMsgTypes": [
"session:new",
@@ -202,6 +211,8 @@
"verb:status",
"llm:event",
"llm:tool-call",
"tts:tokens-result",
"tts:streaming-event",
"jambonz:error"
],
"RecordState": {
@@ -220,7 +231,45 @@
"ToneTimeout": "amd_tone_timeout",
"Stopped": "amd_stopped"
},
"MediaPath": {
"NoMedia": "no-media",
"PartialMedia": "partial-media",
"FullMedia": "full-media"
},
"DeepgramTtsStreamingEvents": {
"Empty": "deepgram_tts_streaming::empty",
"ConnectFailure": "deepgram_tts_streaming::connect_failed",
"Connect": "deepgram_tts_streaming::connect"
},
"CartesiaTtsStreamingEvents": {
"Empty": "cartesia_tts_streaming::empty",
"ConnectFailure": "cartesia_tts_streaming::connect_failed",
"Connect": "cartesia_tts_streaming::connect"
},
"ElevenlabsTtsStreamingEvents": {
"Empty": "elevenlabs_tts_streaming::empty",
"ConnectFailure": "elevenlabs_tts_streaming::connect_failed",
"Connect": "elevenlabs_tts_streaming::connect"
},
"TtsStreamingEvents": {
"Empty": "tts_streaming::empty",
"Pause": "tts_streaming::pause",
"Resume": "tts_streaming::resume",
"ConnectFailure": "tts_streaming::connect_failed"
},
"TtsStreamingConnectionStatus": {
"NotConnected": "not_connected",
"Connected": "connected",
"Connecting": "connecting",
"Failed": "failed"
},
"MAX_SIMRINGS": 10,
"BONG_TONE": "tone_stream://v=-7;%(100,0,941.0,1477.0);v=-7;>=2;+=.1;%(1400,0,350,440)",
"FS_UUID_SET_NAME": "fsUUIDs"
"FS_UUID_SET_NAME": "fsUUIDs",
"SystemState" : {
"Online": "ONLINE",
"Offline": "OFFLINE",
"GracefulShutdownInProgress":"SHUTDOWN_IN_PROGRESS"
},
"FEATURE_SERVER" : "feature-server"
}

View File

@@ -105,6 +105,13 @@ const speechMapper = (cred) => {
obj.voice_engine = o.voice_engine;
obj.options = o.options;
}
else if ('cartesia' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
obj.model_id = o.model_id;
obj.embedding = o.embedding;
obj.options = o.options;
}
else if ('rimelabs' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
@@ -215,11 +222,23 @@ module.exports = (logger, srf) => {
}
};
const lookupVoipCarrierBySid = async(sid) => {
const pp = pool.promise();
try {
const [r] = await pp.query('SELECT * FROM voip_carriers WHERE voip_carrier_sid = ?', [sid]);
return r;
} catch (err) {
logger.error({err}, `lookupVoipCarrierBySid: Error ${sid}`);
}
};
return {
lookupAccountDetails,
updateSpeechCredentialLastUsed,
lookupCarrier,
lookupCarrierByPhoneNumber,
lookupGoogleCustomVoice
lookupGoogleCustomVoice,
lookupVoipCarrierBySid
};
};

View File

@@ -1,9 +1,24 @@
class SpeechCredentialError extends Error {
class NonFatalTaskError extends Error {
constructor(msg) {
super(msg);
}
}
class SpeechCredentialError extends NonFatalTaskError {
constructor(msg) {
super(msg);
}
}
class PlayFileNotFoundError extends NonFatalTaskError {
constructor(url) {
super('File not found');
this.url = url;
}
}
module.exports = {
SpeechCredentialError
SpeechCredentialError,
NonFatalTaskError,
PlayFileNotFoundError
};

View File

@@ -199,7 +199,8 @@ function installSrfLocals(srf, logger) {
} = require('@jambonz/speech-utils')({}, logger);
const {
writeAlerts,
AlertType
AlertType,
writeSystemAlerts
} = require('@jambonz/time-series')(logger, {
host: JAMBONES_TIME_SERIES_HOST,
commitSize: 50,
@@ -269,7 +270,8 @@ function installSrfLocals(srf, logger) {
getFreeswitch,
stats: stats,
writeAlerts,
AlertType
AlertType,
writeSystemAlerts
};
if (localIp) {

View File

@@ -1,5 +1,5 @@
const Emitter = require('events');
const {CallStatus} = require('./constants');
const {CallStatus, MediaPath} = require('./constants');
const SipError = require('drachtio-srf').SipError;
const {TaskPreconditions, CallDirection} = require('../utils/constants');
const CallInfo = require('../session/call-info');
@@ -17,7 +17,9 @@ const HttpRequestor = require('./http-requestor');
const WsRequestor = require('./ws-requestor');
const {makeOpusFirst} = require('./sdp-utils');
const {
JAMBONES_USE_FREESWITCH_TIMER_FD
JAMBONES_USE_FREESWITCH_TIMER_FD,
JAMBONES_MEDIA_TIMEOUT_MS,
JAMBONES_MEDIA_HOLD_TIMEOUT_MS
} = require('../config');
class SingleDialer extends Emitter {
@@ -317,14 +319,19 @@ class SingleDialer extends Emitter {
/**
* kill the call in progress or the stable dialog, whichever we have
*/
async kill() {
async kill(Reason) {
this.killed = true;
if (this.inviteInProgress) await this.inviteInProgress.cancel();
else if (this.dlg && this.dlg.connected) {
const duration = moment().diff(this.dlg.connectTime, 'seconds');
this.logger.debug('SingleDialer:kill hanging up called party');
this.emit('callStatusChange', {callStatus: CallStatus.Completed, duration});
this.dlg.destroy();
const headers = {
...(Reason && {'X-Reason': Reason})
};
this.dlg.destroy({
headers
});
}
if (this.ep) {
this.logger.debug(`SingleDialer:kill - deleting endpoint ${this.ep.uuid}`);
@@ -335,11 +342,22 @@ class SingleDialer extends Emitter {
_configMsEndpoint() {
const opts = {
...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'})
...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'}),
...(JAMBONES_MEDIA_TIMEOUT_MS && {media_timeout: JAMBONES_MEDIA_TIMEOUT_MS}),
...(JAMBONES_MEDIA_HOLD_TIMEOUT_MS && {media_hold_timeout: JAMBONES_MEDIA_HOLD_TIMEOUT_MS})
};
if (Object.keys(opts).length > 0) {
this.ep.set(opts);
}
if (this.dialTask?.inbandDtmfEnabled && !this.ep.inbandDtmfEnabled) {
// https://developer.signalwire.com/freeswitch/FreeSWITCH-Explained/Modules/mod-dptools/6587132/#0-about
try {
this.ep.execute('start_dtmf');
this.ep.inbandDtmfEnabled = true;
} catch (err) {
this.logger.info(err, 'place-outdial:_configMsEndpoint - error enable inband DTMF');
}
}
}
/**
@@ -381,7 +399,8 @@ class SingleDialer extends Emitter {
callInfo: this.callInfo,
accountInfo: this.accountInfo,
tasks,
rootSpan: this.rootSpan
rootSpan: this.rootSpan,
req: this.req
});
await cs.exec();
@@ -455,21 +474,26 @@ class SingleDialer extends Emitter {
return cs;
}
async releaseMediaToSBC(remoteSdp, localSdp) {
async releaseMediaToSBC(remoteSdp, localSdp, releaseMediaEntirely) {
assert(this.dlg && this.dlg.connected && this.ep && typeof remoteSdp === 'string');
const sdp = stripCodecs(this.logger, remoteSdp, localSdp) || remoteSdp;
await this.dlg.modify(sdp, {
headers: {
'X-Reason': 'release-media'
'X-Reason': releaseMediaEntirely ? 'release-media-entirely' : 'release-media'
}
});
this.ep.destroy()
.then(() => this.ep = null)
.catch((err) => this.logger.error({err}, 'SingleDialer:releaseMediaToSBC: Error destroying endpoint'));
try {
await this.ep.destroy();
} catch (err) {
this.logger.error({err}, 'SingleDialer:releaseMediaToSBC: Error destroying endpoint');
}
this.ep = null;
}
async reAnchorMedia() {
async reAnchorMedia(currentMediaRoute = MediaPath.PartialMedia) {
assert(this.dlg && this.dlg.connected && !this.ep);
this.logger.debug('SingleDialer:reAnchorMedia: re-anchoring media after partial media');
this.ep = await this.ms.createEndpoint({remoteSdp: this.dlg.remote.sdp});
this._configMsEndpoint();
await this.dlg.modify(this.ep.local.sdp, {
@@ -477,6 +501,11 @@ class SingleDialer extends Emitter {
'X-Reason': 'anchor-media'
}
});
if (currentMediaRoute === MediaPath.NoMedia) {
this.logger.debug('SingleDialer:reAnchorMedia: repoint endpoint after no media');
await this.ep.modify(this.dlg.remote.sdp);
}
}
_notifyCallStatusChange({callStatus, sipStatus, sipReason, duration}) {

View File

@@ -46,12 +46,24 @@ module.exports = (logger) => {
const {srf} = require('../..');
srf.locals.publicIp = publicIp;
})
.on(LifeCycleEvents.ScaleIn, () => {
.on(LifeCycleEvents.ScaleIn, async() => {
logger.info('AWS scale-in notification: begin drying up calls');
dryUpCalls = true;
lifecycleEmitter.operationalState = LifeCycleEvents.ScaleIn;
const {srf} = require('../..');
const {writeSystemAlerts} = srf.locals;
if (writeSystemAlerts) {
const {SystemState, FEATURE_SERVER} = require('./constants');
await writeSystemAlerts({
system_component: FEATURE_SERVER,
state : SystemState.GracefulShutdownInProgress,
fields : {
detail: `feature-server with process_id ${process.pid} shutdown in progress`,
host: srf.locals?.ipv4
}
});
}
pingProxies(srf);
// if we have zero calls, we can complete the scale-in right

View File

@@ -109,6 +109,8 @@ const stickyVars = {
'SPEECHMATICS_HOST',
'SPEECHMATICS_PATH',
'SPEECHMATICS_SPEECH_HINTS',
'SPEECHMATICS_TRANSLATION_LANGUAGES',
'SPEECHMATICS_TRANSLATION_PARTIALS'
]
};
@@ -183,7 +185,10 @@ const selectDefaultGoogleModel = (task, language, version) => {
(useV2 ? 'long' : 'latest_long');
};
const consolidateTranscripts = (bufferedTranscripts, channel, language, vendor) => {
if (bufferedTranscripts.length === 1) return bufferedTranscripts[0];
if (bufferedTranscripts.length === 1) {
bufferedTranscripts[0].is_final = true;
return bufferedTranscripts[0];
}
let totalConfidence = 0;
const finalTranscript = bufferedTranscripts.reduce((acc, evt) => {
totalConfidence += evt.alternatives[0].confidence;
@@ -937,11 +942,18 @@ module.exports = (logger) => {
};
}
else if ('speechmatics' === vendor) {
const {speechmaticsOptions = {}} = rOpts;
opts = {
...opts,
...(sttCredentials.api_key) && {SPEECHMATICS_API_KEY: sttCredentials.api_key},
...(sttCredentials.speechmatics_stt_uri) && {SPEECHMATICS_HOST: sttCredentials.speechmatics_stt_uri},
...(rOpts.hints?.length > 0 && {SPEECHMATICS_SPEECH_HINTS: rOpts.hints.join(',')}),
...(speechmaticsOptions.translation_config &&
{
SPEECHMATICS_TRANSLATION_LANGUAGES: speechmaticsOptions.translation_config.target_languages.join(','),
SPEECHMATICS_TRANSLATION_PARTIALS: speechmaticsOptions.translation_config.enable_partials ? 1 : 0
}
),
};
}
else if (vendor.startsWith('custom:')) {

View File

@@ -0,0 +1,325 @@
const Emitter = require('events');
const assert = require('assert');
const {
TtsStreamingEvents,
TtsStreamingConnectionStatus
} = require('../utils/constants');
const MAX_CHUNK_SIZE = 1800;
const HIGH_WATER_BUFFER_SIZE = 1000;
const LOW_WATER_BUFFER_SIZE = 200;
const TIMEOUT_RETRY_MSECS = 3000;
class TtsStreamingBuffer extends Emitter {
constructor(cs) {
super();
this.cs = cs;
this.logger = cs.logger;
this.tokens = '';
this.eventHandlers = [];
this._isFull = false;
this._connectionStatus = TtsStreamingConnectionStatus.NotConnected;
this._flushPending = false;
this.timer = null;
}
get isEmpty() {
return this.tokens.length === 0;
}
get isFull() {
return this._isFull;
}
get size() {
return this.tokens.length;
}
get ep() {
return this.cs?.ep;
}
async start() {
assert.ok(
this._connectionStatus === TtsStreamingConnectionStatus.NotConnected,
'TtsStreamingBuffer:start already started, or has failed');
this.vendor = this.cs.getTsStreamingVendor();
if (!this.vendor) {
this.logger.info('TtsStreamingBuffer:start No TTS streaming vendor configured');
throw new Error('No TTS streaming vendor configured');
}
this.logger.info(`TtsStreamingBuffer:start Connecting to TTS streaming with vendor ${this.vendor}`);
this._connectionStatus = TtsStreamingConnectionStatus.Connecting;
try {
if (this.eventHandlers.length === 0) this._initHandlers(this.ep);
await this._api(this.ep, [this.ep.uuid, 'connect']);
} catch (err) {
this.logger.info({err}, 'TtsStreamingBuffer:start Error connecting to TTS streaming');
this._connectionStatus = TtsStreamingConnectionStatus.Failed;
}
}
stop() {
clearTimeout(this.timer);
this.removeCustomEventListeners();
if (this.ep) {
this._api(this.ep, [this.ep.uuid, 'close'])
.catch((err) => this.logger.info({err}, 'TtsStreamingBuffer:kill Error closing TTS streaming'));
}
this.timer = null;
this.tokens = '';
this._connectionStatus = TtsStreamingConnectionStatus.NotConnected;
}
/**
* Add tokens to the buffer and start feeding them to the endpoint if necessary.
*/
async bufferTokens(tokens) {
if (this._connectionStatus === TtsStreamingConnectionStatus.Failed) {
this.logger.info('TtsStreamingBuffer:bufferTokens TTS streaming connection failed, rejecting request');
return {status: 'failed', reason: `connection to ${this.vendor} failed`};
}
const displayedTokens = tokens.length <= 40 ? tokens : tokens.substring(0, 40);
const totalLength = tokens.length;
/* if we crossed the high water mark, reject the request */
if (this.tokens.length + totalLength > HIGH_WATER_BUFFER_SIZE) {
this.logger.info(
`TtsStreamingBuffer throttling: buffer is full, rejecting request to buffer ${totalLength} tokens`);
if (!this._isFull) {
this._isFull = true;
this.emit(TtsStreamingEvents.Pause);
}
return {status: 'failed', reason: 'full'};
}
this.logger.debug(
`TtsStreamingBuffer:bufferTokens "${displayedTokens}" (length: ${totalLength}), starting? ${this.isEmpty}`
);
this.tokens += (tokens || '');
await this._feedTokens();
return {status: 'ok'};
}
flush() {
this.logger.debug('TtsStreamingBuffer:flush');
if (this._connectionStatus === TtsStreamingConnectionStatus.Connecting) {
this.logger.debug('TtsStreamingBuffer:flush TTS stream is not quite ready - wait for connect');
this._flushPending = true;
return;
}
else if (this._connectionStatus === TtsStreamingConnectionStatus.Connected) {
if (this.size === 0) {
this._doFlush();
}
else {
/* we have tokens queued, so flush after they have been sent */
this._pendingFlush = true;
}
}
}
clear() {
this.logger.debug('TtsStreamingBuffer:clear');
if (this._connectionStatus !== TtsStreamingConnectionStatus.Connected) return;
clearTimeout(this.timer);
this._api(this.ep, [this.ep.uuid, 'clear'])
.catch((err) => this.logger.info({err}, 'TtsStreamingBuffer:clear Error clearing TTS streaming'));
this.tokens = '';
this.timer = null;
this._isFull = false;
}
/**
* Send tokens to the TTS engine in sentence chunks for best playout
*/
async _feedTokens(handlingTimeout = false) {
this.logger.debug({tokens: this.tokens}, '_feedTokens');
try {
/* are we in a state where we can feed tokens to the TTS? */
if (!this.cs.isTtsStreamOpen || !this.ep || !this.tokens) {
this.logger.debug('TTS stream is not open or no tokens to send');
return this.tokens?.length || 0;
}
if (this._connectionStatus === TtsStreamingConnectionStatus.NotConnected ||
this._connectionStatus === TtsStreamingConnectionStatus.Failed) {
this.logger.debug('TtsStreamingBuffer:_feedTokens TTS stream is not connected');
return;
}
if (this._connectionStatus === TtsStreamingConnectionStatus.Connecting) {
this.logger.debug('TtsStreamingBuffer:_feedTokens TTS stream is not ready, waiting for connect');
return;
}
/* must send at least one sentence */
const limit = Math.min(MAX_CHUNK_SIZE, this.tokens.length);
let chunkEnd = findSentenceBoundary(this.tokens, limit);
if (chunkEnd <= 0) {
if (handlingTimeout) {
/* on a timeout we've left some tokens sitting around, so be more aggressive now in sending them */
chunkEnd = findWordBoundary(this.tokens, limit);
if (chunkEnd <= 0) {
this.logger.debug('TtsStreamingBuffer:_feedTokens: no word boundary found');
this._setTimerIfNeeded();
return;
}
}
else {
/* if we just received tokens, we wont send unless we have at least a full sentence */
this.logger.debug('TtsStreamingBuffer:_feedTokens: no sentence boundary found');
this._setTimerIfNeeded();
return;
}
}
const chunk = this.tokens.slice(0, chunkEnd);
this.tokens = this.tokens.slice(chunkEnd);
/* freeswitch looks for sequence of 2 newlines to determine end of message, so insert a space */
const modifiedChunk = chunk.replace(/\n\n/g, '\n \n');
await this._api(this.ep, [this.ep.uuid, 'send', modifiedChunk]);
this.logger.debug(`TtsStreamingBuffer:_feedTokens: sent ${chunk.length}, remaining: ${this.tokens.length}`);
if (this._pendingFlush) {
this._doFlush();
this._pendingFlush = false;
}
if (this.isFull && this.tokens.length <= LOW_WATER_BUFFER_SIZE) {
this.logger.info('TtsStreamingBuffer throttling: TTS streaming buffer is no longer full - resuming');
this._isFull = false;
this.emit(TtsStreamingEvents.Resume);
}
} catch (err) {
this.logger.info({err}, 'TtsStreamingBuffer:_feedTokens Error sending TTS chunk');
this.tokens = '';
}
return;
}
async _api(ep, args) {
const apiCmd = `uuid_${this.vendor}_tts_streaming`;
const res = await ep.api(apiCmd, `^^|${args.join('|')}`);
if (!res.body?.startsWith('+OK')) {
this.logger.info({args}, `Error calling ${apiCmd}: ${res.body}`);
throw new Error(`Error calling ${apiCmd}: ${res.body}`);
}
}
_onConnectFailure(vendor) {
this.logger.info(`streaming tts connection failed to ${vendor}`);
this._connectionStatus = TtsStreamingConnectionStatus.Failed;
this.tokens = '';
this.emit(TtsStreamingEvents.ConnectFailure, {vendor});
}
_doFlush() {
this._api(this.ep, [this.ep.uuid, 'flush'])
.catch((err) => this.logger.info({err},
`TtsStreamingBuffer:_doFlush Error flushing TTS streaming: ${JSON.stringify(err)}`));
}
async _onConnect(vendor) {
this.logger.info(`streaming tts connection made to ${vendor}`);
this._connectionStatus = TtsStreamingConnectionStatus.Connected;
if (this.tokens.length > 0) {
await this._feedTokens();
}
if (this._flushPending) {
this.flush();
this._flushPending = false;
}
}
_setTimerIfNeeded() {
if (this.tokens.length > 0 && !this.timer) {
this.timer = setTimeout(this._onTimeout.bind(this), TIMEOUT_RETRY_MSECS);
}
}
_onTimeout() {
this.logger.info('TtsStreamingBuffer:_onTimeout');
this.timer = null;
this._feedTokens(true);
}
_onTtsEmpty(vendor) {
this.emit(TtsStreamingEvents.Empty, {vendor});
}
addCustomEventListener(ep, event, handler) {
this.eventHandlers.push({ep, event, handler});
ep.addCustomEventListener(event, handler);
}
removeCustomEventListeners() {
this.eventHandlers.forEach((h) => h.ep.removeCustomEventListener(h.event, h.handler));
}
_initHandlers(ep) {
[
// DH: add other vendors here as modules are added
'deepgram',
'cartesia',
'elevenlabs'
].forEach((vendor) => {
const eventClassName = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}TtsStreamingEvents`;
const eventClass = require('../utils/constants')[eventClassName];
if (!eventClass) throw new Error(`Event class for vendor ${vendor} not found`);
this.addCustomEventListener(ep, eventClass.Connect, this._onConnect.bind(this, vendor));
this.addCustomEventListener(ep, eventClass.ConnectFailure, this._onConnectFailure.bind(this, vendor));
this.addCustomEventListener(ep, eventClass.Empty, this._onTtsEmpty.bind(this, vendor));
});
}
}
const findSentenceBoundary = (text, limit) => {
// Match traditional sentence boundaries or double newlines
const sentenceEndRegex = /[.!?](?=\s|$)|\n\n/g;
let lastSentenceBoundary = -1;
let match;
while ((match = sentenceEndRegex.exec(text)) && match.index < limit) {
const precedingText = text.slice(0, match.index).trim(); // Extract text before the match and trim whitespace
if (precedingText.length > 0) { // Check if there's actual content
if (
match[0] === '\n\n' || // It's a double newline
(match.index === 0 || !/\d$/.test(text[match.index - 1])) // Standard punctuation rules
) {
lastSentenceBoundary = match.index + (match[0] === '\n\n' ? 2 : 1); // Include the boundary
}
}
}
return lastSentenceBoundary;
};
const findWordBoundary = (text, limit) => {
const wordBoundaryRegex = /\s+/g;
let lastWordBoundary = -1;
let match;
while ((match = wordBoundaryRegex.exec(text)) && match.index < limit) {
lastWordBoundary = match.index;
}
return lastWordBoundary;
};
module.exports = TtsStreamingBuffer;

View File

@@ -12,6 +12,20 @@ const {
JAMBONES_WS_MAX_PAYLOAD,
HTTP_USER_AGENT_HEADER
} = require('../config');
const MTYPE_WANTS_ACK = [
'call:status',
'verb:status',
'jambonz:error',
'llm:event',
'llm:tool-call',
'tts:streaming-event',
'tts:tokens-result',
];
const MTYPE_NO_DATA = [
'llm:tool-output',
'tts:flush',
'tts:clear'
];
class WsRequestor extends BaseRequestor {
constructor(logger, account_sid, hook, secret) {
@@ -44,7 +58,7 @@ class WsRequestor extends BaseRequestor {
async request(type, hook, params, httpHeaders = {}) {
assert(HookMsgTypes.includes(type));
const url = hook.url || hook;
const wantsAck = !['call:status', 'verb:status', 'jambonz:error', 'llm:event', 'llm:tool-call'].includes(type);
const wantsAck = !MTYPE_WANTS_ACK.includes(type);
if (this.maliciousClient) {
this.logger.info({url: this.url}, 'WsRequestor:request - discarding msg to malicious client');
@@ -408,7 +422,7 @@ class WsRequestor extends BaseRequestor {
case 'command':
assert.ok(command, 'command property not supplied');
assert.ok(data || command === 'llm:tool-output', 'data property not supplied');
assert.ok(data || MTYPE_NO_DATA.includes(command), 'data property not supplied');
this._recvCommand(msgid, command, call_sid, queueCommand, tool_call_id, data);
break;

301
package-lock.json generated
View File

@@ -15,10 +15,10 @@
"@jambonz/http-health-check": "^0.0.1",
"@jambonz/mw-registrar": "^0.2.7",
"@jambonz/realtimedb-helpers": "^0.8.8",
"@jambonz/speech-utils": "^0.1.22",
"@jambonz/speech-utils": "^0.2.1",
"@jambonz/stats-collector": "^0.1.10",
"@jambonz/time-series": "^0.2.9",
"@jambonz/verb-specifications": "^0.0.83",
"@jambonz/time-series": "^0.2.13",
"@jambonz/verb-specifications": "^0.0.91",
"@opentelemetry/api": "^1.8.0",
"@opentelemetry/exporter-jaeger": "^1.23.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
@@ -31,7 +31,7 @@
"bent": "^7.3.12",
"debug": "^4.3.4",
"deepcopy": "^2.1.0",
"drachtio-fsmrf": "^3.0.45",
"drachtio-fsmrf": "^3.0.46",
"drachtio-srf": "^4.5.35",
"express": "^4.19.2",
"express-validator": "^7.0.1",
@@ -1379,6 +1379,95 @@
"node": ">=6.9.0"
}
},
"node_modules/@cartesia/cartesia-js": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/@cartesia/cartesia-js/-/cartesia-js-2.1.1.tgz",
"integrity": "sha512-Kwcrxtay6G3hwFGyq7cwB00d+diABhi2/5Mb4vytmN+QqBgTAVYgwFweGnLDL9Ci9zFttgLE1rW8u+EA8U5P9A==",
"dependencies": {
"emittery": "^0.13.1",
"form-data": "^4.0.0",
"form-data-encoder": "^4.0.2",
"formdata-node": "^6.0.3",
"human-id": "^4.1.1",
"node-fetch": "2.7.0",
"qs": "6.11.2",
"readable-stream": "^4.5.2",
"url-join": "4.0.1",
"ws": "^8.15.13"
}
},
"node_modules/@cartesia/cartesia-js/node_modules/form-data": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz",
"integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",
"license": "MIT",
"dependencies": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/@cartesia/cartesia-js/node_modules/form-data-encoder": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-4.0.2.tgz",
"integrity": "sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw==",
"license": "MIT",
"engines": {
"node": ">= 18"
}
},
"node_modules/@cartesia/cartesia-js/node_modules/formdata-node": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-6.0.3.tgz",
"integrity": "sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==",
"license": "MIT",
"engines": {
"node": ">= 18"
}
},
"node_modules/@cartesia/cartesia-js/node_modules/process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
"integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
"license": "MIT",
"engines": {
"node": ">= 0.6.0"
}
},
"node_modules/@cartesia/cartesia-js/node_modules/qs": {
"version": "6.11.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz",
"integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==",
"license": "BSD-3-Clause",
"dependencies": {
"side-channel": "^1.0.4"
},
"engines": {
"node": ">=0.6"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/@cartesia/cartesia-js/node_modules/readable-stream": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz",
"integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==",
"license": "MIT",
"dependencies": {
"abort-controller": "^3.0.0",
"buffer": "^6.0.3",
"events": "^3.3.0",
"process": "^0.11.10",
"string_decoder": "^1.3.0"
},
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
},
"node_modules/@eslint/eslintrc": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz",
@@ -1542,13 +1631,14 @@
}
},
"node_modules/@jambonz/speech-utils": {
"version": "0.1.22",
"resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.1.22.tgz",
"integrity": "sha512-1/ARN6tiY/Roc86e1pw1Te/PDXTWjZb+hgpNdkM3i+GqpmtsuTWnDMxS9sBbsIgYEggJM67fn4Z1j65nERW0ag==",
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.2.1.tgz",
"integrity": "sha512-LIZ3HCW+vra626L18NiudY4v7zXVW4EvqUeeVLFM9S8fi1nGP2lHbDur/DFiN/VsJXYdNIdO4PAsILxeZXoOLg==",
"license": "MIT",
"dependencies": {
"@aws-sdk/client-polly": "^3.496.0",
"@aws-sdk/client-sts": "^3.496.0",
"@cartesia/cartesia-js": "^2.1.0",
"@google-cloud/text-to-speech": "^5.5.0",
"@grpc/grpc-js": "^1.9.14",
"@jambonz/realtimedb-helpers": "^0.8.7",
@@ -1572,18 +1662,19 @@
}
},
"node_modules/@jambonz/time-series": {
"version": "0.2.9",
"resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.9.tgz",
"integrity": "sha512-+/Oal0mjjV4iQ8q0ymtvVJP+GqgGpYUb2bc/FD/xDxOzKtl340l9yoM3oczREJg5IvEkpExz6NogJzUiCSeVnQ==",
"version": "0.2.13",
"resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.13.tgz",
"integrity": "sha512-Kj+l+YUnI27zZA4qoPRzjN7L82W7GuMXYq9ttDjXQ0ZBIdOLAzJjB6R3jJ3b+mvoNEQ6qG5MUtfoc6CpTFH5lw==",
"dependencies": {
"debug": "^4.3.1",
"influx": "^5.9.3"
}
},
"node_modules/@jambonz/verb-specifications": {
"version": "0.0.83",
"resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.83.tgz",
"integrity": "sha512-3m1o3gnWw1yEwNfkZ6IIyUhdUqsQ3aWBNoWJHswe4udvVbEIxPHi+8jKGTJVF2vxddzwfOWp7RmMCV9RmKWzkw==",
"version": "0.0.91",
"resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.91.tgz",
"integrity": "sha512-C9UIKytogOdUp5sGqfEetl82/8lDQ8zgEU5diQKpU4dZeA3M8qItqjJGSXexNh4PK9ECUYNorsPoRGNxGcr2IA==",
"license": "MIT",
"dependencies": {
"debug": "^4.3.4",
"pino": "^8.8.0"
@@ -3556,10 +3647,11 @@
"integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ=="
},
"node_modules/cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"license": "MIT",
"dependencies": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
@@ -3877,9 +3969,10 @@
}
},
"node_modules/drachtio-fsmrf": {
"version": "3.0.45",
"resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.45.tgz",
"integrity": "sha512-YsrevTwGvg9v0OQXB4gZedlmzegB83fyd3NX3X2x7NXsKa5jO6TZCvZVHtBf/jR/ELE3B0aVLpxHw2YviRMIuQ==",
"version": "3.0.46",
"resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.46.tgz",
"integrity": "sha512-ScgyOnsOL45feuKKquT2Gij/jDRdjVha2TnQ6/Me2/M/C+29c9W7cdYlt3UZB3GyodazBukwIy5RrZf1iuXBGw==",
"license": "MIT",
"dependencies": {
"camel-case": "^4.1.2",
"debug": "^2.6.9",
@@ -4005,6 +4098,18 @@
"integrity": "sha512-dWuNH+XUT9hdFHASfMpcZGW5kUyJvllumJkXaXiswuCkoaFIFI89aykBPuHEi1YUWQGRCqvIO0BUdmeFJ4W4Ww==",
"dev": true
},
"node_modules/emittery": {
"version": "0.13.1",
"resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz",
"integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==",
"license": "MIT",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sindresorhus/emittery?sponsor=1"
}
},
"node_modules/emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
@@ -4542,9 +4647,9 @@
}
},
"node_modules/express": {
"version": "4.21.1",
"resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz",
"integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==",
"version": "4.21.2",
"resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
"integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
@@ -4565,7 +4670,7 @@
"methods": "~1.1.2",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
"path-to-regexp": "0.1.10",
"path-to-regexp": "0.1.12",
"proxy-addr": "~2.0.7",
"qs": "6.13.0",
"range-parser": "~1.2.1",
@@ -4580,6 +4685,10 @@
},
"engines": {
"node": ">= 0.10.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/express-validator": {
@@ -5456,6 +5565,15 @@
"node": ">= 14"
}
},
"node_modules/human-id": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/human-id/-/human-id-4.1.1.tgz",
"integrity": "sha512-3gKm/gCSUipeLsRYZbbdA1BD83lBoWUkZ7G9VFrhWPAU76KwYo5KR8V28bpoPm/ygy0x5/GCbpRQdY7VLYCoIg==",
"license": "MIT",
"bin": {
"human-id": "dist/cli.js"
}
},
"node_modules/humanize-ms": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
@@ -7448,10 +7566,9 @@
"integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="
},
"node_modules/path-to-regexp": {
"version": "0.1.10",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz",
"integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==",
"license": "MIT"
"version": "0.1.12",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz",
"integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ=="
},
"node_modules/peek-readable": {
"version": "4.1.0",
@@ -9120,6 +9237,12 @@
"punycode": "^2.1.0"
}
},
"node_modules/url-join": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz",
"integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==",
"license": "MIT"
},
"node_modules/url-parse": {
"version": "1.5.10",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
@@ -10581,6 +10704,70 @@
"to-fast-properties": "^2.0.0"
}
},
"@cartesia/cartesia-js": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/@cartesia/cartesia-js/-/cartesia-js-2.1.1.tgz",
"integrity": "sha512-Kwcrxtay6G3hwFGyq7cwB00d+diABhi2/5Mb4vytmN+QqBgTAVYgwFweGnLDL9Ci9zFttgLE1rW8u+EA8U5P9A==",
"requires": {
"emittery": "^0.13.1",
"form-data": "^4.0.0",
"form-data-encoder": "^4.0.2",
"formdata-node": "^6.0.3",
"human-id": "^4.1.1",
"node-fetch": "2.7.0",
"qs": "6.11.2",
"readable-stream": "^4.5.2",
"url-join": "4.0.1",
"ws": "^8.15.13"
},
"dependencies": {
"form-data": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz",
"integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",
"requires": {
"asynckit": "^0.4.0",
"combined-stream": "^1.0.8",
"mime-types": "^2.1.12"
}
},
"form-data-encoder": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-4.0.2.tgz",
"integrity": "sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw=="
},
"formdata-node": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-6.0.3.tgz",
"integrity": "sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg=="
},
"process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
"integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A=="
},
"qs": {
"version": "6.11.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz",
"integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==",
"requires": {
"side-channel": "^1.0.4"
}
},
"readable-stream": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz",
"integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==",
"requires": {
"abort-controller": "^3.0.0",
"buffer": "^6.0.3",
"events": "^3.3.0",
"process": "^0.11.10",
"string_decoder": "^1.3.0"
}
}
}
},
"@eslint/eslintrc": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz",
@@ -10713,12 +10900,13 @@
}
},
"@jambonz/speech-utils": {
"version": "0.1.22",
"resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.1.22.tgz",
"integrity": "sha512-1/ARN6tiY/Roc86e1pw1Te/PDXTWjZb+hgpNdkM3i+GqpmtsuTWnDMxS9sBbsIgYEggJM67fn4Z1j65nERW0ag==",
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.2.1.tgz",
"integrity": "sha512-LIZ3HCW+vra626L18NiudY4v7zXVW4EvqUeeVLFM9S8fi1nGP2lHbDur/DFiN/VsJXYdNIdO4PAsILxeZXoOLg==",
"requires": {
"@aws-sdk/client-polly": "^3.496.0",
"@aws-sdk/client-sts": "^3.496.0",
"@cartesia/cartesia-js": "^2.1.0",
"@google-cloud/text-to-speech": "^5.5.0",
"@grpc/grpc-js": "^1.9.14",
"@jambonz/realtimedb-helpers": "^0.8.7",
@@ -10742,18 +10930,18 @@
}
},
"@jambonz/time-series": {
"version": "0.2.9",
"resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.9.tgz",
"integrity": "sha512-+/Oal0mjjV4iQ8q0ymtvVJP+GqgGpYUb2bc/FD/xDxOzKtl340l9yoM3oczREJg5IvEkpExz6NogJzUiCSeVnQ==",
"version": "0.2.13",
"resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.13.tgz",
"integrity": "sha512-Kj+l+YUnI27zZA4qoPRzjN7L82W7GuMXYq9ttDjXQ0ZBIdOLAzJjB6R3jJ3b+mvoNEQ6qG5MUtfoc6CpTFH5lw==",
"requires": {
"debug": "^4.3.1",
"influx": "^5.9.3"
}
},
"@jambonz/verb-specifications": {
"version": "0.0.83",
"resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.83.tgz",
"integrity": "sha512-3m1o3gnWw1yEwNfkZ6IIyUhdUqsQ3aWBNoWJHswe4udvVbEIxPHi+8jKGTJVF2vxddzwfOWp7RmMCV9RmKWzkw==",
"version": "0.0.91",
"resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.91.tgz",
"integrity": "sha512-C9UIKytogOdUp5sGqfEetl82/8lDQ8zgEU5diQKpU4dZeA3M8qItqjJGSXexNh4PK9ECUYNorsPoRGNxGcr2IA==",
"requires": {
"debug": "^4.3.4",
"pino": "^8.8.0"
@@ -12240,9 +12428,9 @@
"integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ=="
},
"cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"version": "7.0.6",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
"integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
@@ -12464,9 +12652,9 @@
}
},
"drachtio-fsmrf": {
"version": "3.0.45",
"resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.45.tgz",
"integrity": "sha512-YsrevTwGvg9v0OQXB4gZedlmzegB83fyd3NX3X2x7NXsKa5jO6TZCvZVHtBf/jR/ELE3B0aVLpxHw2YviRMIuQ==",
"version": "3.0.46",
"resolved": "https://registry.npmjs.org/drachtio-fsmrf/-/drachtio-fsmrf-3.0.46.tgz",
"integrity": "sha512-ScgyOnsOL45feuKKquT2Gij/jDRdjVha2TnQ6/Me2/M/C+29c9W7cdYlt3UZB3GyodazBukwIy5RrZf1iuXBGw==",
"requires": {
"camel-case": "^4.1.2",
"debug": "^2.6.9",
@@ -12581,6 +12769,11 @@
"integrity": "sha512-dWuNH+XUT9hdFHASfMpcZGW5kUyJvllumJkXaXiswuCkoaFIFI89aykBPuHEi1YUWQGRCqvIO0BUdmeFJ4W4Ww==",
"dev": true
},
"emittery": {
"version": "0.13.1",
"resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz",
"integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ=="
},
"emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
@@ -13002,9 +13195,9 @@
}
},
"express": {
"version": "4.21.1",
"resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz",
"integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==",
"version": "4.21.2",
"resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
"integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
"requires": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
@@ -13025,7 +13218,7 @@
"methods": "~1.1.2",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
"path-to-regexp": "0.1.10",
"path-to-regexp": "0.1.12",
"proxy-addr": "~2.0.7",
"qs": "6.13.0",
"range-parser": "~1.2.1",
@@ -13662,6 +13855,11 @@
"debug": "4"
}
},
"human-id": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/human-id/-/human-id-4.1.1.tgz",
"integrity": "sha512-3gKm/gCSUipeLsRYZbbdA1BD83lBoWUkZ7G9VFrhWPAU76KwYo5KR8V28bpoPm/ygy0x5/GCbpRQdY7VLYCoIg=="
},
"humanize-ms": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
@@ -15148,9 +15346,9 @@
"integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="
},
"path-to-regexp": {
"version": "0.1.10",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz",
"integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w=="
"version": "0.1.12",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz",
"integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ=="
},
"peek-readable": {
"version": "4.1.0",
@@ -16370,6 +16568,11 @@
"punycode": "^2.1.0"
}
},
"url-join": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz",
"integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA=="
},
"url-parse": {
"version": "1.5.10",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",

View File

@@ -31,10 +31,10 @@
"@jambonz/http-health-check": "^0.0.1",
"@jambonz/mw-registrar": "^0.2.7",
"@jambonz/realtimedb-helpers": "^0.8.8",
"@jambonz/speech-utils": "^0.1.22",
"@jambonz/speech-utils": "^0.2.1",
"@jambonz/stats-collector": "^0.1.10",
"@jambonz/time-series": "^0.2.9",
"@jambonz/verb-specifications": "^0.0.83",
"@jambonz/verb-specifications": "^0.0.91",
"@jambonz/time-series": "^0.2.13",
"@opentelemetry/api": "^1.8.0",
"@opentelemetry/exporter-jaeger": "^1.23.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
@@ -47,7 +47,7 @@
"bent": "^7.3.12",
"debug": "^4.3.4",
"deepcopy": "^2.1.0",
"drachtio-fsmrf": "^3.0.45",
"drachtio-fsmrf": "^3.0.46",
"drachtio-srf": "^4.5.35",
"express": "^4.19.2",
"express-validator": "^7.0.1",

View File

@@ -222,3 +222,62 @@ test('test create-call app_json', async(t) => {
t.error(err);
}
});
test('test create-call timeLimit', async(t) => {
clearModule.all();
const {srf, disconnect} = require('../app');
try {
await connect(srf);
// GIVEN
let from = 'create-call-app-json';
let account_sid = 'bb845d4b-83a9-4cde-a6e9-50f3743bab3f';
// Give UAS app time to come up
const p = sippUac('uas.xml', '172.38.0.10', from);
await waitFor(1000);
const startTime = Date.now();
const app_json = `[
{
"verb": "pause",
"length": 7
}
]`;
const post = bent('http://127.0.0.1:3000/', 'POST', 'json', 201);
post('v1/createCall', {
'account_sid':account_sid,
"call_hook": {
"url": "http://127.0.0.1:3100/",
"method": "POST",
"username": "username",
"password": "password"
},
app_json,
"from": from,
"to": {
"type": "phone",
"number": "15583084809"
},
"timeLimit": 1,
"speech_recognizer_vendor": "google",
"speech_recognizer_language": "en"
});
//THEN
await p;
const endTime = Date.now();
t.ok(endTime - startTime < 2000, 'create-call: timeLimit is respected');
disconnect();
} catch (err) {
console.log(`error received: ${err}`);
disconnect();
t.error(err);
}
});

View File

@@ -351,6 +351,8 @@ speech_credential_sid CHAR(36) NOT NULL,
model VARCHAR(512) NOT NULL,
reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
name VARCHAR(64) NOT NULL,
voice_cloning_key MEDIUMTEXT,
use_voice_cloning_key BOOLEAN DEFAULT false,
PRIMARY KEY (google_custom_voice_sid)
);