Compare commits

..

3 Commits

Author SHA1 Message Date
Quan HL
75d8c7268f Merge branch 'main' of https://github.com/jambonz/jambonz-feature-server into feat/aws_s2s 2025-06-12 13:45:25 +07:00
Quan HL
7f5bb9c141 wip 2025-06-10 13:23:11 +07:00
Quan HL
f8a69b4b79 Support Aws S2S 2025-06-09 15:01:25 +07:00
55 changed files with 2910 additions and 6428 deletions

3
.gitignore vendored
View File

@@ -2,9 +2,6 @@
logs
*.log
.claude/
CLAUDE.md
# Runtime data
pids
*.pid

6
app.js
View File

@@ -29,12 +29,6 @@ const {LifeCycleEvents, FS_UUID_SET_NAME, SystemState, FEATURE_SERVER} = require
const installSrfLocals = require('./lib/utils/install-srf-locals');
const createHttpListener = require('./lib/utils/http-listener');
const healthCheck = require('@jambonz/http-health-check');
const ProcessMonitor = require('./lib/utils/process-monitor');
const monitor = new ProcessMonitor(logger);
// Log startup
monitor.logStartup();
monitor.setupSignalHandlers();
logger.on('level-change', (lvl, _val, prevLvl, _prevVal, instance) => {
if (logger !== instance) {

View File

@@ -174,61 +174,5 @@
"non è raggiungibile",
"lascia pure un messaggio",
"puoi lasciare un messaggio"
],
"ja-JP": [
"この通話は留守番電話に転送されました",
"発信先は現在電話に出ることができません",
"発信音の後でメッセージを録音してください",
"録音を完了したら電話を切ることができます",
"只今電話に出ることができません",
"ただ今電話に出ることができません",
"ただいま電話に出ることができません",
"ピーという発信音の後にお名前とご用件をお話しください",
"ファックスを送られる方はスタートボタンを押してください",
"FAXを送られる方はスタートボタンを押してください",
"おかけになった電話をお呼びしましたが",
"お出になりません",
"おでになりません",
"お掛けになった電話番号は",
"おかけになった電話番号は",
"お掛けになった電話は",
"おかけになった電話は",
"現在使われておりません",
"番号をお確かめになって",
"お掛け直し下さい",
"おかけ直し下さい",
"おかけ直しください",
"こちらはNTTドコモです",
"こちらはエーユーです",
"こちらはソフトバンクです",
"電波の届かない",
"電源が入っていない",
"掛かりません",
"かかりません",
"お繋ぎすることが出来ません",
"お繋ぎ出来ません",
"お繋ぎすることができません",
"お繋ぎできません",
"おつなぎすることができません",
"おつなぎできません",
"メッセージを録音",
"留守番電話",
"お留守番サービス",
"留守番",
"留守電",
"留守",
"接続します",
"合図の音",
"ピーと",
"発信音",
"ご用件",
"伝言",
"お話しください",
"ファックス",
"FAX",
"終了",
"終了しました",
"終了いたしました",
"営業時間"
]
}

View File

@@ -119,7 +119,7 @@ const ENCRYPTION_SECRET = process.env.ENCRYPTION_SECRET;
const HTTP_POOL = process.env.HTTP_POOL && parseInt(process.env.HTTP_POOL);
const HTTP_POOLSIZE = parseInt(process.env.HTTP_POOLSIZE, 10) || 10;
const HTTP_PIPELINING = parseInt(process.env.HTTP_PIPELINING, 10) || 1;
const HTTP_TIMEOUT = parseInt(process.env.JAMBONES_HTTP_TIMEOUT, 10) || 10000;
const HTTP_TIMEOUT = 10000;
const HTTP_PROXY_IP = process.env.JAMBONES_HTTP_PROXY_IP;
const HTTP_PROXY_PORT = process.env.JAMBONES_HTTP_PROXY_PORT;
const HTTP_PROXY_PROTOCOL = process.env.JAMBONES_HTTP_PROXY_PROTOCOL || 'http';
@@ -130,7 +130,7 @@ const OPTIONS_PING_INTERVAL = parseInt(process.env.OPTIONS_PING_INTERVAL, 10) ||
const JAMBONZ_RECORD_WS_BASE_URL = process.env.JAMBONZ_RECORD_WS_BASE_URL || process.env.JAMBONES_RECORD_WS_BASE_URL;
const JAMBONZ_RECORD_WS_USERNAME = process.env.JAMBONZ_RECORD_WS_USERNAME || process.env.JAMBONES_RECORD_WS_USERNAME;
const JAMBONZ_RECORD_WS_PASSWORD = process.env.JAMBONZ_RECORD_WS_PASSWORD || process.env.JAMBONES_RECORD_WS_PASSWORD;
const JAMBONZ_DIAL_PAI_HEADER = process.env.JAMBONZ_DIAL_PAI_HEADER || false;
const JAMBONZ_DISABLE_DIAL_PAI_HEADER = process.env.JAMBONZ_DISABLE_DIAL_PAI_HEADER || false;
const JAMBONES_DISABLE_DIRECT_P2P_CALL = process.env.JAMBONES_DISABLE_DIRECT_P2P_CALL || false;
const JAMBONES_EAGERLY_PRE_CACHE_AUDIO = parseInt(process.env.JAMBONES_EAGERLY_PRE_CACHE_AUDIO, 10) || 0;
@@ -139,11 +139,6 @@ const JAMBONES_USE_FREESWITCH_TIMER_FD = process.env.JAMBONES_USE_FREESWITCH_TIM
const JAMBONES_DIAL_SBC_FOR_REGISTERED_USER = process.env.JAMBONES_DIAL_SBC_FOR_REGISTERED_USER || false;
const JAMBONES_MEDIA_TIMEOUT_MS = process.env.JAMBONES_MEDIA_TIMEOUT_MS || 0;
const JAMBONES_MEDIA_HOLD_TIMEOUT_MS = process.env.JAMBONES_MEDIA_HOLD_TIMEOUT_MS || 0;
const JAMBONES_WEBHOOK_ERROR_RETURN = parseInt(process.env.JAMBONES_WEBHOOK_ERROR_RETURN, 10) || 480;
/* say / tts */
const JAMBONES_SAY_CHUNK_SIZE = parseInt(process.env.JAMBONES_SAY_CHUNK_SIZE, 10) || 900;
// jambonz
const JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS =
process.env.JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS;
@@ -230,13 +225,11 @@ module.exports = {
JAMBONZ_RECORD_WS_BASE_URL,
JAMBONZ_RECORD_WS_USERNAME,
JAMBONZ_RECORD_WS_PASSWORD,
JAMBONZ_DIAL_PAI_HEADER,
JAMBONZ_DISABLE_DIAL_PAI_HEADER,
JAMBONES_DISABLE_DIRECT_P2P_CALL,
JAMBONES_USE_FREESWITCH_TIMER_FD,
JAMBONES_DIAL_SBC_FOR_REGISTERED_USER,
JAMBONES_MEDIA_TIMEOUT_MS,
JAMBONES_MEDIA_HOLD_TIMEOUT_MS,
JAMBONES_SAY_CHUNK_SIZE,
JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS,
JAMBONES_WEBHOOK_ERROR_RETURN
JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS
};

View File

@@ -13,11 +13,10 @@ const WsRequestor = require('../../utils/ws-requestor');
const RootSpan = require('../../utils/call-tracer');
const dbUtils = require('../../utils/db-utils');
const { decrypt } = require('../../utils/encrypt-decrypt');
const { mergeSdpMedia, extractSdpMedia, removeVideoSdp } = require('../../utils/sdp-utils');
const { mergeSdpMedia, extractSdpMedia } = require('../../utils/sdp-utils');
const { createCallSchema, customSanitizeFunction } = require('../schemas/create-call');
const { selectHostPort } = require('../../utils/network');
const { JAMBONES_DIAL_SBC_FOR_REGISTERED_USER } = require('../../config');
const { createMediaEndpoint } = require('../../utils/media-endpoint');
const removeNullProperties = (obj) => (Object.keys(obj).forEach((key) => obj[key] === null && delete obj[key]), obj);
const removeNulls = (req, res, next) => {
@@ -68,7 +67,7 @@ router.post('/',
const {
lookupAppBySid
} = srf.locals.dbHelpers;
const {getSBC} = srf.locals;
const {getSBC, getFreeswitch} = srf.locals;
let sbcAddress = getSBC();
if (!sbcAddress) throw new Error('no available SBCs for outbound call creation');
const target = restDial.to;
@@ -147,7 +146,7 @@ router.post('/',
// find handling sbc sip for called user
if (JAMBONES_DIAL_SBC_FOR_REGISTERED_USER && target.type === 'user') {
const { registrar} = srf.locals.dbHelpers;
const { registrar } = srf.locals.dbHelpers;
const reg = await registrar.query(target.name);
if (reg) {
sbcAddress = selectHostPort(logger, reg.sbcAddress, 'tcp')[1];
@@ -159,9 +158,7 @@ router.post('/',
* trunk isn't specified,
* check if from-number matches any existing numbers on Jambonz
* */
const { lookupLcrByAccount} = srf.locals.dbHelpers;
const lcrs = await lookupLcrByAccount(req.body.account_sid);
if (target.type === 'phone' && !target.trunk && lcrs.length == 0) {
if (target.type === 'phone' && !target.trunk) {
const str = restDial.from || '';
const callingNumber = str.startsWith('+') ? str.substring(1) : str;
const voip_carrier_sid = await lookupCarrierByPhoneNumber(req.body.account_sid, callingNumber);
@@ -173,7 +170,9 @@ router.post('/',
}
/* create endpoint for outdial */
const ep = await createMediaEndpoint(srf, logger);
const ms = getFreeswitch();
if (!ms) throw new Error('no available Freeswitch for outbound call creation');
const ep = await ms.createEndpoint();
logger.debug(`createCall: successfully allocated endpoint, sending INVITE to ${sbcAddress}`);
/* launch outdial */
@@ -182,14 +181,10 @@ router.post('/',
let localSdp = ep.local.sdp;
if (req.body.dual_streams) {
dualEp = await createMediaEndpoint(srf, logger);
dualEp = await ms.createEndpoint();
localSdp = mergeSdpMedia(localSdp, dualEp.local.sdp);
}
if (process.env.JAMBONES_VIDEO_CALLS_ENABLED_IN_FS) {
logger.debug('createCall: removing video sdp');
localSdp = removeVideoSdp(localSdp);
ep.modify(localSdp);
}
const connectStream = async(remoteSdp) => {
if (remoteSdp !== sdp) {
sdp = remoteSdp;
@@ -240,7 +235,6 @@ router.post('/',
if (app.call_hook.url === app.call_status_hook?.url || !app.call_status_hook?.url) {
logger.debug('reusing websocket for call status hook');
app.notifier = app.requestor;
app.call_status_hook = app.call_hook;
}
}
else {
@@ -291,7 +285,7 @@ router.post('/',
}, {
...(account.enable_debug_log && {level: 'debug'})
});
app.requestor.logger = app.notifier.logger = restDial.logger = sipLogger;
app.requestor.logger = app.notifier.logger = sipLogger;
const callInfo = new CallInfo({
direction: CallDirection.Outbound,
req: inviteReq,

View File

@@ -12,8 +12,7 @@ const RootSpan = require('./utils/call-tracer');
const listTaskNames = require('./utils/summarize-tasks');
const {
JAMBONES_MYSQL_REFRESH_TTL,
JAMBONES_DISABLE_DIRECT_P2P_CALL,
JAMBONES_WEBHOOK_ERROR_RETURN
JAMBONES_DISABLE_DIRECT_P2P_CALL
} = require('./config');
const { createJambonzApp } = require('./dynamic-apps');
const { decrypt } = require('./utils/encrypt-decrypt');
@@ -113,14 +112,6 @@ module.exports = function(srf, logger) {
req.locals.callingNumber = sipURIs[1];
}
}
// Feature server INVITE request pipelines taking time to finish,
// while connecting and fetch application from db and invoking webhook.
// call can be canceled without any handling, so we add a listener here
req.once('cancel', (sipMsg) => {
logger.info(`${callId} got CANCEL request`);
req.locals.canceled = true;
});
next();
}
@@ -371,14 +362,13 @@ module.exports = function(srf, logger) {
});
// if transferred call contains callInfo, let update original data to newly created callInfo in this instance.
if (app.transferredCall && app.callInfo) {
const {direction, callerName, from, to, originatingSipIp, originatingSipTrunkName, customerData} = app.callInfo;
const {direction, callerName, from, to, originatingSipIp, originatingSipTrunkName} = app.callInfo;
req.locals.callInfo.direction = direction;
req.locals.callInfo.callerName = callerName;
req.locals.callInfo.from = from;
req.locals.callInfo.to = to;
req.locals.callInfo.originatingSipIp = originatingSipIp;
req.locals.callInfo.originatingSipTrunkName = originatingSipTrunkName;
if (customerData) req.locals.callInfo.customerData = customerData;
delete app.callInfo;
}
next();
@@ -481,7 +471,7 @@ module.exports = function(srf, logger) {
message: `${err?.message}`.trim()
}).catch((err) => this.logger.info({err}, 'Error generating alert for parsing application'));
logger.info({err}, `Error retrieving or parsing application: ${err?.message}`);
res.send(JAMBONES_WEBHOOK_ERROR_RETURN, {headers: {'X-Reason': err?.message || 'unknown'}});
res.send(480, {headers: {'X-Reason': err?.message || 'unknown'}});
app.requestor.close(WS_CLOSE_CODES.GoingAway);
}
}

View File

@@ -10,9 +10,8 @@ const {
RecordState,
AllowedSipRecVerbs,
AllowedConfirmSessionVerbs,
TtsStreamingEvents,
ListenStatus
} = require('../utils/constants.json');
TtsStreamingEvents
} = require('../utils/constants');
const moment = require('moment');
const assert = require('assert');
const sessionTracker = require('./session-tracker');
@@ -30,6 +29,10 @@ const {
JAMBONES_INJECT_CONTENT,
JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
AWS_REGION,
JAMBONES_USE_FREESWITCH_TIMER_FD,
JAMBONES_MEDIA_TIMEOUT_MS,
JAMBONES_MEDIA_HOLD_TIMEOUT_MS,
JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS
} = require('../config');
const bent = require('bent');
const BackgroundTaskManager = require('../utils/background-task-manager');
@@ -37,8 +40,7 @@ const dbUtils = require('../utils/db-utils');
const BADPRECONDITIONS = 'preconditions not met';
const CALLER_CANCELLED_ERR_MSG = 'Response not sent due to unknown transaction';
const { NonFatalTaskError} = require('../utils/error');
const { createMediaEndpoint } = require('../utils/media-endpoint');
const SttLatencyCalculator = require('../utils/stt-latency-calculator');
const { sleepFor } = require('../utils/helpers');
const sqlRetrieveQueueEventHook = `SELECT * FROM webhooks
WHERE webhook_sid =
(
@@ -148,30 +150,6 @@ class CallSession extends Emitter {
this.conversationTurns = [];
this.on('userSaid', this._onUserSaid.bind(this));
this.on('botSaid', this._onBotSaid.bind(this));
/**
* Support STT latency
*/
this.sttLatencyCalculator = new SttLatencyCalculator({
logger,
cs: this
});
this.on('transcribe-start', () => {
this.sttLatencyCalculator.resetTime();
});
this.on('on-transcription', () => {
this.sttLatencyCalculator.onTranscriptionReceived();
});
this.on('transcribe-stop', () => {
this.sttLatencyCalculator.onTranscribeStop();
});
}
get notifySttLatencyEnabled() {
return this._notifySttLatencyEnabled || false;
}
set notifySttLatencyEnabled(enabled) {
this._notifySttLatencyEnabled = enabled;
}
/**
@@ -504,12 +482,7 @@ class CallSession extends Emitter {
}
get isTtsStreamEnabled() {
// 1st background tts stream
return this.backgroundTaskManager.isTaskRunning('ttsStream') ||
// 2nd current task streaming tts
TaskName.Say === this.currentTask?.name && this.currentTask?.isStreamingTts ||
// 3rd nested verb is streaming tts
TaskName.Gather === this.currentTask?.name && this.currentTask.sayTask?.isStreamingTts;
return this.backgroundTaskManager.isTaskRunning('ttsStream');
}
get isListenEnabled() {
@@ -663,15 +636,6 @@ class CallSession extends Emitter {
}
}
// disableTtsCache
get disableTtsCache() {
return this._disableTtsCache || false;
}
set disableTtsCache(d) {
this._disableTtsCache = d;
}
getTsStreamingVendor() {
let v;
if (this.currentTask?.isStreamingTts) {
@@ -724,7 +688,7 @@ class CallSession extends Emitter {
}
get hasGlobalSttPunctuation() {
hasGlobalSttPunctuation() {
return this._globalSttPunctuation !== undefined;
}
@@ -756,25 +720,10 @@ class CallSession extends Emitter {
return this._fillerNoise;
}
async pauseOrResumeBackgroundListenIfRequired(action, silence = false) {
if ((action == 'pauseCallRecording' || action == 'resumeCallRecording') &&
this.backgroundTaskManager.isTaskRunning('record')) {
this.logger.debug({action, silence}, 'CallSession:pauseOrResumeBackgroundListenIfRequired');
const backgroundListenTask = this.backgroundTaskManager.getTask('record');
const status = action === 'pauseCallRecording' ? ListenStatus.Pause : ListenStatus.Resume;
backgroundListenTask.updateListen(
status,
silence
);
}
}
async notifyRecordOptions(opts) {
const {action, silence} = opts;
const {action} = opts;
this.logger.debug({opts}, 'CallSession:notifyRecordOptions');
this.pauseOrResumeBackgroundListenIfRequired(action, silence);
/* if we have not answered yet, just save the details for later */
if (!this.dlg) {
if (action === 'startCallRecording') {
@@ -932,7 +881,7 @@ class CallSession extends Emitter {
this.logger.debug('CallSession:enableBackgroundTtsStream - ttsStream enabled');
} else {
this.logger.debug(
'CallSession:enableBackgroundTtsStream - ignoring request; conditions not met (probably not using ws api)');
'CallSession:enableBackgroundTtsStream - ignoring request as call does not have required conditions');
}
} catch (err) {
this.logger.info({err, say}, 'CallSession:enableBackgroundTtsStream - Error creating background tts stream task');
@@ -946,25 +895,15 @@ class CallSession extends Emitter {
}
}
clearTtsStream() {
if (this.isTtsStreamEnabled) {
this.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'user_interruption'})
.catch((err) => this.logger.info({err}, 'CallSession:clearTtsStream - Error sending user_interruption'));
this.ttsStreamingBuffer?.clear();
}
this.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'user_interruption'})
.catch((err) => this.logger.info({err}, 'CallSession:clearTtsStream - Error sending user_interruption'));
this.ttsStreamingBuffer?.clear();
}
startTtsStream() {
this.ttsStreamingBuffer?.start();
}
stopTtsStream() {
if (this.isTtsStreamEnabled) {
this.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_closed'})
.catch((err) => this.logger.info({err}, 'CallSession:clearTtsStream - Error sending user_interruption'));
this.ttsStreamingBuffer?.stop();
}
}
async enableBotMode(gather, autoEnable) {
try {
let task;
@@ -988,7 +927,7 @@ class CallSession extends Emitter {
task.sticky = autoEnable;
// listen to the bargein-done from background manager
this.backgroundTaskManager.on('bargeIn-done', () => {
if (this.appIsUsingWebsockets) {
if (this.requestor instanceof WsRequestor) {
try {
this.kill(true);
} catch (err) {}
@@ -1041,6 +980,8 @@ class CallSession extends Emitter {
(type === 'tts' && credential.use_for_tts) ||
(type === 'stt' && credential.use_for_stt)
)) {
this.logger.debug(
`${type}: ${credential.vendor} ${credential.label ? `, label: ${credential.label}` : ''} `);
if ('google' === vendor) {
if (type === 'tts' && !credential.tts_tested_ok ||
type === 'stt' && !credential.stt_tested_ok) {
@@ -1110,13 +1051,6 @@ class CallSession extends Emitter {
deepgram_stt_use_tls: credential.deepgram_stt_use_tls
};
}
else if ('gladia' === vendor) {
return {
speech_credential_sid: credential.speech_credential_sid,
api_key: credential.api_key,
region: credential.region,
};
}
else if ('soniox' === vendor) {
return {
speech_credential_sid: credential.speech_credential_sid,
@@ -1148,7 +1082,6 @@ class CallSession extends Emitter {
return {
api_key: credential.api_key,
model_id: credential.model_id,
api_uri: credential.api_uri,
options: credential.options
};
}
@@ -1164,7 +1097,6 @@ class CallSession extends Emitter {
return {
api_key: credential.api_key,
model_id: credential.model_id,
stt_model_id: credential.stt_model_id,
embedding: credential.embedding,
options: credential.options
};
@@ -1176,40 +1108,10 @@ class CallSession extends Emitter {
options: credential.options
};
}
else if ('resemble' === vendor) {
return {
api_key: credential.api_key,
resemble_tts_use_tls: credential.resemble_tts_use_tls,
resemble_tts_uri: credential.resemble_tts_uri,
};
}
else if ('inworld' === vendor) {
return {
api_key: credential.api_key,
model_id: credential.model_id,
options: credential.options
};
}
else if ('assemblyai' === vendor) {
return {
speech_credential_sid: credential.speech_credential_sid,
api_key: credential.api_key,
service_version: credential.service_version
};
}
else if ('houndify' === vendor) {
return {
speech_credential_sid: credential.speech_credential_sid,
client_id: credential.client_id,
client_key: credential.client_key,
user_id: credential.user_id,
houndify_server_uri: credential.houndify_server_uri
};
}
else if ('deepgramflux' === vendor) {
return {
speech_credential_sid: credential.speech_credential_sid,
api_key: credential.api_key,
api_key: credential.api_key
};
}
else if ('voxist' === vendor) {
@@ -1255,10 +1157,9 @@ class CallSession extends Emitter {
}
else {
writeAlerts({
alert_type: type === 'tts' ? AlertType.TTS_NOT_PROVISIONED : AlertType.STT_NOT_PROVISIONED,
alert_type: AlertType.STT_NOT_PROVISIONED,
account_sid: this.accountSid,
vendor,
label,
target_sid: this.callSid
}).catch((err) => this.logger.error({err}, 'Error writing tts alert'));
}
@@ -1289,7 +1190,6 @@ class CallSession extends Emitter {
this.ttsStreamingBuffer.on(TtsStreamingEvents.Pause, this._onTtsStreamingPause.bind(this));
this.ttsStreamingBuffer.on(TtsStreamingEvents.Resume, this._onTtsStreamingResume.bind(this));
this.ttsStreamingBuffer.on(TtsStreamingEvents.ConnectFailure, this._onTtsStreamingConnectFailure.bind(this));
this.ttsStreamingBuffer.on(TtsStreamingEvents.Connected, this._onTtsStreamingConnected.bind(this));
}
else {
this.logger.info(`CallSession:exec - not a normal call session: ${this.constructor.name}`);
@@ -1348,7 +1248,7 @@ class CallSession extends Emitter {
}
if (0 === this.tasks.length &&
this.appIsUsingWebsockets &&
this.requestor instanceof WsRequestor &&
!this.requestor.closedGracefully &&
!this.callGone &&
!this.isConfirmCallSession
@@ -1458,11 +1358,7 @@ class CallSession extends Emitter {
}
else {
if (this.req && !this.dlg) {
try {
this.req.cancel();
} catch (err) {
this.logger.error({err}, 'CallSession:_lccCallStatus error cancelling request');
}
this.req.cancel();
this._callReleased();
}
}
@@ -1525,14 +1421,6 @@ class CallSession extends Emitter {
method: 'POST'
};
}
/* if given a relative url then send to same base url as current session */
if (this.requestor._isRelativeUrl(opts.call_hook?.url)) {
opts.call_hook.url = `${this.requestor.baseUrl}${opts.call_hook?.url}`;
}
if (this.requestor._isRelativeUrl(opts.call_status_hook?.url)) {
opts.call_status_hook.url = `${this.requestor.baseUrl}${opts.call_status_hook?.url}`;
}
}
/**
@@ -1760,8 +1648,8 @@ Duration=${duration} `
async _lccWhisper(opts, callSid) {
const {whisper} = opts;
let tasks;
//const b3 = this.b3;
//const httpHeaders = b3 && {b3};
const b3 = this.b3;
const httpHeaders = b3 && {b3};
// this whole thing requires us to be in a Dial verb
const task = this.currentTask;
@@ -1770,15 +1658,12 @@ Duration=${duration} `
}
// allow user to provide a url object, a url string, an array of tasks, or a single task
// Disable passing a URL as not functional Sam Machin - 17/07/2025
//if (typeof whisper === 'string' || (typeof whisper === 'object' && whisper.url && !whisper.verb)) {
// // retrieve a url
// const json = await this.requestor(opts.call_hook, this.callInfo.toJSON(), httpHeaders);
// tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
//}
//else if (Array.isArray(whisper)) {
if (Array.isArray(whisper)) {
if (typeof whisper === 'string' || (typeof whisper === 'object' && whisper.url)) {
// retrieve a url
const json = await this.requestor(opts.call_hook, this.callInfo.toJSON(), httpHeaders);
tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
}
else if (Array.isArray(whisper)) {
// an inline array of tasks
tasks = normalizeJambones(this.logger, whisper).map((tdata) => makeTask(this.logger, tdata));
}
@@ -1909,7 +1794,7 @@ Duration=${duration} `
return;
}
else if (tokens === undefined) {
this.logger.info({opts}, 'CallSession:_lccTtsTokens - invalid command since tokens is missing');
this.logger.info({opts}, 'CallSession:_lccTtsTokens - invalid command since id is missing');
return this.requestor.request('tts:tokens-result', '/tokens-result', {
id,
status: 'failed',
@@ -2015,17 +1900,6 @@ Duration=${duration} `
}
} catch (err) {
this.logger.info({err, opts, callSid}, 'CallSession:updateCall - error updating call');
const {writeAlerts} = this.srf.locals;
try {
writeAlerts({
alert_type: 'error-updating-call',
account_sid: this.accountSid,
message: err.message,
target_sid: callSid
});
} catch (err) {
this.logger.error({err}, 'Error writing error-updating-call alert');
}
}
}
@@ -2427,22 +2301,23 @@ Duration=${duration} `
// need to allocate an endpoint
try {
const ep = await this._createMediaEndpoint({
if (!this.ms) this.ms = this.getMS();
const ep = await this.ms.createEndpoint({
headers: {
'X-Jambones-Call-ID': this.callId,
'X-Call-Sid': this.callSid,
},
remoteSdp: this.req.body
});
//ep.cs = this;
this.ep = ep;
this.logger.info(`allocated endpoint ${ep.uuid}`);
this._configMsEndpoint();
this.ep.on('destroy', () => {
this.logger.debug(`endpoint was destroyed!! ${this.ep.uuid}`);
});
if (this.direction === CallDirection.Inbound || this.application?.transferredCall) {
if (this.direction === CallDirection.Inbound) {
if (task.earlyMedia && !this.req.finalResponseSent) {
this.res.send(183, {body: ep.local.sdp});
return {ep};
@@ -2507,6 +2382,9 @@ Duration=${duration} `
this.logger.error('CallSession:replaceEndpoint cannot be called without stable dlg');
return;
}
// When this call kicked out from conference, session need to replace endpoint
// but this.ms might be undefined/null at this case.
this.ms = this.ms || this.getMS();
// Destroy previous ep if it's still running.
if (this.ep?.connected) this.ep.destroy();
@@ -2531,7 +2409,8 @@ Duration=${duration} `
* This prevents call failures during media renegotiation.
*/
this.ep = await this._createMediaEndpoint();
this.ep = await this.ms.createEndpoint();
this._configMsEndpoint();
const sdp = await this.dlg.modify(this.ep.local.sdp);
await this.ep.modify(sdp);
@@ -2574,9 +2453,7 @@ Duration=${duration} `
this.backgroundTaskManager.stopAll();
this.clearOrRestoreActionHookDelayProcessor().catch((err) => {});
this.stopTtsStream();
this.sttLatencyCalculator?.stop();
this.ttsStreamingBuffer?.stop();
}
/**
@@ -2728,7 +2605,7 @@ Duration=${duration} `
*/
_onRefer(req, res) {
const task = this.currentTask;
const sd = task?.sd;
const sd = task.sd;
if (task && TaskName.Dial === task.name && sd && task.referHook) {
task.handleRefer(this, req, res);
}
@@ -2793,8 +2670,15 @@ Duration=${duration} `
async createOrRetrieveEpAndMs() {
if (this.ms && this.ep) return {ms: this.ms, ep: this.ep};
// get a media server
if (!this.ms) {
const ms = this.srf.locals.getFreeswitch();
if (!ms) throw new Error('no available freeswitch');
this.ms = ms;
}
if (!this.ep) {
this.ep = await this._createMediaEndpoint({remoteSdp: this.req.body});
this.ep = await this.ms.createEndpoint({remoteSdp: this.req.body});
this._configMsEndpoint();
}
return {ms: this.ms, ep: this.ep};
}
@@ -2947,7 +2831,8 @@ Duration=${duration} `
async reAnchorMedia(currentMediaRoute = MediaPath.PartialMedia) {
assert(this.dlg && this.dlg.connected && !this.ep);
this.ep = await this._createMediaEndpoint({remoteSdp: this.dlg.remote.sdp});
this.ep = await this.ms.createEndpoint({remoteSdp: this.dlg.remote.sdp});
this._configMsEndpoint();
await this.dlg.modify(this.ep.local.sdp, {
headers: {
'X-Reason': 'anchor-media'
@@ -3022,26 +2907,72 @@ Duration=${duration} `
}
}
_configMsEndpoint() {
this._enableInbandDtmfIfRequired(this.ep);
this.ep.once('destroy', this._handleMediaTimeout.bind(this));
const opts = {
...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'}),
...(JAMBONES_MEDIA_TIMEOUT_MS && {media_timeout: JAMBONES_MEDIA_TIMEOUT_MS}),
...(JAMBONES_MEDIA_HOLD_TIMEOUT_MS && {media_hold_timeout: JAMBONES_MEDIA_HOLD_TIMEOUT_MS})
};
if (Object.keys(opts).length > 0) {
this.ep.set(opts);
}
const origDestroy = this.ep.destroy.bind(this.ep);
this.ep.destroy = async() => {
try {
if (this.currentTask?.name === TaskName.Transcribe && JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS) {
// transcribe task is being used, wait for some time before destroy
// if final transcription is received but endpoint is already closed,
// freeswitch module will not be able to send the transcription
this.logger.debug('callSession:_configMsEndpoint -' +
' transcribe task, wait for some time before destroy');
await sleepFor(JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS);
}
await origDestroy();
} catch (err) {
this.logger.error(err, 'callSession:_configMsEndpoint - error destroying endpoint');
}
};
}
async _handleMediaTimeout(evt) {
if (evt?.reason === 'MEDIA_TIMEOUT' && !this.callGone) {
if (evt.reason === 'MEDIA_TIMEOUT' && !this.callGone) {
this.logger.info('CallSession:_handleMediaTimeout: received MEDIA_TIMEOUT, hangup the call');
this._jambonzHangup('Media Timeout');
}
}
async _enableInbandDtmfIfRequired(ep) {
if (ep.inbandDtmfEnabled) return;
// only enable inband dtmf detection if voip carrier dtmf_type === tones
if (this.inbandDtmfEnabled) {
// https://developer.signalwire.com/freeswitch/FreeSWITCH-Explained/Modules/mod-dptools/6587132/#0-about
try {
ep.execute('start_dtmf');
ep.inbandDtmfEnabled = true;
} catch (err) {
this.logger.info(err, 'CallSession:_enableInbandDtmf - error enable inband DTMF');
}
}
}
/**
* notifyTaskError - only used when websocket connection is used instead of webhooks
*/
_notifyTaskError(obj) {
if (this.appIsUsingWebsockets) {
if (this.requestor instanceof WsRequestor) {
this.requestor.request('jambonz:error', '/error', obj)
.catch((err) => this.logger.debug({err}, 'CallSession:_notifyTaskError - Error sending'));
}
}
_notifyTaskStatus(task, evt) {
if (this.notifyEvents && this.appIsUsingWebsockets) {
if (this.notifyEvents && this.requestor instanceof WsRequestor) {
const obj = {...evt, id: task.id, name: task.name};
this.requestor.request('verb:status', '/status', obj)
.catch((err) => this.logger.debug({err}, 'CallSession:_notifyTaskStatus - Error sending'));
@@ -3080,20 +3011,8 @@ Duration=${duration} `
});
}
async _enableInbandDtmfIfRequired(ep) {
if (ep.inbandDtmfEnabled) return;
// only enable inband dtmf detection if voip carrier dtmf_type === tones
if (this.inbandDtmfEnabled) {
// https://developer.signalwire.com/freeswitch/FreeSWITCH-Explained/Modules/mod-dptools/6587132/#0-about
ep.execute('start_dtmf').catch((err) => {
this.logger.info({err}, 'CallSession:_enableInbandDtmfIfRequired - error starting DTMF');
});
ep.inbandDtmfEnabled = true;
}
}
_clearTasks(backgroundGather, evt) {
if (this.appIsUsingWebsockets && !backgroundGather.cleared) {
if (this.requestor instanceof WsRequestor && !backgroundGather.cleared) {
this.logger.debug({evt}, 'CallSession:_clearTasks on event from background gather');
try {
backgroundGather.cleared = true;
@@ -3121,20 +3040,10 @@ Duration=${duration} `
}
}
_onTtsStreamingConnected() {
this.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_open'})
.catch((err) => this.logger.info({err}, 'CallSession:_onTtsStreamingConnected - Error sending'));
}
_onTtsStreamingEmpty() {
const task = this.currentTask;
if (task && TaskName.Say === task.name) {
task.notifyTtsStreamIsEmpty();
} else if (
// If Gather nested say task is streaming
task && TaskName.Gather === task.name && task.sayTask && task.sayTask.isStreamingTts) {
const sayTask = task.sayTask;
sayTask.notifyTtsStreamIsEmpty();
}
}
@@ -3209,16 +3118,6 @@ Duration=${duration} `
}
}
async _createMediaEndpoint(drachtioFsmrfOptions = {}) {
return await createMediaEndpoint(this.srf, this.logger, {
activeMs: this.getMS(),
drachtioFsmrfOptions,
onHoldMusic: this.onHoldMusic,
inbandDtmfEnabled: this.inbandDtmfEnabled,
mediaTimeoutHandler: this._handleMediaTimeout.bind(this),
});
}
getFormattedConversation(numTurns) {
const turns = this.conversationTurns.slice(-numTurns);
if (turns.length === 0) return null;
@@ -3229,20 +3128,6 @@ Duration=${duration} `
return `assistant: ${t.text}`;
}).join('\n');
}
startSttLatencyVad() {
if (this.notifySttLatencyEnabled) {
this.sttLatencyCalculator.start();
}
}
stopSttLatencyVad() {
this.sttLatencyCalculator.stop();
}
calculateSttLatency() {
return this.sttLatencyCalculator.calculateLatency();
}
}
module.exports = CallSession;

View File

@@ -8,8 +8,7 @@ const CallSession = require('./call-session');
*/
class ConfirmCallSession extends CallSession {
// eslint-disable-next-line max-len
constructor({logger, application, dlg, ep, tasks, callInfo, accountInfo, memberId, confName, rootSpan, req, tmpFiles}) {
constructor({logger, application, dlg, ep, tasks, callInfo, accountInfo, memberId, confName, rootSpan, req}) {
super({
logger,
application,
@@ -25,7 +24,6 @@ class ConfirmCallSession extends CallSession {
this.dlg = dlg;
this.ep = ep;
this.req = req;
this.tmpFiles = tmpFiles;
}
/**

View File

@@ -22,12 +22,6 @@ class InboundCallSession extends CallSession {
this.req = req;
this.res = res;
// if the call was canceled before we got here, handle it
if (this.req.locals.canceled) {
req.locals.logger.info('InboundCallSession: constructor - call was already canceled');
this._onCancel();
}
req.once('cancel', this._onCancel.bind(this));
this.on('callStatusChange', this._notifyCallStatusChange.bind(this));

View File

@@ -27,13 +27,10 @@ class RestCallSession extends CallSession {
}
this.on('callStatusChange', this._notifyCallStatusChange.bind(this));
setImmediate(() => {
this._notifyCallStatusChange({
callStatus: CallStatus.Trying,
sipStatus: 100,
sipReason: 'Trying'
});
this._notifyCallStatusChange({
callStatus: CallStatus.Trying,
sipStatus: 100,
sipReason: 'Trying'
});
}

View File

@@ -45,11 +45,12 @@ class SipRecCallSession extends InboundCallSession {
async answerSipRecCall() {
try {
this.ms = this.getMS();
let remoteSdp = this.sdp1.replace(/sendonly/, 'sendrecv');
this.ep = await this._createMediaEndpoint({remoteSdp});
this.ep = await this.ms.createEndpoint({remoteSdp});
//this.logger.debug({remoteSdp, localSdp: this.ep.local.sdp}, 'SipRecCallSession - allocated first endpoint');
remoteSdp = this.sdp2.replace(/sendonly/, 'sendrecv');
this.ep2 = await this._createMediaEndpoint({remoteSdp});
this.ep2 = await this.ms.createEndpoint({remoteSdp});
//this.logger.debug({remoteSdp, localSdp: this.ep2.local.sdp}, 'SipRecCallSession - allocated second endpoint');
await this.ep.bridge(this.ep2);
const combinedSdp = await createSipRecPayload(this.ep.local.sdp, this.ep2.local.sdp, this.logger);

View File

@@ -1,31 +0,0 @@
const Task = require('./task');
const {TaskName} = require('../utils/constants');
class TaskAlert extends Task {
constructor(logger, opts, parentTask) {
super(logger, opts);
this.message = this.data.message;
}
get name() { return TaskName.Alert; }
async exec(cs) {
const {srf, accountSid:account_sid, callSid:target_sid, applicationSid:application_sid} = cs;
const {writeAlerts, AlertType} = srf.locals;
await super.exec(cs);
writeAlerts({
account_sid,
alert_type: AlertType.APPLICATION,
detail: `Application SID ${application_sid}`,
message: this.message,
target_sid
}).catch((err) => this.logger.info({err}, 'Error generating alert application'));
}
async kill(cs) {
super.kill(cs);
this.notifyTaskDone();
}
}
module.exports = TaskAlert;

View File

@@ -49,8 +49,7 @@ class Conference extends Task {
this.confName = this.data.name;
[
'beep', 'startConferenceOnEnter', 'endConferenceOnExit', 'joinMuted',
'maxParticipants', 'waitHook', 'statusHook', 'endHook', 'enterHook',
'endConferenceDuration', 'distributeDtmf'
'maxParticipants', 'waitHook', 'statusHook', 'endHook', 'enterHook', 'endConferenceDuration'
].forEach((attr) => this[attr] = this.data[attr]);
this.record = this.data.record || {};
this.statusEvents = [];
@@ -357,7 +356,6 @@ class Conference extends Task {
//https://developer.signalwire.com/freeswitch/FreeSWITCH-Explained/Modules/mod_conference_3965534/
// mute | Enter conference muted
...((this.joinMuted || this.speakOnlyTo) && {mute: true}),
...(this.distributeDtmf && {'dist-dtmf': true})
}});
/**
@@ -675,8 +673,7 @@ class Conference extends Task {
confName: this.confName,
tasks,
rootSpan: cs.rootSpan,
req: cs.req,
tmpFiles: cs.tmpFiles,
req: cs.req
});
await this._playSession.exec();
this._playSession = null;

View File

@@ -18,16 +18,12 @@ class TaskConfig extends Task {
'boostAudioSignal',
'vad',
'ttsStream',
'autoStreamTts',
'disableTtsCache'
'autoStreamTts'
].forEach((k) => this[k] = this.data[k] || {});
if ('notifyEvents' in this.data) {
this.notifyEvents = !!this.data.notifyEvents;
}
if (this.hasNotifySttLatency) {
this.notifySttLatency = !!this.data.notifySttLatency;
}
if (this.bargeIn.enable) {
this.gatherOpts = {
@@ -87,9 +83,7 @@ class TaskConfig extends Task {
get hasVad() { return Object.keys(this.vad).length; }
get hasFillerNoise() { return Object.keys(this.fillerNoise).length; }
get hasReferHook() { return Object.keys(this.data).includes('referHook'); }
get hasNotifySttLatency() { return Object.keys(this.data).includes('notifySttLatency'); }
get hasTtsStream() { return Object.keys(this.ttsStream).length; }
get hasDisableTtsCache() { return Object.keys(this.data).includes('disableTtsCache'); }
get summary() {
const phrase = [];
@@ -118,8 +112,6 @@ class TaskConfig extends Task {
if (this.hasFillerNoise) phrase.push(`fillerNoise ${this.fillerNoise.enable ? 'on' : 'off'}`);
if (this.data.amd) phrase.push('enable amd');
if (this.notifyEvents) phrase.push(`event notification ${this.notifyEvents ? 'on' : 'off'}`);
if (this.hasNotifySttLatency) phrase.push(
`notifySttLatency ${this.notifySttLatency ? 'on' : 'off'}`);
if (this.onHoldMusic) phrase.push(`onHoldMusic: ${this.onHoldMusic}`);
if ('boostAudioSignal' in this.data) phrase.push(`setGain ${this.data.boostAudioSignal}`);
if (this.hasReferHook) phrase.push('set referHook');
@@ -127,7 +119,6 @@ class TaskConfig extends Task {
phrase.push(`${this.ttsStream.enable ? 'enable' : 'disable'} ttsStream`);
}
if ('autoStreamTts' in this.data) phrase.push(`enable Say.stream value ${this.data.autoStreamTts ? 'on' : 'off'}`);
if (this.hasDisableTtsCache) phrase.push(`disableTtsCache ${this.data.disableTtsCache ? 'on' : 'off'}`);
return `${this.name}{${phrase.join(',')}}`;
}
@@ -139,11 +130,6 @@ class TaskConfig extends Task {
cs.notifyEvents = !!this.data.notifyEvents;
}
if (this.hasNotifySttLatency) {
this.logger.debug(`turning notifySttLatency ${this.notifySttLatency ? 'on' : 'off'}`);
cs.notifySttLatencyEnabled = this.notifySttLatency;
}
if (this.onHoldMusic) {
cs.onHoldMusic = this.onHoldMusic;
}
@@ -332,10 +318,7 @@ class TaskConfig extends Task {
voiceMs: this.vad.voiceMs || 250,
silenceMs: this.vad.silenceMs || 150,
strategy: this.vad.strategy || 'one-shot',
mode: (this.vad.mode !== undefined && this.vad.mode !== null) ? this.vad.mode : 2,
vendor: this.vad.vendor || 'silero',
threshold: this.vad.threshold || 0.5,
speechPadMs: this.vad.speechPadMs || 30,
mode: (this.vad.mode !== undefined && this.vad.mode !== null) ? this.vad.mode : 2
};
}
@@ -360,11 +343,6 @@ class TaskConfig extends Task {
this.logger.info('Config: disabling ttsStream');
cs.disableTtsStream();
}
if (this.hasDisableTtsCache) {
this.logger.info(`set disableTtsCache = ${this.disableTtsCache}`);
cs.disableTtsCache = this.data.disableTtsCache;
}
}
async kill(cs) {

View File

@@ -19,9 +19,9 @@ const parseDecibels = require('../utils/parse-decibels');
const debug = require('debug')('jambonz:feature-server');
const {parseUri} = require('drachtio-srf');
const {ANCHOR_MEDIA_ALWAYS,
JAMBONZ_DIAL_PAI_HEADER,
JAMBONZ_DISABLE_DIAL_PAI_HEADER,
JAMBONES_DIAL_SBC_FOR_REGISTERED_USER} = require('../config');
const { isOnhold, isOpusFirst, getLeadingCodec } = require('../utils/sdp-utils');
const { isOnhold, isOpusFirst } = require('../utils/sdp-utils');
const { normalizeJambones } = require('@jambonz/verb-specifications');
const { selectHostPort } = require('../utils/network');
const { sleepFor } = require('../utils/helpers');
@@ -109,7 +109,6 @@ class TaskDial extends Task {
this.tag = this.data.tag;
this.boostAudioSignal = this.data.boostAudioSignal;
this._mediaPath = MediaPath.FullMedia;
this.forwardPAI = this.data.forwardPAI;
if (this.dtmfHook) {
const {parentDtmfCollector, childDtmfCollector} = parseDtmfOptions(logger, this.data.dtmfCapture || {});
@@ -158,7 +157,6 @@ class TaskDial extends Task {
get canReleaseMedia() {
const keepAnchor = this.data.anchorMedia ||
this.isTranscoding ||
this.cs.isBackGroundListen ||
this.cs.onHoldMusic ||
ANCHOR_MEDIA_ALWAYS ||
@@ -399,11 +397,7 @@ class TaskDial extends Task {
...customHeaders
}
}, httpHeaders);
res.send(202);
this.logger.info('DialTask:handleRefer - sent 202 Accepted');
const returnedInstructions = !!json && Array.isArray(json);
if (returnedInstructions) {
if (json && Array.isArray(json)) {
try {
const logger = isChild ? this.logger : this.sd.logger;
const tasks = normalizeJambones(logger, json).map((tdata) => makeTask(this.logger, tdata));
@@ -421,21 +415,16 @@ class TaskDial extends Task {
/* need to update the callSid of the child with its own (new) AdultingCallSession */
sessionTracker.add(adultingSession.callSid, adultingSession);
}
if (this.ep) this.ep.unbridge();
/* if we got the REFER on the parent leg, end the dial task after completing the refer */
if (!isChild) {
logger.info('DialTask:handleRefer - killing dial task after processing REFER on parent leg');
cs.currentTask?.kill(cs, KillReason.ReferComplete);
}
}
} catch (err) {
this.logger.info(err, 'Dial:handleRefer - error setting new application after receiving REFER');
}
}
else {
this.logger.info('DialTask:handleRefer - no tasks returned from referHook, not setting new application');
}
//caller and callee legs are briged together, accept refer with 202 will release callee leg endpoint
//that makes freeswitch release endpoint for caller leg.
if (this.ep) this.ep.unbridge();
res.send(202);
this.logger.info('DialTask:handleRefer - sent 202 Accepted');
} catch (err) {
this.logger.info({err}, 'DialTask:handleRefer - error processing incoming REFER');
res.send(err.statusCode || 501);
@@ -550,14 +539,13 @@ class TaskDial extends Task {
let sbcAddress = this.proxy || getSBC();
const teamsInfo = {};
let fqdn;
const forwardPAI = this.forwardPAI ?? JAMBONZ_DIAL_PAI_HEADER; // dial verb overides env var
this.logger.debug(forwardPAI, 'forwardPAI value');
if (!sbcAddress) throw new Error('no SBC found for outbound call');
this.headers = {
'X-Account-Sid': cs.accountSid,
...(req && req.has('X-CID') && {'X-CID': req.get('X-CID')}),
...(direction === 'outbound' && callInfo.sbcCallid && {'X-CID': callInfo.sbcCallid}),
...(req && forwardPAI && {
...(!JAMBONZ_DISABLE_DIAL_PAI_HEADER && req && {
...(req.has('P-Asserted-Identity') && {'P-Asserted-Identity': req.get('P-Asserted-Identity')}),
...(req.has('Privacy') && {'Privacy': req.get('Privacy')}),
}),
@@ -576,7 +564,7 @@ class TaskDial extends Task {
proxy: `sip:${sbcAddress}`,
callingNumber: this.callerId || fromUri.user,
...(this.callerName && {callingName: this.callerName}),
opusFirst: isOpusFirst(this.cs.ep.local.sdp),
opusFirst: isOpusFirst(this.cs.ep.remote.sdp),
isVideoCall: this.cs.ep.remote.sdp.includes('m=video')
};
@@ -642,9 +630,7 @@ class TaskDial extends Task {
* trunk isn't specified,
* check if number matches any existing numbers
* */
const { lookupLcrByAccount} = srf.locals.dbHelpers;
const lcrs = await lookupLcrByAccount(cs.accountSid);
if (t.type === 'phone' && !t.trunk && lcrs.length == 0) {
if (t.type === 'phone' && !t.trunk) {
const str = this.callerId || req.callingNumber || '';
const callingNumber = str.startsWith('+') ? str.substring(1) : str;
const voip_carrier_sid = await lookupCarrierByPhoneNumber(cs.accountSid, callingNumber);
@@ -677,8 +663,7 @@ class TaskDial extends Task {
rootSpan: cs.rootSpan,
startSpan: this.startSpan.bind(this),
dialTask: this,
onHoldMusic: this.cs.onHoldMusic,
tmpFiles: this.cs.tmpFiles,
onHoldMusic: this.cs.onHoldMusic
});
this.dials.set(sd.callSid, sd);
@@ -773,24 +758,12 @@ class TaskDial extends Task {
}
async _connectSingleDial(cs, sd) {
// start connect with dialed leg, this is the soonest we can identify transcoding
if (this.epOther && sd.ep) {
const codecA = getLeadingCodec(this.epOther.local.sdp);
const codecB = getLeadingCodec(sd.ep.remote.sdp);
this.isTranscoding = (codecA !== codecB);
if (this.isTranscoding) {
this.logger.info(`Dial:_connectSingleDial - transcoding from ${codecA} (A leg) to ${codecB} (B leg)`);
}
}
if (!this.bridged && !this.canReleaseMedia) {
this.logger.debug('Dial:_connectSingleDial bridging endpoints');
if (this.epOther) {
this.epOther.api('uuid_break', this.epOther.uuid);
this.epOther.bridge(sd.ep);
}
else {
this.logger.error('Dial:_connectSingleDial - no other endpoint to bridge!');
}
this.bridged = true;
}
@@ -939,6 +912,7 @@ class TaskDial extends Task {
this.logger.info({err}, 'Dial:_selectSingleDial - Error boosting audio signal');
}
}
/* if we can release the media back to the SBC, do so now */
if (this.canReleaseMedia || this.shouldExitMediaPathEntirely) {
setTimeout(this._releaseMedia.bind(this, cs, sd, this.shouldExitMediaPathEntirely), 200);
@@ -1012,7 +986,7 @@ class TaskDial extends Task {
this.epOther = null;
this._mediaPath = releaseEntirely ? MediaPath.NoMedia : MediaPath.PartialMedia;
this.logger.info(
`Dial:_releaseMedia - successfully released media from freeswitch, media path is now ${this._mediaPath}`);
`Dial:_releaseMedia - successfully released media from freewitch, media path is now ${this._mediaPath}`);
} catch (err) {
this.logger.info({err}, 'Dial:_releaseMedia error');
}
@@ -1021,7 +995,7 @@ class TaskDial extends Task {
async reAnchorMedia(cs, sd) {
if (cs.ep && sd.ep) return;
this.logger.info('Dial:reAnchorMedia - re-anchoring media to freeswitch');
this.logger.info('Dial:reAnchorMedia - re-anchoring media to freewitch');
await Promise.all([sd.reAnchorMedia(this._mediaPath), cs.reAnchorMedia(this._mediaPath)]);
this.epOther = cs.ep;
@@ -1029,7 +1003,7 @@ class TaskDial extends Task {
this._mediaPath = MediaPath.FullMedia;
this.logger.info(
`Dial:_releaseMedia - successfully re-anchored media to freeswitch, media path is now ${this._mediaPath}`);
`Dial:_releaseMedia - successfully re-anchored media to freewitch, media path is now ${this._mediaPath}`);
}
// Handle RE-INVITE hold from caller leg.
@@ -1113,8 +1087,7 @@ class TaskDial extends Task {
accountInfo: this.cs.accountInfo,
tasks,
rootSpan: this.cs.rootSpan,
req: this.cs.req,
tmpFiles: this.cs.tmpFiles,
req: this.cs.req
});
await this._onHoldSession.exec();
this._onHoldSession = null;

View File

@@ -83,8 +83,7 @@ class TaskDub extends TtsTask {
action: 'playOnTrack',
track: this.track,
play: this.play,
// drachtio-fsmrf will convert loop from boolean to 'loop' or 'once'
loop: this.loop,
loop: this.loop ? 'loop' : 'once',
gain: this.gain
});
}

View File

@@ -370,8 +370,7 @@ class TaskEnqueue extends Task {
accountInfo: cs.accountInfo,
tasks: tasksToRun,
rootSpan: cs.rootSpan,
req: cs.req,
tmpFiles: cs.tmpFiles,
req: cs.req
});
await this._playSession.exec();
this._playSession = null;

View File

@@ -5,17 +5,13 @@ const {
AwsTranscriptionEvents,
AzureTranscriptionEvents,
DeepgramTranscriptionEvents,
GladiaTranscriptionEvents,
SonioxTranscriptionEvents,
CobaltTranscriptionEvents,
IbmTranscriptionEvents,
NvidiaTranscriptionEvents,
JambonzTranscriptionEvents,
AssemblyAiTranscriptionEvents,
HoundifyTranscriptionEvents,
DeepgramfluxTranscriptionEvents,
VoxistTranscriptionEvents,
CartesiaTranscriptionEvents,
OpenAITranscriptionEvents,
VadDetection,
VerbioTranscriptionEvents,
@@ -95,8 +91,6 @@ class TaskGather extends SttTask {
get needsStt() { return this.input.includes('speech'); }
get isBackgroundGather() { return this.bugname_prefix === 'background_bargeIn_'; }
get wantsSingleUtterance() {
return this.data.recognizer?.singleUtterance === true;
}
@@ -144,11 +138,7 @@ class TaskGather extends SttTask {
try {
await this.handling(cs, obj);
} catch (error) {
if (
// avoid bargein task with sticky will restart forever
// throw exception to stop the loop.
!this.sticky &&
error instanceof SpeechCredentialError) {
if (error instanceof SpeechCredentialError) {
this.logger.info('Gather failed due to SpeechCredentialError, finished!');
this.notifyTaskDone();
return;
@@ -231,9 +221,7 @@ class TaskGather extends SttTask {
const startListening = async(cs, ep) => {
this._startTimer();
if (this.isContinuousAsr && 0 === this.timeout && !this.isBackgroundGather) {
this._startAsrTimer();
}
if (this.isContinuousAsr && 0 === this.timeout) this._startAsrTimer();
if (this.input.includes('speech') && !this.listenDuringPrompt) {
try {
await this._setSpeechHandlers(cs, ep);
@@ -258,7 +246,7 @@ class TaskGather extends SttTask {
startDtmfListener();
}
this._stopVad();
if (!this.killed && !this.resolved) {
if (!this.killed) {
startListening(cs, ep);
if (this.input.includes('speech') && this.vendor === 'nuance' && this.listenDuringPrompt) {
this.logger.debug('Gather:exec - starting transcription timers after say completes');
@@ -270,22 +258,15 @@ class TaskGather extends SttTask {
};
this.sayTask.span = span;
this.sayTask.ctx = ctx;
this.sayTask
.exec(cs, {ep}) // kicked off, _not_ waiting for it to complete
.then(() => {
if (this.sayTask.isStreamingTts) return;
this.logger.debug('Gather:exec - nested say task completed');
span.end();
process();
return;
})
this.sayTask.exec(cs, {ep}) // kicked off, _not_ waiting for it to complete
.catch((err) => {
process();
});
if (this.sayTask.isStreamingTts && !this.sayTask.closeOnStreamEmpty) {
// if streaming tts, we do not wait for it to complete if it is not closing the stream automatically
this.sayTask.on('playDone', (err) => {
span.end();
if (err) this.logger.error({err}, 'Gather:exec Error playing tts');
process();
}
});
}
else if (this.playTask) {
const {span, ctx} = this.startChildSpan(`nested:${this.playTask.summary}`);
@@ -296,7 +277,7 @@ class TaskGather extends SttTask {
startDtmfListener();
}
this._stopVad();
if (!this.killed && !this.resolved) {
if (!this.killed) {
startListening(cs, ep);
if (this.input.includes('speech') && this.vendor === 'nuance' && this.listenDuringPrompt) {
this.logger.debug('Gather:exec - starting transcription timers after play completes');
@@ -308,17 +289,15 @@ class TaskGather extends SttTask {
};
this.playTask.span = span;
this.playTask.ctx = ctx;
this.playTask
.exec(cs, {ep}) // kicked off, _not_ waiting for it to complete
.then(() => {
this.logger.debug('Gather:exec - nested play task completed');
span.end();
process();
return;
})
this.playTask.exec(cs, {ep}) // kicked off, _not_ waiting for it to complete
.catch((err) => {
process();
});
this.playTask.on('playDone', (err) => {
span.end();
if (err) this.logger.error({err}, 'Gather:exec Error playing url');
process();
});
}
else {
if (this.killed) {
@@ -362,7 +341,6 @@ class TaskGather extends SttTask {
this.sayTask?.span.end();
this._stopVad();
this._resolve('killed');
cs.stopSttLatencyVad();
}
updateTaskInProgress(opts) {
@@ -378,9 +356,6 @@ class TaskGather extends SttTask {
_onDtmf(cs, ep, evt) {
this.logger.debug(evt, 'TaskGather:_onDtmf');
if (!this._timeoutTimer && this.timeout > 0) {
this._startTimer();
}
clearTimeout(this.interDigitTimer);
let resolved = false;
if (this.dtmfBargein) {
@@ -405,8 +380,12 @@ class TaskGather extends SttTask {
if (this.digitBuffer.length === 0 && this.needsStt) {
// DTMF is higher priority than STT.
this.removeCustomEventListeners();
this._clearAsrTimer(); //clear ASR timer as we're now using dtmf
this._stopTranscribing(ep);
ep.stopTranscription({
vendor: this.vendor,
bugname: this.bugname,
})
.catch((err) => this.logger.error({err},
` Received DTMF, Error stopping transcription for vendor ${this.vendor}`));
}
this.digitBuffer += evt.dtmf;
const len = this.digitBuffer.length;
@@ -420,7 +399,6 @@ class TaskGather extends SttTask {
const ms = this.interDigitTimeout * 1000;
this.logger.debug(`starting interdigit timer of ${ms}`);
this.interDigitTimer = setTimeout(() => this._resolve('dtmf-interdigit-timeout'), ms);
this._clearTimer(); //clear main timer as we're now using interdigit dtmf timer
}
}
@@ -478,28 +456,6 @@ class TaskGather extends SttTask {
this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, DeepgramTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep));
this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
break;
case 'deepgramflux':
this.bugname = `${this.bugname_prefix}deepgramflux_transcribe`;
this.addCustomEventListener(
ep, DeepgramfluxTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
this.addCustomEventListener(
ep, DeepgramfluxTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, DeepgramfluxTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep));
this.addCustomEventListener(ep, DeepgramfluxTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
break;
case 'gladia':
this.bugname = `${this.bugname_prefix}gladia_transcribe`;
this.addCustomEventListener(
ep, GladiaTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
this.addCustomEventListener(ep, GladiaTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, GladiaTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep));
this.addCustomEventListener(ep, GladiaTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
break;
case 'soniox':
@@ -579,18 +535,6 @@ class TaskGather extends SttTask {
this._onVendorConnectFailure.bind(this, cs, ep));
break;
case 'houndify':
this.bugname = `${this.bugname_prefix}houndify_transcribe`;
this.addCustomEventListener(ep, HoundifyTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep));
this.addCustomEventListener(ep, HoundifyTranscriptionEvents.Error,
this._onVendorError.bind(this, cs, ep));
this.addCustomEventListener(ep, HoundifyTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep));
this.addCustomEventListener(ep, HoundifyTranscriptionEvents.Connect,
this._onVendorConnect.bind(this, cs, ep));
break;
case 'voxist':
this.bugname = `${this.bugname_prefix}voxist_transcribe`;
this.addCustomEventListener(ep, VoxistTranscriptionEvents.Transcription,
@@ -602,17 +546,6 @@ class TaskGather extends SttTask {
this._onVendorConnectFailure.bind(this, cs, ep));
break;
case 'cartesia':
this.bugname = `${this.bugname_prefix}cartesia_transcribe`;
this.addCustomEventListener(ep, CartesiaTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep));
this.addCustomEventListener(
ep, CartesiaTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, CartesiaTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
this.addCustomEventListener(ep, CartesiaTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep));
break;
case 'speechmatics':
this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
this.addCustomEventListener(
@@ -728,18 +661,14 @@ class TaskGather extends SttTask {
target_sid: this.cs.callSid
});
}).catch((err) => this.logger.info({err}, 'Error generating alert for tts failure'));
// Some vendor use single connection, that we cannot use onConnect event to track transcription start
this.cs.emit('transcribe-start');
}
_startTimer() {
if (0 === this.timeout) return;
this.logger.debug(`Starting timoutTimer of ${this.timeout}ms`);
this._clearTimer();
this._timeoutTimer = setTimeout(() => {
// If continuousASR in use then extend by the asr window for more transcripts.
if (this.isContinuousAsr) this._startAsrTimer();
if (this.interDigitTimer) return; // let the inter-digit timer complete
else {
this._resolve(this.digitBuffer.length >= this.minDigits ? 'dtmf-num-digits' : 'timeout');
}
@@ -881,15 +810,17 @@ class TaskGather extends SttTask {
this._fillerNoiseOn = false; // in a race, if we just started audio it may sneak through here
this.ep.api('uuid_break', this.ep.uuid)
.catch((err) => this.logger.info(err, 'Error killing audio'));
if (cs.isTtsStreamEnabled) cs.clearTtsStream();
cs.clearTtsStream();
}
return;
}
if (this.sayTask && !this.sayTask.killed) {
this.sayTask.removeAllListeners('playDone');
this.sayTask.kill(cs);
this.sayTask = null;
}
if (this.playTask && !this.playTask.killed) {
this.playTask.removeAllListeners('playDone');
this.playTask.kill(cs);
this.playTask = null;
}
@@ -897,10 +828,6 @@ class TaskGather extends SttTask {
}
_onTranscription(cs, ep, evt, fsEvent) {
// check if we are in graceful shutdown mode
if (ep.gracefulShutdownResolver) {
ep.gracefulShutdownResolver();
}
// make sure this is not a transcript from answering machine detection
const bugname = fsEvent.getHeader('media-bugname');
const finished = fsEvent.getHeader('transcription-session-finished');
@@ -913,10 +840,6 @@ class TaskGather extends SttTask {
if (finished === 'true') return;
if (this.vendor === 'ibm' && evt?.state === 'listening') return;
// emit an event to the call session to track the time transcription is received
cs.emit('on-transcription');
if (this.vendor === 'deepgram' && evt.type === 'UtteranceEnd') {
/* we will only get this when we have set utterance_end_ms */
if (this._bufferedTranscripts.length === 0) {
@@ -944,7 +867,7 @@ class TaskGather extends SttTask {
evt = this.normalizeTranscription(evt, this.vendor, 1, this.language,
this.shortUtterance, this.data.recognizer.punctuation);
this.logger.debug({evt, bugname, finished, vendor: this.vendor}, 'Gather:_onTranscription normalized transcript');
//this.logger.debug({evt, bugname, finished, vendor: this.vendor}, 'Gather:_onTranscription normalized transcript');
if (evt.alternatives.length === 0) {
this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, continue listening');
@@ -1000,12 +923,6 @@ class TaskGather extends SttTask {
}
}
// receive a final transcript, calculate the stt latency for this transcript
const sttLatency = this.cs.calculateSttLatency();
if (!emptyTranscript && sttLatency) {
this.stt_latency_ms += `${sttLatency.stt_latency_ms},`;
}
if (this.isContinuousAsr) {
/* append the transcript and start listening again for asrTimeout */
const t = evt.alternatives[0].transcript;
@@ -1110,11 +1027,6 @@ class TaskGather extends SttTask {
this.cs.requestor.request('verb:hook', this.partialResultHook, Object.assign({speech: evt},
this.cs.callInfo, httpHeaders));
}
else if (this.vendor === 'deepgramflux' &&
['EagerEndOfTurn', 'TurnResumed'].includes(evt.vendor.evt?.event)) {
this.logger.debug(`Gather:_onTranscription - deepgramflux event detected: ${evt.event}`);
this.performAction({speech: evt, reason: 'speechDetected'}, false);
}
if (this.vendor === 'soniox') {
if (evt.vendor.finalWords.length) {
this.logger.debug({evt}, 'TaskGather:_onTranscription - buffering soniox transcript');
@@ -1161,8 +1073,12 @@ class TaskGather extends SttTask {
}
async _startFallback(cs, ep, evt) {
if (this.canFallback()) {
this._stopTranscribing(ep);
if (this.canFallback) {
ep.stopTranscription({
vendor: this.vendor,
bugname: this.bugname
})
.catch((err) => this.logger.error({err}, `Error stopping transcription for primary vendor ${this.vendor}`));
try {
this.logger.debug('gather:_startFallback');
this.notifyError({ msg: 'ASR error',
@@ -1291,26 +1207,17 @@ class TaskGather extends SttTask {
}
}
async _stopTranscribing(ep) {
// Fix for https://github.com/jambonz/jambonz-feature-server/issues/1281
// We should immediately call stop transcription from gather
// so that next gather can start transcription immediately
ep.stopTranscription({
vendor: this.vendor,
bugname: this.bugname,
gracefulShutdown: false
})
.catch((err) => {
if (this.resolved) return;
this.logger.error({err}, 'Error stopping transcription');
});
this.cs.emit('transcribe-stop');
}
async _resolve(reason, evt) {
this.logger.info({evt}, `TaskGather:resolve with reason ${reason}`);
if (this.needsStt && this.ep && this.ep.connected) {
this._stopTranscribing(this.ep);
this.ep.stopTranscription({
vendor: this.vendor,
bugname: this.bugname
})
.catch((err) => {
if (this.resolved) return;
this.logger.error({err}, 'Error stopping transcription');
});
}
if (this.resolved) {
this.logger.debug('TaskGather:_resolve - already resolved');
@@ -1318,8 +1225,6 @@ class TaskGather extends SttTask {
}
this.resolved = true;
// gather is resolved, prevent any further transcription events while resolve in progress
this.removeCustomEventListeners();
// If bargin is false and ws application return ack to verb:hook
// the gather should not play any audio
this._killAudio(this.cs);
@@ -1331,28 +1236,11 @@ class TaskGather extends SttTask {
this._clearAsrTimer();
this._clearFinalAsrTimer();
let sttLatencyMetrics = {};
if (this.needsStt) {
const sttLatency = this.cs.calculateSttLatency();
if (sttLatency) {
this.stt_latency_ms = this.stt_latency_ms.endsWith(',') ?
this.stt_latency_ms.slice(0, -1) : this.stt_latency_ms;
sttLatencyMetrics = {
'stt.latency_ms': this.stt_latency_ms,
'stt.talkspurts': JSON.stringify(sttLatency.talkspurts),
'stt.start_time': sttLatency.stt_start_time,
'stt.stop_time': sttLatency.stt_stop_time,
'stt.usage': sttLatency.stt_usage,
};
}
}
this.span.setAttributes({
channel: 1,
'stt.label': this.label || 'None',
'stt.resolve': reason,
'stt.result': JSON.stringify(evt),
...sttLatencyMetrics
'stt.result': JSON.stringify(evt)
});
if (this.callSession && this.callSession.callGone) {
@@ -1380,9 +1268,6 @@ class TaskGather extends SttTask {
let returnedVerbs = false;
try {
const latencies = Object.fromEntries(
Object.entries(sttLatencyMetrics).map(([key, value]) => [key.replace('stt.', 'stt_'), value])
);
if (reason.startsWith('dtmf')) {
if (this.parentTask) this.parentTask.emit('dtmf', evt);
else {
@@ -1396,7 +1281,7 @@ class TaskGather extends SttTask {
else {
this.emit('transcription', evt);
this.logger.debug('TaskGather:_resolve - invoking performAction');
returnedVerbs = await this.performAction({speech: evt, reason: 'speechDetected', ...latencies});
returnedVerbs = await this.performAction({speech: evt, reason: 'speechDetected'});
this.logger.debug({returnedVerbs}, 'TaskGather:_resolve - back from performAction');
}
}
@@ -1404,20 +1289,20 @@ class TaskGather extends SttTask {
if (this.parentTask) this.parentTask.emit('timeout', evt);
else {
this.emit('timeout', evt);
returnedVerbs = await this.performAction({reason: 'timeout', ...latencies});
returnedVerbs = await this.performAction({reason: 'timeout'});
}
}
else if (reason.startsWith('stt-error')) {
if (this.parentTask) this.parentTask.emit('stt-error', evt);
else {
this.emit('stt-error', evt);
returnedVerbs = await this.performAction({reason: 'error', details: evt.error, ...latencies});
returnedVerbs = await this.performAction({reason: 'error', details: evt.error});
}
} else if (reason.startsWith('stt-low-confidence')) {
if (this.parentTask) this.parentTask.emit('stt-low-confidence', evt);
else {
this.emit('stt-low-confidence', evt);
returnedVerbs = await this.performAction({speech:evt, reason: 'stt-low-confidence', ...latencies});
returnedVerbs = await this.performAction({speech:evt, reason: 'stt-low-confidence'});
}
}
} catch (err) { /*already logged error*/ }

View File

@@ -1,21 +1,10 @@
const Task = require('./task');
const {TaskName, TaskPreconditions, ListenEvents, ListenStatus} = require('../utils/constants.json');
const {TaskName, TaskPreconditions, ListenEvents, ListenStatus} = require('../utils/constants');
const makeTask = require('./make_task');
const moment = require('moment');
const MAX_PLAY_AUDIO_QUEUE_SIZE = 10;
const DTMF_SPAN_NAME = 'dtmf';
function escapeString(str) {
return str
.replace(/\\/g, '\\\\') // Escape backslashes
.replace(/"/g, '\\"') // Escape double quotes
.replace(/[\b]/g, '\\b') // Escape backspace (NOTE: [\b] not \b)
.replace(/\f/g, '\\f') // Escape formfeed
.replace(/\n/g, '\\n') // Escape newlines
.replace(/\r/g, '\\r') // Escape carriage returns
.replace(/\t/g, '\\t'); // Escape tabs
}
class TaskListen extends Task {
constructor(logger, opts, parentTask) {
super(logger, opts);
@@ -27,21 +16,10 @@ class TaskListen extends Task {
this.preconditions = TaskPreconditions.Endpoint;
[
'action', 'auth', 'method', 'url', 'finishOnKey', 'maxLength', 'mixType', 'passDtmf', 'playBeep',
'action', 'auth', 'method', 'url', 'finishOnKey', 'maxLength', 'metadata', 'mixType', 'passDtmf', 'playBeep',
'sampleRate', 'timeout', 'transcribe', 'wsAuth', 'disableBidirectionalAudio', 'channel'
].forEach((k) => this[k] = this.data[k]);
//Escape JSON special characters in metadata
if (this.data.metadata) {
this.metadata = {};
for (const key in this.data.metadata) {
if (this.data.metadata.hasOwnProperty(key)) {
const value = this.data.metadata[key];
this.metadata[key] = typeof value === 'string' ? escapeString(value) : value;
}
}
}
this.mixType = this.mixType || 'mono';
this.sampleRate = this.sampleRate || 8000;
this.earlyMedia = this.data.earlyMedia === true;
@@ -94,7 +72,7 @@ class TaskListen extends Task {
} catch (err) {
this.logger.info(err, `TaskListen:exec - error ${this.url}`);
}
if (this.transcribeTask) this.transcribeTask.kill(cs);
if (this.transcribeTask) this.transcribeTask.kill();
this._removeListeners(ep);
}
@@ -125,12 +103,9 @@ class TaskListen extends Task {
this.notifyTaskDone();
}
async updateListen(status, silence = false) {
async updateListen(status) {
if (!this.killed && this.ep && this.ep.connected) {
const args = [
...(this._bugname ? [this._bugname] : []),
...(status === ListenStatus.Pause ? ([silence]) : []),
];
const args = this._bugname ? [this._bugname] : [];
this.logger.info(`TaskListen:updateListen status ${status}`);
switch (status) {
case ListenStatus.Pause:

View File

@@ -6,6 +6,7 @@ const TaskLlmUltravox_S2S = require('./llms/ultravox_s2s');
const TaskLlmElevenlabs_S2S = require('./llms/elevenlabs_s2s');
const TaskLlmGoogle_S2S = require('./llms/google_s2s');
const LlmMcpService = require('../../utils/llm-mcp');
const TaskLlmAws_S2S = require('./llms/aws_s2s');
class TaskLlm extends Task {
constructor(logger, opts) {
@@ -85,6 +86,10 @@ class TaskLlm extends Task {
llm = new TaskLlmGoogle_S2S(this.logger, this.data, this);
break;
case 'aws':
llm = new TaskLlmAws_S2S(this.logger, this.data, this);
break;
default:
throw new Error(`Unsupported vendor ${this.vendor} for LLM`);
}

View File

@@ -0,0 +1,437 @@
const Task = require('../../task');
const crypto = require('crypto');
const TaskName = 'Llm_Aws_s2s';
const {LlmEvents_Aws} = require('../../../utils/constants');
const ClientEvent = 'client.event';
const SessionDelete = 'session.delete';
const aws_server_events = [
'error',
'session.created',
'session.updated',
];
const expandWildcards = (events) => {
const expandedEvents = [];
events.forEach((evt) => {
if (evt.endsWith('.*')) {
const prefix = evt.slice(0, -2); // Remove the wildcard ".*"
const matchingEvents = aws_server_events.filter((e) => e.startsWith(prefix));
expandedEvents.push(...matchingEvents);
} else {
expandedEvents.push(evt);
}
});
return expandedEvents;
};
class TaskLlmAws_S2S extends Task {
constructor(logger, opts, parentTask) {
super(logger, opts, parentTask);
this.parent = parentTask;
this.vendor = this.parent.vendor;
this.model = this.parent.model;
this.auth = this.parent.auth;
this.connectionOptions = this.parent.connectOptions;
const {access_key_id, secret_access_key, aws_region} = this.auth || {};
if (!access_key_id) throw new Error('auth.access_key_id is required for Aws S2S');
if (!secret_access_key) throw new Error('auth.secret_access_key is required for Aws S2S');
if (!aws_region) throw new Error('auth.aws_region is required for Aws S2S');
this.access_key_id = access_key_id;
this.secret_access_key = secret_access_key;
this.aws_region = aws_region;
this.actionHook = this.data.actionHook;
this.eventHook = this.data.eventHook;
this.toolHook = this.data.toolHook;
const {inferenceConfiguration, toolConfiguration, voiceId, systemPrompt} = this.data.llmOptions;
this.inferenceConfiguration = inferenceConfiguration;
this.toolConfiguration = toolConfiguration;
this.voiceId = voiceId;
this.systemPrompt = systemPrompt;
this.promptName = crypto.randomUUID();
this.results = {
completionReason: 'normal conversation end'
};
/**
* only one of these will have items,
* if includeEvents, then these are the events to include
* if excludeEvents, then these are the events to exclude
*/
this.includeEvents = [];
this.excludeEvents = [];
/* default to all events if user did not specify */
this._populateEvents(this.data.events || aws_server_events);
this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
}
get name() { return TaskName; }
get host() {
const {host} = this.connectionOptions || {};
return host || (this.vendor === 'aws' ? 'api.aws.com' : void 0);
}
get path() {
const {path} = this.connectionOptions || {};
if (path) return path;
switch (this.vendor) {
case 'aws':
return `v1/realtime?model=${this.model}`;
case 'microsoft':
return `aws/realtime?api-version=2024-10-01-preview&deployment=${this.model}`;
}
}
async _api(ep, args) {
const res = await ep.api('uuid_aws_s2s', `^^|${args.join('|')}`);
if (!res.body?.startsWith('+OK')) {
throw new Error({args}, `Error calling uuid_aws_s2s: ${res.body}`);
}
}
async exec(cs, {ep}) {
await super.exec(cs);
await this._startListening(cs, ep);
await this.awaitTaskDone();
/* note: the parent llm verb started the span, which is why this is necessary */
await this.parent.performAction(this.results);
this._unregisterHandlers();
}
async kill(cs) {
super.kill(cs);
this._api(cs.ep, [cs.ep.uuid, SessionDelete])
.catch((err) => this.logger.info({err}, 'TaskLlmAws_S2S:kill - error deleting session'));
this.notifyTaskDone();
}
/**
* Send function call output to the Aws server in the form of conversation.item.create
* per https://platform.aws.com/docs/guides/realtime/function-calls
*/
async processToolOutput(ep, tool_call_id, data) {
try {
this.logger.debug({tool_call_id, data}, 'TaskLlmAws_S2S:processToolOutput');
if (!data.type || data.type !== 'conversation.item.create') {
this.logger.info({data},
'TaskLlmAws_S2S:processToolOutput - invalid tool output, must be conversation.item.create');
}
else {
await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
// spec also recommends to send immediate response.create
await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify({type: 'response.create'})]);
}
} catch (err) {
this.logger.info({err}, 'TaskLlmAws_S2S:processToolOutput');
}
}
/**
* Send a session.update to the Aws server
* Note: creating and deleting conversation items also supported as well as interrupting the assistant
*/
async processLlmUpdate(ep, data, _callSid) {
try {
this.logger.debug({data, _callSid}, 'TaskLlmAws_S2S:processLlmUpdate');
if (!data.type || ![
'session.update',
'conversation.item.create',
'conversation.item.delete',
'response.cancel'
].includes(data.type)) {
this.logger.info({data}, 'TaskLlmAws_S2S:processLlmUpdate - invalid mid-call request');
}
else {
await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
}
} catch (err) {
this.logger.info({err}, 'TaskLlmAws_S2S:processLlmUpdate');
}
}
async _startListening(cs, ep) {
this._registerHandlers(ep);
try {
const args = [ep.uuid, 'session.create', this.aws_region, this.model, this.access_key_id, this.secret_access_key];
await this._api(ep, args);
} catch (err) {
this.logger.error({err}, 'TaskLlmAws_S2S:_startListening');
this.notifyTaskDone();
}
}
async _sendClientEvent(ep, obj) {
let ok = true;
this.logger.debug({obj}, 'TaskLlmAws_S2S:_sendClientEvent');
try {
const args = [ep.uuid, ClientEvent, JSON.stringify(obj)];
await this._api(ep, args);
} catch (err) {
ok = false;
this.logger.error({err}, 'TaskLlmAws_S2S:_sendClientEvent - Error');
}
return ok;
}
async _sendInitialMessage(ep) {
const settionStart = {
event: {
sessionStart: {
inferenceConfiguration: this.inferenceConfiguration || {
maxTokens: 1024,
temperature: 0.7,
topP: 0.9,
},
}
}
};
if (!await this._sendClientEvent(ep, settionStart)) {
return this.notifyTaskDone();
}
const promptStart = {
event: {
promptStart: {
promptName: this.promptName,
textOutputConfiguration: {
mediaType: 'text/plain',
},
audioOutputConfiguration: {
voiceId: this.voiceId || 'tiffany',
},
toolUseOutputConfiguration: {
mediaType: 'application/json',
},
...(this.toolConfiguration && {
toolConfiguration: this.toolConfiguration
})
}
}
};
if (!await this._sendClientEvent(ep, promptStart)) {
return this.notifyTaskDone();
}
if (this.systemPrompt) {
const contentName = crypto.randomUUID();
const systemPromptContentStart = {
event: {
contentStart: {
promptName: this.promptName,
contentName,
type: 'TEXT',
interactive: true,
role: 'SYSTEM',
textInputConfiguration: {
mediaType: 'text/plain',
}
}
}
};
if (!await this._sendClientEvent(ep, systemPromptContentStart)) {
return this.notifyTaskDone();
}
const systemPromptContent = {
event: {
textInput: {
promptName: this.promptName,
contentName,
content: this.systemPrompt
}
}
};
if (!await this._sendClientEvent(ep, systemPromptContent)) {
return this.notifyTaskDone();
}
const systemPromptContentEnd = {
event: {
contentEnd: {
promptName: this.promptName,
contentName,
}
}
};
if (!await this._sendClientEvent(ep, systemPromptContentEnd)) {
return this.notifyTaskDone();
}
}
const audioContentStart = {
event: {
contentStart: {
promptName: this.promptName,
contentName: crypto.randomUUID(),
type: 'AUDIO',
interactive: true,
role: 'USER',
}
}
};
if (!await this._sendClientEvent(ep, audioContentStart)) {
return this.notifyTaskDone();
}
}
_registerHandlers(ep) {
this.addCustomEventListener(ep, LlmEvents_Aws.Connect, this._onConnect.bind(this, ep));
this.addCustomEventListener(ep, LlmEvents_Aws.ConnectFailure, this._onConnectFailure.bind(this, ep));
this.addCustomEventListener(ep, LlmEvents_Aws.Disconnect, this._onDisconnect.bind(this, ep));
this.addCustomEventListener(ep, LlmEvents_Aws.ServerEvent, this._onServerEvent.bind(this, ep));
}
_unregisterHandlers() {
this.removeCustomEventListeners();
}
_onError(ep, evt) {
this.logger.info({evt}, 'TaskLlmAws_S2S:_onError');
this.notifyTaskDone();
}
_onConnect(ep) {
this.logger.debug('TaskLlmAws_S2S:_onConnect');
this._sendInitialMessage(ep);
}
_onConnectFailure(_ep, evt) {
this.logger.info(evt, 'TaskLlmAws_S2S:_onConnectFailure');
this.results = {completionReason: 'connection failure'};
this.notifyTaskDone();
}
_onDisconnect(_ep, evt) {
this.logger.info(evt, 'TaskLlmAws_S2S:_onConnectFailure');
this.results = {completionReason: 'disconnect from remote end'};
this.notifyTaskDone();
}
async _onServerEvent(ep, evt) {
let endConversation = false;
const type = evt.type;
this.logger.info({evt}, 'TaskLlmAws_S2S:_onServerEvent');
/* check for failures, such as rate limit exceeded, that should terminate the conversation */
if (type === 'response.done' && evt.response.status === 'failed') {
endConversation = true;
this.results = {
completionReason: 'server failure',
error: evt.response.status_details?.error
};
}
/* server errors of some sort */
else if (type === 'error') {
endConversation = true;
this.results = {
completionReason: 'server error',
error: evt.error
};
}
/* tool calls */
else if (type === 'response.output_item.done' && evt.item?.type === 'function_call') {
this.logger.debug({evt}, 'TaskLlmAws_S2S:_onServerEvent - function_call');
const {name, call_id} = evt.item;
const args = JSON.parse(evt.item.arguments);
const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
if (mcpTools.some((tool) => tool.name === name)) {
this.logger.debug({call_id, name, args}, 'TaskLlmAws_S2S:_onServerEvent - calling mcp tool');
try {
const res = await this.parent.mcpService.callMcpTool(name, args);
this.logger.debug({res}, 'TaskLlmAws_S2S:_onServerEvent - function_call - mcp result');
this.processToolOutput(ep, call_id, {
type: 'conversation.item.create',
item: {
type: 'function_call_output',
call_id,
output: res.content[0]?.text || 'There is no output from the function call',
}
});
return;
} catch (err) {
this.logger.info({err, evt}, 'TaskLlmAws_S2S - error calling function');
this.results = {
completionReason: 'client error calling mcp function',
error: err
};
endConversation = true;
}
}
else if (!this.toolHook) {
this.logger.warn({evt}, 'TaskLlmAws_S2S:_onServerEvent - no toolHook defined!');
}
else {
try {
await this.parent.sendToolHook(call_id, {name, args});
} catch (err) {
this.logger.info({err, evt}, 'TaskLlmAws - error calling function');
this.results = {
completionReason: 'client error calling function',
error: err
};
endConversation = true;
}
}
}
/* check whether we should notify on this event */
if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
this.parent.sendEventHook(evt)
.catch((err) => this.logger.info({err}, 'TaskLlmAws_S2S:_onServerEvent - error sending event hook'));
}
if (endConversation) {
this.logger.info({results: this.results}, 'TaskLlmAws_S2S:_onServerEvent - ending conversation due to error');
this.notifyTaskDone();
}
}
_populateEvents(events) {
if (events.includes('all')) {
/* work by excluding specific events */
const exclude = events
.filter((evt) => evt.startsWith('-'))
.map((evt) => evt.slice(1));
if (exclude.length === 0) this.includeEvents = aws_server_events;
else this.excludeEvents = expandWildcards(exclude);
}
else {
/* work by including specific events */
const include = events
.filter((evt) => !evt.startsWith('-'));
this.includeEvents = expandWildcards(include);
}
this.logger.debug({
includeEvents: this.includeEvents,
excludeEvents: this.excludeEvents
}, 'TaskLlmAws_S2S:_populateEvents');
}
}
module.exports = TaskLlmAws_S2S;

View File

@@ -32,14 +32,12 @@ class TaskLlmUltravox_S2S extends Task {
this.auth = this.parent.auth;
this.connectionOptions = this.parent.connectOptions;
const {apiKey, agent_id} = this.auth || {};
const {apiKey} = this.auth || {};
if (!apiKey) throw new Error('auth.apiKey is required for Vendor: Ultravox');
this.apiKey = apiKey;
this.agentId = agent_id;
this.actionHook = this.data.actionHook;
this.eventHook = this.data.eventHook;
this.toolHook = this.data.toolHook;
this.llmOptions = this.data.llmOptions || {};
this.results = {
completionReason: 'normal conversation end'
@@ -107,29 +105,24 @@ class TaskLlmUltravox_S2S extends Task {
}
);
// merge with any existing tools
this.llmOptions.selectedTools = [
this.data.llmOptions.selectedTools = [
...convertedTools,
...(this.llmOptions.selectedTools || [])
...(this.data.llmOptions.selectedTools || [])
];
}
const payload = {
...this.llmOptions,
...(!this.agentId && {
model: this.model,
}),
...this.data.llmOptions,
model: this.model,
medium: {
...(this.llmOptions.medium || {}),
...(this.data.llmOptions.medium || {}),
serverWebSocket: {
inputSampleRate: 8000,
outputSampleRate: 8000,
}
}
};
const baseUrl = 'https://api.ultravox.ai';
const url = this.agentId ?
`${baseUrl}/api/agents/${this.agentId}/calls` : `${baseUrl}/api/calls`;
const {statusCode, body} = await request(url, {
const {statusCode, body} = await request('https://api.ultravox.ai/api/calls', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -146,9 +139,8 @@ class TaskLlmUltravox_S2S extends Task {
return data;
}
_unregisterHandlers(ep) {
_unregisterHandlers() {
this.removeCustomEventListeners();
ep.removeAllListeners('dtmf');
}
_registerHandlers(ep) {
@@ -156,7 +148,6 @@ class TaskLlmUltravox_S2S extends Task {
this.addCustomEventListener(ep, LlmEvents_Ultravox.ConnectFailure, this._onConnectFailure.bind(this, ep));
this.addCustomEventListener(ep, LlmEvents_Ultravox.Disconnect, this._onDisconnect.bind(this, ep));
this.addCustomEventListener(ep, LlmEvents_Ultravox.ServerEvent, this._onServerEvent.bind(this, ep));
ep.on('dtmf', this._onDtmf.bind(this, ep));
}
async _startListening(cs, ep) {
@@ -191,7 +182,7 @@ class TaskLlmUltravox_S2S extends Task {
/* note: the parent llm verb started the span, which is why this is necessary */
await this.parent.performAction(this.results);
this._unregisterHandlers(ep);
this._unregisterHandlers();
}
async kill(cs) {
@@ -220,7 +211,7 @@ class TaskLlmUltravox_S2S extends Task {
async _onServerEvent(_ep, evt) {
let endConversation = false;
const type = evt.type;
//this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent');
this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent');
/* server errors of some sort */
if (type === 'error') {
@@ -348,18 +339,6 @@ class TaskLlmUltravox_S2S extends Task {
excludeEvents: this.excludeEvents
}, 'TaskLlmUltravox_S2S:_populateEvents');
}
_onDtmf(ep, evt) {
this.logger.info({evt}, 'TaskLlmUltravox_S2S:_onDtmf - DTMF received');
const {dtmf} = evt;
const data = {
type: 'user_text_message',
text: `DTMF received: ${dtmf}`,
urgency: 'immediate'
};
this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)])
.catch((err) => this.logger.info({err, evt}, 'TaskLlmUltravox_S2S:_onDtmf - Error sending DTMF as text message'));
}
}
module.exports = TaskLlmUltravox_S2S;

View File

@@ -96,9 +96,6 @@ function makeTask(logger, obj, parent) {
case TaskName.Tag:
const TaskTag = require('./tag');
return new TaskTag(logger, data, parent);
case TaskName.Alert:
const TaskAlert = require('./alert');
return new TaskAlert(logger, data, parent);
}
// should never reach

View File

@@ -6,21 +6,7 @@ class TaskPlay extends Task {
super(logger, opts);
this.preconditions = TaskPreconditions.Endpoint;
//Cleanup URLs that contain a querystring with a . unless that querystring is the filename
// see https://github.com/jambonz/jambonz-feature-server/pull/1293
// and https://github.com/jambonz/jambonz-feature-server/issues/1394 for background
if (this.data.url.includes('?')) {
if (['.mp3', '.wav'].includes(this.data.url.slice(-4))) {
this.url = this.data.url;
}
else {
this.url = this.data.url.split('?')[0] + '?' + this.data.url.split('?')[1].replaceAll('.', '%2E');
}
}
else {
this.url = this.data.url;
}
this.url = this.data.url;
this.seekOffset = this.data.seekOffset || -1;
this.timeoutSecs = this.data.timeoutSecs || -1;
this.loop = this.data.loop || 1;

View File

@@ -17,33 +17,28 @@ class TaskRedirect extends Task {
async exec(cs) {
await super.exec(cs);
const isAbsoluteUrl = cs.application?.requestor?._isAbsoluteUrl(this.actionHook);
if (isAbsoluteUrl) {
this.logger.info(`TaskRedirect redirecting to new absolute URL ${this.actionHook}, requires new requestor`);
if (cs.requestor instanceof WsRequestor) {
try {
const requestor = new WsRequestor(this.logger, cs.accountSid, {url: this.actionHook},
cs.accountInfo.account.webhook_secret) ;
cs.requestor.emit('handover', requestor);
} catch (err) {
this.logger.info(err, `TaskRedirect error redirecting to ${this.actionHook}`);
}
if (cs.requestor instanceof WsRequestor && cs.application.requestor._isAbsoluteUrl(this.actionHook)) {
this.logger.info(`Task:performAction redirecting to ${this.actionHook}, requires new ws connection`);
try {
this.cs.requestor.close();
const requestor = new WsRequestor(this.logger, cs.accountSid, {url: this.actionHook}, this.webhook_secret) ;
this.cs.application.requestor = requestor;
} catch (err) {
this.logger.info(err, `Task:performAction error redirecting to ${this.actionHook}`);
}
else {
const baseUrl = this.cs.application.requestor.baseUrl;
const newUrl = URL.parse(this.actionHook);
const newBaseUrl = newUrl.protocol + '//' + newUrl.host;
if (baseUrl != newBaseUrl) {
try {
this.logger.info(`Task:redirect updating base url to ${newBaseUrl}`);
const newRequestor = new HttpRequestor(this.logger, cs.accountSid, {url: this.actionHook},
cs.accountInfo.account.webhook_secret);
cs.requestor.emit('handover', newRequestor);
} catch (err) {
this.logger.info(err, `TaskRedirect error updating base url to ${this.actionHook}`);
}
} else if (cs.application.requestor._isAbsoluteUrl(this.actionHook)) {
const baseUrl = this.cs.application.requestor.baseUrl;
const newUrl = URL.parse(this.actionHook);
const newBaseUrl = newUrl.protocol + '//' + newUrl.host;
if (baseUrl != newBaseUrl) {
try {
this.logger.info(`Task:redirect updating base url to ${newBaseUrl}`);
const newRequestor = new HttpRequestor(this.logger, cs.accountSid, {url: this.actionHook},
cs.accountInfo.account.webhook_secret);
this.cs.requestor.removeAllListeners();
this.cs.application.requestor = newRequestor;
} catch (err) {
this.logger.info(err, `Task:redirect error updating base url to ${this.actionHook}`);
}
}
}

View File

@@ -19,7 +19,6 @@ class TaskRestDial extends Task {
this.timeout = this.data.timeout || 60;
this.sipRequestWithinDialogHook = this.data.sipRequestWithinDialogHook;
this.referHook = this.data.referHook;
this.recentCallStatus = 0;
this.on('connect', this._onConnect.bind(this));
this.on('callStatus', this._onCallStatus.bind(this));
@@ -58,11 +57,7 @@ class TaskRestDial extends Task {
this._clearCallTimer();
if (this.canCancel) {
this.canCancel = false;
try {
cs?.req?.cancel();
} catch (err) {
this.logger.error({err}, 'TaskRestDial: error cancelling call');
}
cs?.req?.cancel();
}
this.notifyTaskDone();
}
@@ -123,8 +118,7 @@ class TaskRestDial extends Task {
}
_onCallStatus(status) {
this.logger.debug(`RestDial CallStatus: ${status}`);
this.recentCallStatus = status;
this.logger.debug(`CallStatus: ${status}`);
if (status >= 200) {
this.canCancel = false;
this._clearCallTimer();
@@ -142,16 +136,11 @@ class TaskRestDial extends Task {
}
_onCallTimeout() {
this.logger.debug(`TaskRestDial: timeout expired without answer, last status ${this.recentCallStatus}`);
this.logger.debug('TaskRestDial: timeout expired without answer, killing task');
this.timer = null;
if (this.canCancel && this.recentCallStatus < 200) {
this.logger.debug('TaskRestDial: cancelling call attempt');
if (this.canCancel) {
this.canCancel = false;
try {
this.cs?.req?.cancel();
} catch (err) {
this.logger.error({err}, 'TaskRestDial: error cancelling call');
}
this.cs?.req?.cancel();
}
}

View File

@@ -1,38 +1,14 @@
const assert = require('assert');
const TtsTask = require('./tts-task');
const {TaskName, TaskPreconditions} = require('../utils/constants');
const {JAMBONES_SAY_CHUNK_SIZE} = require('../config');
const pollySSMLSplit = require('polly-ssml-split');
const { SpeechCredentialError, NonFatalTaskError } = require('../utils/error');
const { SpeechCredentialError } = require('../utils/error');
const { sleepFor } = require('../utils/helpers');
const { NON_FANTAL_ERRORS } = require('../utils/constants.json');
/**
* Discard unmatching responses:
* (1) I sent a playback id but get a response with a different playback id
* (2) I sent a playback id but get a response with no playback id
* (3) I did not send a playback id but get a response with a playback id
* (4) I sent a cache file but get a response with a different cache file
*/
const isMatchingEvent = (logger, filename, playbackId, evt) => {
if (!!playbackId && !!evt.variable_tts_playback_id && evt.variable_tts_playback_id === playbackId) {
//logger.debug({filename, playbackId, evt}, 'Say:isMatchingEvent - playbackId matched');
return true;
}
if (!!filename && !!evt.file && evt.file === filename) {
//logger.debug({filename, playbackId, evt}, 'Say:isMatchingEvent - filename matched');
return true;
}
logger.info({filename, playbackId, evt}, 'Say:isMatchingEvent - no match');
return false;
};
const breakLengthyTextIfNeeded = (logger, text) => {
// As The text can be used for tts streaming, we need to break lengthy text into smaller chunks
// HIGH_WATER_BUFFER_SIZE defined in tts-streaming-buffer.js
const chunkSize = JAMBONES_SAY_CHUNK_SIZE;
const chunkSize = 900;
const isSSML = text.startsWith('<speak>');
const options = {
softLimit: 100,
@@ -122,11 +98,13 @@ class TaskSay extends TtsTask {
}
if (this.isStreamingTts) await this.handlingStreaming(cs, obj);
else await this.handling(cs, obj);
this.emit('playDone');
} catch (error) {
if (error instanceof SpeechCredentialError) {
// if say failed due to speech credentials, alarm is writtern and error notification is sent
// finished this say to move to next task.
this.logger.info({error}, 'Say failed due to SpeechCredentialError, finished!');
this.emit('playDone');
return;
}
throw error;
@@ -147,6 +125,9 @@ class TaskSay extends TtsTask {
await cs.startTtsStream();
cs.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_open'})
.catch((err) => this.logger.info({err}, 'TaskSay:handlingStreaming - Error sending'));
if (this.text.length !== 0) {
this.logger.info('TaskSay:handlingStreaming - sending text to TTS stream');
for (const t of this.text) {
@@ -266,7 +247,6 @@ class TaskSay extends TtsTask {
throw new SpeechCredentialError(error.message);
}
};
let filepath;
try {
filepath = await this._synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label});
@@ -278,145 +258,77 @@ class TaskSay extends TtsTask {
while (!this.killed && (this.loop === 'forever' || this.loop--) && ep?.connected) {
let segment = 0;
while (!this.killed && segment < filepath.length) {
const filename = filepath[segment];
if (cs.isInConference) {
const {memberId, confName, confUuid} = cs;
await this.playToConfMember(ep, memberId, confName, confUuid, filename);
await this.playToConfMember(ep, memberId, confName, confUuid, filepath[segment]);
}
else {
const isStreaming = filename.startsWith('say:{');
const isStreaming = filepath[segment].startsWith('say:{');
if (isStreaming) {
const arr = /^say:\{.*\}\s*(.*)$/.exec(filename);
if (arr) this.logger.debug(`Say:exec sending streaming tts request ${arr[1].substring(0, 64)}..`);
else this.logger.debug(`Say:exec sending ${filename.substring(0, 64)}`);
const arr = /^say:\{.*\}\s*(.*)$/.exec(filepath[segment]);
if (arr) this.logger.debug(`Say:exec sending streaming tts request: ${arr[1].substring(0, 64)}..`);
}
const onPlaybackStop = (evt) => {
try {
const playbackId = this.getPlaybackId(segment);
const isMatch = isMatchingEvent(this.logger, filename, playbackId, evt);
if (!isMatch) {
this.logger.info({currentPlaybackId: playbackId, stopPlaybackId: evt.variable_tts_playback_id},
'Say:exec discarding playback-stop for earlier play');
ep.once('playback-stop', this._boundOnPlaybackStop);
return;
else this.logger.debug(`Say:exec sending ${filepath[segment].substring(0, 64)}`);
ep.once('playback-start', (evt) => {
this.logger.debug({evt}, 'Say got playback-start');
if (this.otelSpan) {
this._addStreamingTtsAttributes(this.otelSpan, evt, vendor);
this.otelSpan.end();
this.otelSpan = null;
if (evt.variable_tts_cache_filename) {
cs.trackTmpFile(evt.variable_tts_cache_filename);
}
this.logger.debug({evt},
`Say got playback-stop ${evt.variable_tts_playback_id ? evt.variable_tts_playback_id : ''}`);
this.notifyStatus({event: 'stop-playback'});
this.notifiedPlayBackStop = true;
const tts_error = evt.variable_tts_error;
// some tts vendor may not provide response code, so we default to 200
let response_code = 200;
// Check if any property ends with _response_code
for (const [key, value] of Object.entries(evt)) {
if (key.endsWith('_response_code')) {
response_code = parseInt(value, 10);
if (isNaN(response_code)) {
this.logger.info(`Say:exec playback-stop - Invalid response code: ${value}`);
response_code = 0;
}
break;
}
}
if (tts_error ||
// error response codes indicate failure
response_code <= 199 || response_code >= 300) {
writeAlerts({
account_sid,
alert_type: AlertType.TTS_FAILURE,
vendor,
detail: evt.variable_tts_error || `TTS playback failed with response code ${response_code}`,
target_sid
}).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
}
if (
!tts_error &&
//2xx response codes indicate success
199 < response_code && response_code < 300 &&
evt.variable_tts_cache_filename &&
!this.killed &&
// if tts cache is not disabled, add the file to cache
!this.disableTtsCache
) {
const text = parseTextFromSayString(this.text[segment]);
this.logger.debug({text, cacheFile: evt.variable_tts_cache_filename}, 'Say:exec cache tts');
addFileToCache(evt.variable_tts_cache_filename, {
account_sid,
vendor,
language,
voice,
engine,
model: this.model || this.model_id,
text,
instructions: this.instructions
}).catch((err) => this.logger.info({err}, 'Error adding file to cache'));
}
if (this._playResolve) {
(tts_error ||
// error response codes indicate failure
response_code <= 199 || response_code >= 300
) ?
this._playReject(
new Error(evt.variable_tts_error || `TTS playback failed with response code ${response_code}`)
) : this._playResolve();
}
} catch (err) {
this.logger.info({err}, 'Error handling playback-stop event');
}
};
this._boundOnPlaybackStop = onPlaybackStop.bind(this);
const onPlaybackStart = (evt) => {
try {
const playbackId = this.getPlaybackId(segment);
const isMatch = isMatchingEvent(this.logger, filename, playbackId, evt);
if (!isMatch) {
this.logger.info({currentPlaybackId: playbackId, startPlaybackId: evt.variable_tts_playback_id},
'Say:exec playback-start - unmatched playback_id');
ep.once('playback-start', this._boundOnPlaybackStart);
return;
});
ep.once('playback-stop', (evt) => {
this.logger.debug({evt}, 'Say got playback-stop');
this.notifyStatus({event: 'stop-playback'});
this.notifiedPlayBackStop = true;
const tts_error = evt.variable_tts_error;
let response_code = 200;
// Check if any property ends with _response_code
for (const [key, value] of Object.entries(evt)) {
if (key.endsWith('_response_code')) {
response_code = parseInt(value, 10) || 200;
break;
}
ep.once('playback-stop', this._boundOnPlaybackStop);
this.logger.debug({evt},
`Say got playback-start ${evt.variable_tts_playback_id ? evt.variable_tts_playback_id : ''}`);
if (this.otelSpan) {
this._addStreamingTtsAttributes(this.otelSpan, evt, vendor);
this.otelSpan.end();
this.otelSpan = null;
if (evt.variable_tts_cache_filename) {
cs.trackTmpFile(evt.variable_tts_cache_filename);
}
}
} catch (err) {
this.logger.info({err}, 'Error handling playback-start event');
}
};
this._boundOnPlaybackStart = onPlaybackStart.bind(this);
ep.once('playback-start', this._boundOnPlaybackStart);
if (tts_error) {
writeAlerts({
account_sid,
alert_type: AlertType.TTS_FAILURE,
vendor,
detail: evt.variable_tts_error,
target_sid
}).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
}
if (!tts_error && response_code < 300 && evt.variable_tts_cache_filename && !this.killed) {
const text = parseTextFromSayString(this.text[segment]);
addFileToCache(evt.variable_tts_cache_filename, {
account_sid,
vendor,
language,
voice,
engine,
model: this.model || this.model_id,
text,
instructions: this.instructions
}).catch((err) => this.logger.info({err}, 'Error adding file to cache'));
}
if (this._playResolve) {
(tts_error || response_code >= 300) ? this._playReject(new Error(evt.variable_tts_error)) :
this._playResolve();
}
});
// wait for playback-stop event received to confirm if the playback is successful
this._playPromise = new Promise((resolve, reject) => {
this._playResolve = resolve;
this._playReject = reject;
});
try {
const r = await ep.play(filename);
this.logger.debug({r}, 'Say:exec play result');
if (r.playbackSeconds == null && r.playbackMilliseconds == null && r.playbackLastOffsetPos == null) {
this._playReject(new Error('Playback failed to start'));
}
} catch (err) {
if (NON_FANTAL_ERRORS.includes(err.message)) {
throw new NonFatalTaskError(err.message);
}
throw err;
}
const r = await ep.play(filepath[segment]);
this.logger.debug({r}, 'Say:exec play result');
try {
// wait for playback-stop event received to confirm if the playback is successful
await this._playPromise;
@@ -433,12 +345,12 @@ class TaskSay extends TtsTask {
this._playResolve = null;
this._playReject = null;
}
if (filename.startsWith('say:{')) {
const arr = /^say:\{.*\}\s*(.*)$/.exec(filename);
if (filepath[segment].startsWith('say:{')) {
const arr = /^say:\{.*\}\s*(.*)$/.exec(filepath[segment]);
if (arr) this.logger.debug(`Say:exec complete playing streaming tts request: ${arr[1].substring(0, 64)}..`);
} else {
// This log will print spech credentials in say command for tts stream mode
this.logger.debug(`Say:exec completed play file ${filename}`);
this.logger.debug(`Say:exec completed play file ${filepath[segment]}`);
}
}
segment++;
@@ -454,8 +366,8 @@ class TaskSay extends TtsTask {
const {memberId, confName} = cs;
this.killPlayToConfMember(this.ep, memberId, confName);
} else if (this.isStreamingTts) {
this.logger.debug('TaskSay:kill - stopping TTS stream for streaming audio');
cs.stopTtsStream();
this.logger.debug('TaskSay:kill - clearing TTS stream for streaming audio');
cs.clearTtsStream();
} else {
if (!this.notifiedPlayBackStop) {
this.notifyStatus({event: 'stop-playback'});
@@ -485,8 +397,6 @@ class TaskSay extends TtsTask {
.replace('playht_', 'playht.')
.replace('cartesia_', 'cartesia.')
.replace('rimelabs_', 'rimelabs.')
.replace('resemble_', 'resemble.')
.replace('inworld_', 'inworld.')
.replace('verbio_', 'verbio.')
.replace('elevenlabs_', 'elevenlabs.');
if (spanMapping[newKey]) newKey = spanMapping[newKey];
@@ -551,14 +461,6 @@ const spanMapping = {
'rimelabs.name_lookup_time_ms': 'name_lookup_ms',
'rimelabs.connect_time_ms': 'connect_ms',
'rimelabs.final_response_time_ms': 'final_response_ms',
// Resemble
'resemble.connect_time_ms': 'connect_ms',
'resemble.final_response_time_ms': 'final_response_ms',
// inworld
'inworld.name_lookup_time_ms': 'name_lookup_ms',
'inworld.connect_time_ms': 'connect_ms',
'inworld.final_response_time_ms': 'final_response_ms',
'inworld.x_envoy_upstream_service_time': 'upstream_service_time',
// verbio
'verbio.name_lookup_time_ms': 'name_lookup_ms',
'verbio.connect_time_ms': 'connect_ms',

View File

@@ -111,12 +111,7 @@ class TaskSipRefer extends Task {
/* get IP address of the SBC to use as hostname if needed */
const {host} = parseUri(dlg.remote.uri);
if (
!referTo.startsWith('<') &&
!referTo.startsWith('sip:') &&
!referTo.startsWith('"') &&
!referTo.startsWith('tel:')
) {
if (!referTo.startsWith('<') && !referTo.startsWith('sip') && !referTo.startsWith('"')) {
/* they may have only provided a phone number/user */
referTo = `sip:${referTo}@${host}`;
}
@@ -129,12 +124,7 @@ class TaskSipRefer extends Task {
if (!referredByDisplayName) {
referredByDisplayName = cs.req?.callingName;
}
if (
!referredBy.startsWith('<') &&
!referredBy.startsWith('sip:') &&
!referredBy.startsWith('"') &&
!referredBy.startsWith('tel:')
) {
if (!referredBy.startsWith('<') && !referredBy.startsWith('sip') && !referredBy.startsWith('"')) {
/* they may have only provided a phone number/user */
referredBy = `${referredByDisplayName ? `"${referredByDisplayName}"` : ''}<sip:${referredBy}@${host}>`;
}

View File

@@ -4,7 +4,6 @@ const crypto = require('crypto');
const { TaskPreconditions, CobaltTranscriptionEvents } = require('../utils/constants');
const { SpeechCredentialError } = require('../utils/error');
const {JAMBONES_AWS_TRANSCRIBE_USE_GRPC} = require('../config');
const {TaskName} = require('../utils/constants.json');
/**
* "Please insert turns here: {{turns:4}}"
@@ -85,9 +84,6 @@ class SttTask extends Task {
/*bug name prefix */
this.bugname_prefix = '';
// stt latency calculator
this.stt_latency_ms = '';
}
async exec(cs, {ep, ep2}) {
@@ -95,12 +91,6 @@ class SttTask extends Task {
this.ep = ep;
this.ep2 = ep2;
// start vad from stt latency calculator
if (this.name !== TaskName.Gather ||
this.name === TaskName.Gather && this.needsStt) {
cs.startSttLatencyVad();
}
// use session preferences if we don't have specific verb-level settings.
if (cs.recognizer) {
for (const k in cs.recognizer) {
@@ -171,7 +161,7 @@ class SttTask extends Task {
try {
this.sttCredentials = await this._initSpeechCredentials(this.cs, this.vendor, this.label);
} catch (error) {
if (this.canFallback()) {
if (this.canFallback) {
this.notifyError(
{
msg: 'ASR error', details:`Invalid vendor ${this.vendor}, Error: ${error}`,
@@ -203,56 +193,6 @@ class SttTask extends Task {
if (cs.hasGlobalSttPunctuation && !this.data.recognizer.punctuation) {
this.data.recognizer.punctuation = cs.globalSttPunctuation;
}
if (this.vendor === 'gladia') {
const { api_key, region } = this.sttCredentials;
const {url} = await this.createGladiaLiveSession({
api_key, region,
model: this.data.recognizer.model || 'solaria-1',
options: this.data.recognizer.gladiaOptions || {}
});
const {host, pathname, search} = new URL(url);
this.sttCredentials.host = host;
this.sttCredentials.path = `${pathname}${search}`;
}
}
async createGladiaLiveSession({
api_key,
region = 'us-west',
model = 'solaria-1',
options = {},
}) {
const url = `https://api.gladia.io/v2/live?region=${region}`;
const response = await fetch(url, {
method: 'POST',
headers: {
'x-gladia-key': api_key,
'Content-Type': 'application/json'
},
body: JSON.stringify({
encoding: 'wav/pcm',
bit_depth: 16,
sample_rate: 8000,
channels: 1,
model,
...options,
messages_config: {
receive_final_transcripts: true,
receive_speech_events: true,
receive_errors: true,
}
})
});
if (!response.ok) {
const error = await response.text();
this.logger.error({url, status: response.status, error}, 'Error creating Gladia live session');
throw new Error(`Error creating Gladia live session: ${response.status} ${error}`);
}
const data = await response.json();
this.logger.debug({url: data.url}, 'Gladia Call registered');
return data;
}
addCustomEventListener(ep, event, handler) {
@@ -260,19 +200,8 @@ class SttTask extends Task {
ep.addCustomEventListener(event, handler);
}
removeCustomEventListeners(ep) {
if (ep) {
// for specific endpoint
this.eventHandlers.filter((h) => h.ep === ep).forEach((h) => {
h.ep.removeCustomEventListener(h.event, h.handler);
});
this.eventHandlers = this.eventHandlers.filter((h) => h.ep !== ep);
return;
} else {
// for all endpoints
this.eventHandlers.forEach((h) => h.ep.removeCustomEventListener(h.event, h.handler));
this.eventHandlers = [];
}
removeCustomEventListeners() {
this.eventHandlers.forEach((h) => h.ep.removeCustomEventListener(h.event, h.handler));
}
async _initSpeechCredentials(cs, vendor, label) {
@@ -286,7 +215,6 @@ class SttTask extends Task {
account_sid: cs.accountSid,
alert_type: AlertType.STT_NOT_PROVISIONED,
vendor,
label,
target_sid: cs.callSid
}).catch((err) => this.logger.info({err}, 'Error generating alert for no stt'));
// the ASR might have fallback configuration, should not done task here.
@@ -341,13 +269,11 @@ class SttTask extends Task {
return credentials;
}
canFallback() {
get canFallback() {
return this.fallbackVendor && this.isHandledByPrimaryProvider && !this.cs.hasFallbackAsr;
}
// ep is optional for gather or any verb that have single ep,
// but transcribe does need as it might has 2 eps
async _initFallback(ep) {
async _initFallback() {
assert(this.fallbackVendor, 'fallback failed without fallbackVendor configuration');
this.logger.info(`Failed to use primary STT provider, fallback to ${this.fallbackVendor}`);
this.isHandledByPrimaryProvider = false;
@@ -360,7 +286,7 @@ class SttTask extends Task {
this.data.recognizer.label = this.label;
this.sttCredentials = await this._initSpeechCredentials(this.cs, this.vendor, this.label);
// cleanup previous listener from previous vendor
this.removeCustomEventListeners(ep);
this.removeCustomEventListeners();
}
async compileHintsForCobalt(ep, hostport, model, token, hints) {
@@ -474,7 +400,7 @@ class SttTask extends Task {
dgOptions.utteranceEndMs = dgOptions.utteranceEndMs || asrTimeout;
}
_onVendorConnect(cs, _ep) {
_onVendorConnect(_cs, _ep) {
this.logger.debug(`TaskGather:_on${this.vendor}Connect`);
}
@@ -487,7 +413,6 @@ class SttTask extends Task {
message: 'STT failure reported by vendor',
detail: evt.error,
vendor: this.vendor,
label: this.label,
target_sid: cs.callSid
}).catch((err) => this.logger.info({err}, `Error generating alert for ${this.vendor} connection failure`));
}
@@ -501,7 +426,6 @@ class SttTask extends Task {
alert_type: AlertType.STT_FAILURE,
message: `Failed connecting to ${this.vendor} speech recognizer: ${reason}`,
vendor: this.vendor,
label: this.label,
target_sid: cs.callSid
}).catch((err) => this.logger.info({err}, `Error generating alert for ${this.vendor} connection failure`));
}

View File

@@ -6,8 +6,6 @@ const {
AwsTranscriptionEvents,
AzureTranscriptionEvents,
DeepgramTranscriptionEvents,
GladiaTranscriptionEvents,
DeepgramfluxTranscriptionEvents,
SonioxTranscriptionEvents,
CobaltTranscriptionEvents,
IbmTranscriptionEvents,
@@ -15,9 +13,7 @@ const {
JambonzTranscriptionEvents,
TranscribeStatus,
AssemblyAiTranscriptionEvents,
HoundifyTranscriptionEvents,
VoxistTranscriptionEvents,
CartesiaTranscriptionEvents,
OpenAITranscriptionEvents,
VerbioTranscriptionEvents,
SpeechmaticsTranscriptionEvents
@@ -70,9 +66,6 @@ class TaskTranscribe extends SttTask {
this._bufferedTranscripts = [ [], [] ]; // for channel 1 and 2
this.bugname_prefix = 'transcribe_';
this.paused = false;
// fallback flags
this.isHandledByPrimaryProviderForEp1 = true;
this.isHandledByPrimaryProviderForEp2 = true;
}
get name() { return TaskName.Transcribe; }
@@ -143,30 +136,22 @@ class TaskTranscribe extends SttTask {
stopTranscription = true;
this.ep.stopTranscription({
vendor: this.vendor,
bugname: this.bugname,
gracefulShutdown: this.paused ? false : true
bugname: this.bugname
})
.catch((err) => this.logger.info(err, 'Error TaskTranscribe:kill'));
}
if (this.transcribing2 && this.ep2?.connected) {
stopTranscription = true;
this.ep2.stopTranscription({
vendor: this.vendor,
bugname: this.bugname,
gracefulShutdown: this.paused ? false : true
})
this.ep2.stopTranscription({vendor: this.vendor, bugname: this.bugname})
.catch((err) => this.logger.info(err, 'Error TaskTranscribe:kill'));
}
this.cs.emit('transcribe-stop');
return stopTranscription;
}
async kill(cs) {
super.kill(cs);
const stopTranscription = this._stopTranscription();
cs.stopSttLatencyVad();
// hangup after 1 sec if we don't get a final transcription
if (stopTranscription) this._timer = setTimeout(() => this.notifyTaskDone(), 1500);
else this.notifyTaskDone();
@@ -242,35 +227,10 @@ class TaskTranscribe extends SttTask {
this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, DeepgramTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep, channel));
this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
/* if app sets deepgramOptions.utteranceEndMs they essentially want continuous asr */
//if (opts.DEEPGRAM_SPEECH_UTTERANCE_END_MS) this.isContinuousAsr = true;
break;
case 'deepgramflux':
this.bugname = `${this.bugname_prefix}deepgramflux_transcribe`;
this.addCustomEventListener(ep, DeepgramfluxTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
this.addCustomEventListener(ep, DeepgramfluxTranscriptionEvents.Connect,
this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, DeepgramfluxTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep, channel));
this.addCustomEventListener(ep, DeepgramfluxTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
break;
case 'gladia':
this.bugname = `${this.bugname_prefix}gladia_transcribe`;
this.addCustomEventListener(ep, GladiaTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
this.addCustomEventListener(ep, GladiaTranscriptionEvents.Connect,
this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, GladiaTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep, channel));
this.addCustomEventListener(ep, GladiaTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
break;
case 'soniox':
this.bugname = `${this.bugname_prefix}soniox_transcribe`;
@@ -341,18 +301,6 @@ class TaskTranscribe extends SttTask {
this._onVendorConnectFailure.bind(this, cs, ep, channel));
break;
case 'houndify':
this.bugname = `${this.bugname_prefix}houndify_transcribe`;
this.addCustomEventListener(ep, HoundifyTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
this.addCustomEventListener(ep, HoundifyTranscriptionEvents.Error,
this._onVendorError.bind(this, cs, ep));
this.addCustomEventListener(ep, HoundifyTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep, channel));
this.addCustomEventListener(ep, HoundifyTranscriptionEvents.Connect,
this._onVendorConnect.bind(this, cs, ep));
break;
case 'voxist':
this.bugname = `${this.bugname_prefix}voxist_transcribe`;
this.addCustomEventListener(ep, VoxistTranscriptionEvents.Transcription,
@@ -364,17 +312,6 @@ class TaskTranscribe extends SttTask {
this._onVendorConnectFailure.bind(this, cs, ep, channel));
break;
case 'cartesia':
this.bugname = `${this.bugname_prefix}cartesia_transcribe`;
this.addCustomEventListener(ep, CartesiaTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
this.addCustomEventListener(ep,
CartesiaTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, CartesiaTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
this.addCustomEventListener(ep, CartesiaTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep, channel));
break;
case 'speechmatics':
this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
this.addCustomEventListener(
@@ -469,16 +406,9 @@ class TaskTranscribe extends SttTask {
bugname: this.bugname,
hostport: this.hostport
});
// Some vendor use single connection, that we cannot use onConnect event to track transcription start
this.cs.emit('transcribe-start');
}
async _onTranscription(cs, ep, channel, evt, fsEvent) {
// check if we are in graceful shutdown mode
if (ep.gracefulShutdownResolver) {
ep.gracefulShutdownResolver();
}
// make sure this is not a transcript from answering machine detection
const bugname = fsEvent.getHeader('media-bugname');
const finished = fsEvent.getHeader('transcription-session-finished');
@@ -490,9 +420,6 @@ class TaskTranscribe extends SttTask {
if (this.vendor === 'ibm' && evt?.state === 'listening') return;
// emit an event to the call session to track the time transcription is received
cs.emit('on-transcription');
if (this.vendor === 'deepgram' && evt.type === 'UtteranceEnd') {
/* we will only get this when we have set utterance_end_ms */
@@ -654,28 +581,14 @@ class TaskTranscribe extends SttTask {
}
async _resolve(channel, evt) {
let sttLatencyMetrics = {};
if (evt.is_final) {
const sttLatency = this.cs.calculateSttLatency();
if (sttLatency) {
sttLatencyMetrics = {
'stt.latency_ms': `${sttLatency.stt_latency_ms}`,
'stt.talkspurts': JSON.stringify(sttLatency.talkspurts),
'stt.start_time': sttLatency.stt_start_time,
'stt.stop_time': sttLatency.stt_stop_time,
'stt.usage': sttLatency.stt_usage,
};
}
// time to reset the stt latency
this.cs.emit('transcribe-start');
/* we've got a final transcript, so end the otel child span for this channel */
if (this.childSpan[channel - 1] && this.childSpan[channel - 1].span) {
this.childSpan[channel - 1].span.setAttributes({
channel,
'stt.label': this.label || 'None',
'stt.resolve': 'transcript',
'stt.result': JSON.stringify(evt),
...sttLatencyMetrics
'stt.result': JSON.stringify(evt)
});
this.childSpan[channel - 1].span.end();
}
@@ -684,13 +597,9 @@ class TaskTranscribe extends SttTask {
if (this.transcriptionHook) {
const b3 = this.getTracingPropagation();
const httpHeaders = b3 && {b3};
const latencies = Object.fromEntries(
Object.entries(sttLatencyMetrics).map(([key, value]) => [key.replace('stt.', 'stt_'), value])
);
const payload = {
...this.cs.callInfo,
...httpHeaders,
...latencies,
...(evt.alternatives && {speech: evt}),
...(evt.type && {speechEvent: evt})
};
@@ -779,17 +688,16 @@ class TaskTranscribe extends SttTask {
}
async _startFallback(cs, _ep, evt) {
if (this.canFallback(_ep)) {
if (this.canFallback) {
_ep.stopTranscription({
vendor: this.vendor,
bugname: this.bugname,
gracefulShutdown: false
bugname: this.bugname
})
.catch((err) => this.logger.error({err}, `Error stopping transcription for primary vendor ${this.vendor}`));
try {
this.notifyError({ msg: 'ASR error',
details:`STT Vendor ${this.vendor} error: ${evt.error || evt.reason}`, failover: 'in progress'});
await this._initFallback(_ep);
await this._initFallback();
let channel = 1;
if (this.ep !== _ep) {
channel = 2;
@@ -898,41 +806,6 @@ class TaskTranscribe extends SttTask {
if (this._asrTimer) clearTimeout(this._asrTimer);
this._asrTimer = null;
}
// We need to keep track the fallback is happened for each endpoint
// override the canFallback and _initFallback methods to make sure that
// we only fallback once per endpoint
// we want to keep track this on task level instead of endpoint level
// because the endpoint instance is used across multiple tasks.
canFallback(ep) {
let isHandledByPrimaryProvider = this.isHandledByPrimaryProvider;
if (ep === this.ep) {
isHandledByPrimaryProvider = this.isHandledByPrimaryProviderForEp1;
} else if (ep === this.ep2) {
isHandledByPrimaryProvider = this.isHandledByPrimaryProviderForEp2;
}
const isOneOfEndpointAlreadyFallenBack = !!this.ep && !!this.ep2 &&
this.isHandledByPrimaryProviderForEp1 !== this.isHandledByPrimaryProviderForEp2;
// fallback is configured
return this.fallbackVendor &&
// has this endpoint already fallen back
isHandledByPrimaryProvider &&
// in global level, is there any fallback is already happened
// one fallen endpoint will mark cs.hasFallbackAsr to true,
// so if one endpoint was fallen, the other endpoint would be able to fallback.
(isOneOfEndpointAlreadyFallenBack || !this.cs.hasFallbackAsr);
}
_initFallback(ep) {
if (ep === this.ep) {
this.isHandledByPrimaryProviderForEp1 = false;
} else if (ep === this.ep2) {
this.isHandledByPrimaryProviderForEp2 = false;
}
return super._initFallback(ep);
}
}
module.exports = TaskTranscribe;

View File

@@ -3,16 +3,6 @@ const { TaskPreconditions } = require('../utils/constants');
const { SpeechCredentialError } = require('../utils/error');
const dbUtils = require('../utils/db-utils');
const extractPlaybackId = (str) => {
// Match say:{...} and capture the content inside braces
const match = str.match(/say:\{([^}]*)\}/);
if (!match) return null;
// Look for playback_id=value within the captured content
const playbackMatch = match[1].match(/playback_id=([^,]*)/);
return playbackMatch ? playbackMatch[1] : null;
};
class TtsTask extends Task {
constructor(logger, data, parentTask) {
@@ -32,19 +22,10 @@ class TtsTask extends Task {
this.disableTtsCache = this.data.disableTtsCache;
this.options = this.synthesizer.options || {};
this.instructions = this.data.instructions;
this.playbackIds = [];
}
getPlaybackId(offset) {
return this.playbackIds[offset];
}
async exec(cs) {
super.exec(cs);
// update disableTtsCache from call session if not set in task
if (this.data.disableTtsCache == null) {
this.disableTtsCache = cs.disableTtsCache;
}
if (cs.synthesizer) {
this.options = {...cs.synthesizer.options, ...this.options};
this.data.synthesizer = this.data.synthesizer || {};
@@ -85,67 +66,55 @@ class TtsTask extends Task {
}
async setTtsStreamingChannelVars(vendor, language, voice, credentials, ep) {
const {api_key, model_id, api_uri, custom_tts_streaming_url, auth_token, options} = credentials;
// api_key, model_id, api_uri, custom_tts_streaming_url, and auth_token are encoded in the credentials
// allow them to be overriden via config, using options
// give preference to options passed in via config
const parsed_options = options ? JSON.parse(options) : {};
const local_options = {...parsed_options, ...this.options};
const local_voice_settings = {...(parsed_options.voice_settings || {}), ...(this.options.voice_settings || {})};
const local_api_key = local_options.api_key ?? api_key;
const local_model_id = local_options.model_id ?? model_id;
const local_api_uri = local_options.api_uri ?? api_uri;
const local_custom_tts_streaming_url = local_options.custom_tts_streaming_url ?? custom_tts_streaming_url;
const local_auth_token = local_options.auth_token ?? auth_token;
const {api_key, model_id, custom_tts_streaming_url, auth_token} = credentials;
let obj;
this.logger.debug({credentials},
`setTtsStreamingChannelVars: vendor: ${vendor}, language: ${language}, voice: ${voice}`);
switch (vendor) {
case 'deepgram':
obj = {
DEEPGRAM_API_KEY: local_api_key,
DEEPGRAM_API_KEY: api_key,
DEEPGRAM_TTS_STREAMING_MODEL: voice
};
break;
case 'cartesia':
obj = {
CARTESIA_API_KEY: local_api_key,
CARTESIA_TTS_STREAMING_MODEL_ID: local_model_id,
CARTESIA_API_KEY: api_key,
CARTESIA_TTS_STREAMING_MODEL_ID: model_id,
CARTESIA_TTS_STREAMING_VOICE_ID: voice,
CARTESIA_TTS_STREAMING_LANGUAGE: language || 'en',
};
break;
case 'elevenlabs':
// eslint-disable-next-line max-len
const {stability, similarity_boost, use_speaker_boost, style, speed} = local_voice_settings || {};
const {stability, similarity_boost, use_speaker_boost, style, speed} = this.options.voice_settings || {};
obj = {
ELEVENLABS_API_KEY: local_api_key,
...(api_uri && {ELEVENLABS_API_URI: local_api_uri}),
ELEVENLABS_TTS_STREAMING_MODEL_ID: local_model_id,
ELEVENLABS_API_KEY: api_key,
ELEVENLABS_TTS_STREAMING_MODEL_ID: model_id,
ELEVENLABS_TTS_STREAMING_VOICE_ID: voice,
// 20/12/2024 - only eleven_turbo_v2_5 support multiple language
...(['eleven_turbo_v2_5'].includes(local_model_id) && {ELEVENLABS_TTS_STREAMING_LANGUAGE: language}),
...(['eleven_turbo_v2_5'].includes(model_id) && {ELEVENLABS_TTS_STREAMING_LANGUAGE: language}),
...(stability && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_STABILITY: stability}),
...(similarity_boost && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_SIMILARITY_BOOST: similarity_boost}),
...(use_speaker_boost && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_USE_SPEAKER_BOOST: use_speaker_boost}),
...(style && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_STYLE: style}),
// speed has value 0.7 to 1.2, 1.0 is default, make sure we send the value event it's 0
...(speed !== null && speed !== undefined && {ELEVENLABS_TTS_STREAMING_VOICE_SETTINGS_SPEED: `${speed}`}),
...(local_options.pronunciation_dictionary_locators &&
Array.isArray(local_options.pronunciation_dictionary_locators) && {
...(this.options.pronunciation_dictionary_locators &&
Array.isArray(this.options.pronunciation_dictionary_locators) && {
ELEVENLABS_TTS_STREAMING_PRONUNCIATION_DICTIONARY_LOCATORS:
JSON.stringify(local_options.pronunciation_dictionary_locators)
JSON.stringify(this.options.pronunciation_dictionary_locators)
}),
};
break;
case 'rimelabs':
const {
pauseBetweenBrackets, phonemizeBetweenBrackets, inlineSpeedAlpha, speedAlpha, reduceLatency
} = local_options;
} = this.options;
obj = {
RIMELABS_API_KEY: local_api_key,
RIMELABS_TTS_STREAMING_MODEL_ID: local_model_id,
RIMELABS_API_KEY: api_key,
RIMELABS_TTS_STREAMING_MODEL_ID: model_id,
RIMELABS_TTS_STREAMING_VOICE_ID: voice,
RIMELABS_TTS_STREAMING_LANGUAGE: language || 'en',
...(pauseBetweenBrackets && {RIMELABS_TTS_STREAMING_PAUSE_BETWEEN_BRACKETS: pauseBetweenBrackets}),
@@ -160,8 +129,8 @@ class TtsTask extends Task {
if (vendor.startsWith('custom:')) {
const use_tls = custom_tts_streaming_url.startsWith('wss://');
obj = {
CUSTOM_TTS_STREAMING_HOST: local_custom_tts_streaming_url.replace(/^(ws|wss):\/\//, ''),
CUSTOM_TTS_STREAMING_API_KEY: local_auth_token,
CUSTOM_TTS_STREAMING_HOST: custom_tts_streaming_url.replace(/^(ws|wss):\/\//, ''),
CUSTOM_TTS_STREAMING_API_KEY: auth_token,
CUSTOM_TTS_STREAMING_VOICE_ID: voice,
CUSTOM_TTS_STREAMING_LANGUAGE: language || 'en',
CUSTOM_TTS_STREAMING_USE_TLS: use_tls
@@ -216,9 +185,6 @@ class TtsTask extends Task {
} else if (vendor === 'rimelabs') {
credentials = credentials || {};
credentials.model_id = this.options.model_id || credentials.model_id;
} else if (vendor === 'inworld') {
credentials = credentials || {};
credentials.model_id = this.options.model_id || credentials.model_id;
} else if (vendor === 'whisper') {
credentials = credentials || {};
credentials.model_id = this.options.model_id || credentials.model_id;
@@ -274,16 +240,15 @@ class TtsTask extends Task {
account_sid,
alert_type: AlertType.TTS_NOT_PROVISIONED,
vendor,
label,
target_sid: cs.callSid
}).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
throw new SpeechCredentialError('no provisioned speech credentials for TTS');
}
/* produce an audio segment from the provided text */
const generateAudio = async(text, index) => {
if (this.killed) return {index, filePath: null};
if (text.startsWith('silence_stream://')) return {index, filePath: text};
const generateAudio = async(text) => {
if (this.killed) return;
if (text.startsWith('silence_stream://')) return text;
/* otel: trace time for tts */
if (!preCache && !this._disableTracing) {
@@ -325,35 +290,16 @@ class TtsTask extends Task {
vendor,
language,
characters: text.length,
elapsedTime: rtt,
servedFromCache,
'id': this.id
elapsedTime: rtt
});
}
if (servedFromCache) {
this.notifyStatus({
event: 'synthesized-audio',
vendor,
language,
servedFromCache,
'id': this.id
});
}
return {index, filePath, playbackId: null};
}
else {
const playbackId = extractPlaybackId(filePath);
this.logger.debug('Say: a streaming tts api will be used');
const modifiedPath = filePath.replace('say:{', `say:{session-uuid=${ep.uuid},`);
this.notifyStatus({
event: 'synthesized-audio',
vendor,
language,
servedFromCache,
'id': this.id
});
return {index, filePath: modifiedPath, playbackId};
return modifiedPath;
}
return filePath;
} catch (err) {
this.logger.info({err}, 'Error synthesizing tts');
if (this.otelSpan) this.otelSpan.end();
@@ -361,7 +307,6 @@ class TtsTask extends Task {
account_sid: cs.accountSid,
alert_type: AlertType.TTS_FAILURE,
vendor,
label,
detail: err.message,
target_sid: cs.callSid
}).catch((err) => this.logger.info({err}, 'Error generating alert for tts failure'));
@@ -369,20 +314,8 @@ class TtsTask extends Task {
}
};
// process all text segments in parallel will cause ordering issue
// so we attach index to each promise result and sort them later
const arr = this.text.map((t, index) => (this._validateURL(t) ?
Promise.resolve({index, filePath: t, playbackId: null}) : generateAudio(t, index)));
const results = await Promise.all(arr);
const sorted = results.sort((a, b) => a.index - b.index);
return sorted
.filter((fp) => fp.filePath && fp.filePath.length)
.map((r) => {
this.playbackIds.push(r.playbackId);
return r.filePath;
});
const arr = this.text.map((t) => (this._validateURL(t) ? t : generateAudio(t)));
return (await Promise.all(arr)).filter((fp) => fp && fp.length);
} catch (err) {
this.logger.info(err, 'TaskSay:exec error');
throw err;

View File

@@ -281,17 +281,13 @@ module.exports = (logger) => {
/* set stt options */
logger.info(`starting amd for vendor ${vendor} and language ${language}`);
/* if opts contains recognizer object use that config for stt, otherwise use defaults */
const rOpts = opts.recognizer ?
opts.recognizer :
{
vendor,
hints,
enhancedModel: true,
altLanguages: opts.recognizer?.altLanguages || [],
initialSpeechTimeoutMs: opts.resolveTimeoutMs,
};
const sttOpts = amd.setChannelVarsForStt({name: TaskName.Gather}, sttCredentials, language, rOpts);
const sttOpts = amd.setChannelVarsForStt({name: TaskName.Gather}, sttCredentials, language, {
vendor,
hints,
enhancedModel: true,
altLanguages: opts.recognizer?.altLanguages || [],
initialSpeechTimeoutMs: opts.resolveTimeoutMs,
});
await ep.set(sttOpts).catch((err) => logger.info(err, 'Error setting channel variables'));
@@ -405,30 +401,24 @@ module.exports = (logger) => {
if (ep.amd) {
vendor = ep.amd.vendor;
ep.amd.stopAllTimers();
try {
ep.removeListener(GoogleTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(GoogleTranscriptionEvents.EndOfUtterance, ep.amd.EndOfUtteranceHandler);
ep.removeListener(AwsTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(AzureTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(AzureTranscriptionEvents.NoSpeechDetected, ep.amd.noSpeechHandler);
ep.removeListener(NuanceTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(DeepgramTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(SonioxTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(IbmTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(NvidiaTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(JambonzTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
} catch (error) {
logger.error('Unable to Remove AMD Listener', error);
}
ep.removeListener(GoogleTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(GoogleTranscriptionEvents.EndOfUtterance, ep.amd.EndOfUtteranceHandler);
ep.removeListener(AwsTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(AzureTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(AzureTranscriptionEvents.NoSpeechDetected, ep.amd.noSpeechHandler);
ep.removeListener(NuanceTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(DeepgramTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(SonioxTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(IbmTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(NvidiaTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.removeListener(JambonzTranscriptionEvents.Transcription, ep.amd.transcriptionHandler);
ep.amd = null;
}
if (ep.connected) {
ep.stopTranscription({
vendor,
bugname,
gracefulShutdown: false
})
ep.stopTranscription({vendor, bugname})
.catch((err) => logger.info(err, 'stopAmd: Error stopping transcription'));
task.emit('amd', {type: AmdEvents.Stopped});
ep.execute('avmd_stop').catch((err) => this.logger.info(err, 'Error stopping avmd'));

View File

@@ -65,7 +65,7 @@ class BackgroundTaskManager extends Emitter {
this.logger.info(`stopping background task: ${type}`);
task.removeAllListeners();
task.span.end();
task.kill(this.cs);
task.kill();
// Remove task from managed List
this.tasks.delete(type);
}

View File

@@ -1,6 +1,5 @@
{
"TaskName": {
"Alert": "alert",
"Answer": "answer",
"Conference": "conference",
"Config": "config",
@@ -94,20 +93,7 @@
"DeepgramTranscriptionEvents": {
"Transcription": "deepgram_transcribe::transcription",
"ConnectFailure": "deepgram_transcribe::connect_failed",
"Connect": "deepgram_transcribe::connect",
"Error": "deepgram_transcribe::error"
},
"DeepgramfluxTranscriptionEvents": {
"Transcription": "deepgramflux_transcribe::transcription",
"ConnectFailure": "deepgramflux_transcribe::connect_failed",
"Connect": "deepgramflux_transcribe::connect",
"Error": "deepgramflux_transcribe::error"
},
"GladiaTranscriptionEvents": {
"Transcription": "gladia_transcribe::transcription",
"ConnectFailure": "gladia_transcribe::connect_failed",
"Connect": "gladia_transcribe::connect",
"Error": "gladia_transcribe::error"
"Connect": "deepgram_transcribe::connect"
},
"SonioxTranscriptionEvents": {
"Transcription": "soniox_transcribe::transcription",
@@ -175,30 +161,15 @@
"ConnectFailure": "assemblyai_transcribe::connect_failed",
"Connect": "assemblyai_transcribe::connect"
},
"HoundifyTranscriptionEvents": {
"Transcription": "houndify_transcribe::transcription",
"Error": "houndify_transcribe::error",
"ConnectFailure": "houndify_transcribe::connect_failed",
"Connect": "houndify_transcribe::connect"
},
"VoxistTranscriptionEvents": {
"Transcription": "voxist_transcribe::transcription",
"Error": "voxist_transcribe::error",
"ConnectFailure": "voxist_transcribe::connect_failed",
"Connect": "voxist_transcribe::connect"
},
"CartesiaTranscriptionEvents": {
"Transcription": "cartesia_transcribe::transcription",
"Error": "cartesia_transcribe::error",
"ConnectFailure": "cartesia_transcribe::connect_failed",
"Connect": "cartesia_transcribe::connect"
},
"VadDetection": {
"Detection": "vad_detect:detection"
},
"SileroVadDetection": {
"Detection": "vad_silero:detect"
},
"ListenEvents": {
"Connect": "mod_audio_fork::connect",
"ConnectFailure": "mod_audio_fork::connect_failed",
@@ -223,6 +194,13 @@
"Disconnect": "openai_s2s::disconnect",
"ServerEvent": "openai_s2s::server_event"
},
"LlmEvents_Aws": {
"Error": "error",
"Connect": "aws_s2s::connect",
"ConnectFailure": "aws_s2s::connect_failed",
"Disconnect": "aws_s2s::disconnect",
"ServerEvent": "aws_s2s::server_event"
},
"LlmEvents_Google": {
"Error": "error",
"Connect": "google_s2s::connect",
@@ -266,7 +244,6 @@
"KillReason": {
"Hangup": "hangup",
"Replaced": "replaced",
"ReferComplete": "refer-complete",
"MediaTimeout": "media_timeout"
},
"HookMsgTypes": [
@@ -335,8 +312,7 @@
"Empty": "tts_streaming::empty",
"Pause": "tts_streaming::pause",
"Resume": "tts_streaming::resume",
"ConnectFailure": "tts_streaming::connect_failed",
"Connected": "tts_streaming::connected"
"ConnectFailure": "tts_streaming::connect_failed"
},
"TtsStreamingConnectionStatus": {
"NotConnected": "not_connected",
@@ -356,8 +332,5 @@
"WS_CLOSE_CODES": {
"NormalClosure": 1000,
"GoingAway": 1001
},
"NON_FANTAL_ERRORS": [
"File Not Found"
]
}
}

View File

@@ -81,15 +81,6 @@ const speechMapper = (cred) => {
obj.deepgram_tts_uri = o.deepgram_tts_uri;
obj.deepgram_stt_use_tls = o.deepgram_stt_use_tls;
}
else if ('gladia' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
obj.region = o.region;
}
else if ('deepgramflux' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
}
else if ('soniox' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
@@ -106,7 +97,6 @@ const speechMapper = (cred) => {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
obj.model_id = o.model_id;
obj.api_uri = o.api_uri;
obj.options = o.options;
}
else if ('playht' === obj.vendor) {
@@ -120,7 +110,6 @@ const speechMapper = (cred) => {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
obj.model_id = o.model_id;
obj.stt_model_id = o.stt_model_id;
obj.embedding = o.embedding;
obj.options = o.options;
}
@@ -130,29 +119,9 @@ const speechMapper = (cred) => {
obj.model_id = o.model_id;
obj.options = o.options;
}
else if ('resemble' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
obj.resemble_tts_use_tls = o.resemble_tts_use_tls;
obj.resemble_tts_uri = o.resemble_tts_uri;
}
else if ('inworld' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
obj.model_id = o.model_id;
obj.options = o.options;
}
else if ('assemblyai' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
obj.service_version = o.service_version;
}
else if ('houndify' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.client_id = o.client_id;
obj.client_key = o.client_key;
obj.user_id = o.user_id;
obj.houndify_server_uri = o.houndify_server_uri;
}
else if ('voxist' === obj.vendor) {
const o = JSON.parse(decrypt(credential));

View File

@@ -41,7 +41,7 @@ class HttpRequestor extends BaseRequestor {
constructor(logger, account_sid, hook, secret) {
super(logger, account_sid, hook, secret);
this.method = hook.method?.toUpperCase() || 'POST';
this.method = hook.method || 'POST';
this.authHeader = basicAuth(hook.username, hook.password);
this.backoffMs = 500;
@@ -111,7 +111,7 @@ class HttpRequestor extends BaseRequestor {
const payload = params ? snakeCaseKeys(params, ['customerData', 'sip', 'env_vars', 'args']) : null;
const url = hook.url || hook;
const method = hook.method?.toUpperCase() || 'POST';
const method = hook.method || 'POST';
let buf = '';
httpHeaders = {
...httpHeaders,
@@ -119,7 +119,7 @@ class HttpRequestor extends BaseRequestor {
};
assert.ok(url, 'HttpRequestor:request url was not provided');
assert.ok(['GET', 'POST'].includes(method), `HttpRequestor:request method must be 'GET' or 'POST' not ${method}`);
assert.ok, (['GET', 'POST'].includes(method), `HttpRequestor:request method must be 'GET' or 'POST' not ${method}`);
const startAt = process.hrtime();
/* if we have an absolute url, and it is ws then do a websocket connection */
@@ -191,7 +191,7 @@ class HttpRequestor extends BaseRequestor {
method,
headers: hdrs,
...('POST' === method && {body: JSON.stringify(payload)}),
headersTimeout: HTTP_TIMEOUT,
timeout: HTTP_TIMEOUT,
followRedirects: false
};

View File

@@ -173,8 +173,7 @@ function installSrfLocals(srf, logger, {
lookupAccountCapacitiesBySid,
lookupSmppGateways,
lookupClientByAccountAndUsername,
lookupSystemInformation,
lookupLcrByAccount
lookupSystemInformation
} = require('@jambonz/db-helpers')({
host: JAMBONES_MYSQL_HOST,
user: JAMBONES_MYSQL_USER,
@@ -280,8 +279,7 @@ function installSrfLocals(srf, logger, {
retrieveByPatternSortedSet,
sortedSetLength,
sortedSetPositionByPattern,
getVerbioAccessToken,
lookupLcrByAccount
getVerbioAccessToken
},
parentLogger: logger,
getSBC,

View File

@@ -1,115 +0,0 @@
const {
JAMBONES_USE_FREESWITCH_TIMER_FD,
JAMBONES_MEDIA_TIMEOUT_MS,
JAMBONES_MEDIA_HOLD_TIMEOUT_MS,
JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS,
} = require('../config');
const { sleepFor } = require('./helpers');
const createMediaEndpoint = async(srf, logger, {
activeMs,
drachtioFsmrfOptions = {},
onHoldMusic,
inbandDtmfEnabled,
mediaTimeoutHandler,
} = {}) => {
const { getFreeswitch } = srf.locals;
const ms = activeMs || getFreeswitch();
if (!ms)
throw new Error('no available Freeswitch for creating media endpoint');
const ep = await ms.createEndpoint(drachtioFsmrfOptions);
// Configure the endpoint
const opts = {
...(onHoldMusic && {holdMusic: `shout://${onHoldMusic.replace(/^https?:\/\//, '')}`}),
...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'}),
...(JAMBONES_MEDIA_TIMEOUT_MS && {media_timeout: JAMBONES_MEDIA_TIMEOUT_MS}),
...(JAMBONES_MEDIA_HOLD_TIMEOUT_MS && {media_hold_timeout: JAMBONES_MEDIA_HOLD_TIMEOUT_MS})
};
if (Object.keys(opts).length > 0) {
ep.set(opts);
}
// inbandDtmfEnabled
if (inbandDtmfEnabled) {
// https://developer.signalwire.com/freeswitch/FreeSWITCH-Explained/Modules/mod-dptools/6587132/#0-about
ep.execute('start_dtmf').catch((err) => {
logger.error('Error starting inband DTMF', { error: err });
});
ep.inbandDtmfEnabled = true;
}
// Handle Media Timeout
if (mediaTimeoutHandler) {
ep.once('destroy', (evt) => {
mediaTimeoutHandler(evt, ep);
});
}
// Handle graceful shutdown for endpoint if required
if (JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS > 0) {
const getEpGracefulShutdownPromise = () => {
if (!ep.gracefulShutdownPromise) {
ep.gracefulShutdownPromise = new Promise((resolve) => {
// this resolver will be called when stt task received transcription.
ep.gracefulShutdownResolver = () => {
resolve();
ep.gracefulShutdownPromise = null;
};
});
}
return ep.gracefulShutdownPromise;
};
const gracefulShutdownHandler = async() => {
// resolve when one of the following happens:
// 1. stt task received transcription
// 2. JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS passed
await Promise.race([
getEpGracefulShutdownPromise(),
sleepFor(JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS)
]);
};
const origStartTranscription = ep.startTranscription.bind(ep);
ep.startTranscription = async(...args) => {
try {
const result = await origStartTranscription(...args);
ep.isTranscribeActive = true;
return result;
} catch (err) {
ep.isTranscribeActive = false;
throw err;
}
};
const origStopTranscription = ep.stopTranscription.bind(ep);
ep.stopTranscription = async(opts = {}, ...args) => {
const { gracefulShutdown = true, ...others } = opts;
if (ep.isTranscribeActive && gracefulShutdown) {
// only wait for graceful shutdown if transcription is active
await gracefulShutdownHandler();
}
try {
const result = await origStopTranscription({...others}, ...args);
ep.isTranscribeActive = false;
return result;
} catch (err) {
ep.isTranscribeActive = false;
throw err;
}
};
const origDestroy = ep.destroy.bind(ep);
ep.destroy = async() => {
if (ep.isTranscribeActive) {
await gracefulShutdownHandler();
}
return await origDestroy();
};
}
return ep;
};
module.exports = {
createMediaEndpoint,
};

View File

@@ -16,11 +16,17 @@ const crypto = require('crypto');
const HttpRequestor = require('./http-requestor');
const WsRequestor = require('./ws-requestor');
const {makeOpusFirst, removeVideoSdp} = require('./sdp-utils');
const { createMediaEndpoint } = require('./media-endpoint');
const {
JAMBONES_USE_FREESWITCH_TIMER_FD,
JAMBONES_MEDIA_TIMEOUT_MS,
JAMBONES_MEDIA_HOLD_TIMEOUT_MS,
JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS
} = require('../config');
const { sleepFor } = require('./helpers');
class SingleDialer extends Emitter {
constructor({logger, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask,
onHoldMusic, tmpFiles}) {
onHoldMusic}) {
super();
assert(target.type);
@@ -44,7 +50,6 @@ class SingleDialer extends Emitter {
this.callSid = crypto.randomUUID();
this.dialTask = dialTask;
this.onHoldMusic = onHoldMusic;
this.tmpFiles = tmpFiles;
this.on('callStatusChange', this._notifyCallStatusChange.bind(this));
}
@@ -93,7 +98,6 @@ class SingleDialer extends Emitter {
};
}
this.ms = ms;
this.srf = srf;
let uri, to, inviteSpan;
try {
switch (this.target.type) {
@@ -135,7 +139,8 @@ class SingleDialer extends Emitter {
this.updateCallStatus = srf.locals.dbHelpers.updateCallStatus;
this.serviceUrl = srf.locals.serviceUrl;
this.ep = await this._createMediaEndpoint();
this.ep = await ms.createEndpoint();
this._configMsEndpoint();
this.logger.debug(`SingleDialer:exec - created endpoint ${this.ep.uuid}`);
/**
@@ -329,13 +334,7 @@ class SingleDialer extends Emitter {
*/
async kill(Reason) {
this.killed = true;
if (this.inviteInProgress) {
try {
await this.inviteInProgress.cancel();
} catch (err) {
this.logger.error({err}, 'SingleDialer:kill error cancelling invite');
}
}
if (this.inviteInProgress) await this.inviteInProgress.cancel();
else if (this.dlg && this.dlg.connected) {
const duration = moment().diff(this.dlg.connectTime, 'seconds');
this.logger.debug('SingleDialer:kill hanging up called party');
@@ -353,19 +352,43 @@ class SingleDialer extends Emitter {
}
}
async _handleMediaTimeout(evt, ep) {
this.logger.info({evt}, 'SingleDialer:_handleMediaTimeout - media timeout event received');
this.dialTask.kill(this.dialTask.cs, 'media-timeout');
}
_configMsEndpoint() {
const opts = {
...(this.onHoldMusic && {holdMusic: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`}),
...(JAMBONES_USE_FREESWITCH_TIMER_FD && {timer_name: 'timerfd'}),
...(JAMBONES_MEDIA_TIMEOUT_MS && {media_timeout: JAMBONES_MEDIA_TIMEOUT_MS}),
...(JAMBONES_MEDIA_HOLD_TIMEOUT_MS && {media_hold_timeout: JAMBONES_MEDIA_HOLD_TIMEOUT_MS})
};
if (Object.keys(opts).length > 0) {
this.ep.set(opts);
}
if (this.dialTask?.inbandDtmfEnabled && !this.ep.inbandDtmfEnabled) {
// https://developer.signalwire.com/freeswitch/FreeSWITCH-Explained/Modules/mod-dptools/6587132/#0-about
try {
this.ep.execute('start_dtmf');
this.ep.inbandDtmfEnabled = true;
} catch (err) {
this.logger.info(err, 'place-outdial:_configMsEndpoint - error enable inband DTMF');
}
}
async _createMediaEndpoint(drachtioFsmrfOptions = {}) {
return await createMediaEndpoint(this.srf, this.logger, {
acactiveMs: this.ms,
drachtioFsmrfOptions,
onHoldMusic: this.onHoldMusic,
inbandDtmfEnabled: this.dialTask?.inbandDtmfEnabled,
mediaTimeoutHandler: this._handleMediaTimeout.bind(this),
});
const origDestroy = this.ep.destroy.bind(this.ep);
this.ep.destroy = async() => {
try {
if (this.dialTask.transcribeTask && JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS) {
// transcribe task is being used, wait for some time before destroy
// if final transcription is received but endpoint is already closed,
// freeswitch module will not be able to send the transcription
this.logger.info('SingleDialer:_configMsEndpoint -' +
' Dial with transcribe task, wait for some time before destroy');
await sleepFor(JAMBONES_TRANSCRIBE_EP_DESTROY_DELAY_MS);
}
await origDestroy();
} catch (err) {
this.logger.error(err, 'SingleDialer:_configMsEndpoint - error destroying endpoint');
}
};
}
/**
@@ -408,8 +431,7 @@ class SingleDialer extends Emitter {
accountInfo: this.accountInfo,
tasks,
rootSpan: this.rootSpan,
req: this.req,
tmpFiles: this.tmpFiles,
req: this.req
});
await cs.exec();
@@ -506,7 +528,8 @@ class SingleDialer extends Emitter {
assert(this.dlg && this.dlg.connected && !this.ep);
this.logger.debug('SingleDialer:reAnchorMedia: re-anchoring media after partial media');
this.ep = await this._createMediaEndpoint({remoteSdp: this.dlg.remote.sdp});
this.ep = await this.ms.createEndpoint({remoteSdp: this.dlg.remote.sdp});
this._configMsEndpoint();
await this.dlg.modify(this.ep.local.sdp, {
headers: {
'X-Reason': 'anchor-media'
@@ -543,12 +566,12 @@ class SingleDialer extends Emitter {
function placeOutdial({
logger, srf, ms, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask,
onHoldMusic, tmpFiles
onHoldMusic
}) {
const myOpts = deepcopy(opts);
const sd = new SingleDialer({
logger, sbcAddress, target, opts: myOpts, application, callInfo,
accountInfo, rootSpan, startSpan, dialTask, onHoldMusic, tmpFiles
accountInfo, rootSpan, startSpan, dialTask, onHoldMusic
});
sd.exec(srf, ms, myOpts);
return sd;

View File

@@ -1,91 +0,0 @@
// lib/utils/process-monitor.js
const fs = require('fs');
const path = require('path');
class ProcessMonitor {
constructor(logger) {
this.logger = logger;
this.packageInfo = this.getPackageInfo();
this.processName = this.packageInfo.name || 'unknown-app';
}
getPackageInfo() {
try {
const packagePath = path.join(process.cwd(), 'package.json');
return JSON.parse(fs.readFileSync(packagePath, 'utf8'));
} catch (e) {
return { name: 'unknown', version: 'unknown' };
}
}
logStartup(additionalInfo = {}) {
const startupInfo = {
msg: `${this.processName} started`,
app_name: this.processName,
app_version: this.packageInfo.version,
pid: process.pid,
ppid: process.ppid,
pm2_instance_id: process.env.NODE_APP_INSTANCE || 'not_pm2',
pm2_id: process.env.pm_id,
is_pm2: !!process.env.PM2,
node_version: process.version,
uptime: process.uptime(),
timestamp: new Date().toISOString(),
...additionalInfo
};
this.logger.info(startupInfo);
return startupInfo;
}
setupSignalHandlers() {
// Log when we receive signals that would cause restart
process.on('SIGINT', () => {
this.logger.info({
msg: 'SIGINT received',
app_name: this.processName,
pid: process.pid,
ppid: process.ppid,
uptime: process.uptime(),
timestamp: new Date().toISOString()
});
process.exit(0);
});
process.on('SIGTERM', () => {
this.logger.info({
msg: 'SIGTERM received',
app_name: this.processName,
pid: process.pid,
ppid: process.ppid,
uptime: process.uptime(),
timestamp: new Date().toISOString()
});
process.exit(0);
});
process.on('uncaughtException', (error) => {
this.logger.error({
msg: 'Uncaught exception - process will restart',
app_name: this.processName,
error: error.message,
stack: error.stack,
pid: process.pid,
timestamp: new Date().toISOString()
});
process.exit(1);
});
process.on('unhandledRejection', (reason, promise) => {
this.logger.error({
msg: 'Unhandled rejection',
app_name: this.processName,
reason,
pid: process.pid,
timestamp: new Date().toISOString()
});
});
}
}
module.exports = ProcessMonitor;

View File

@@ -100,39 +100,6 @@ module.exports = (logger) => {
else if (K8S) {
lifecycleEmitter.scaleIn = () => process.exit(0);
}
else {
process.on('SIGUSR1', () => {
logger.info('received SIGUSR1: begin drying up calls for scale-in');
dryUpCalls = true;
const {srf} = require('../..');
const {writeSystemAlerts} = srf.locals;
if (writeSystemAlerts) {
const {SystemState, FEATURE_SERVER} = require('./constants');
writeSystemAlerts({
system_component: FEATURE_SERVER,
state : SystemState.GracefulShutdownInProgress,
fields : {
detail: `feature-server with process_id ${process.pid} shutdown in progress`,
host: srf.locals?.ipv4
}
});
}
pingProxies(srf);
// if we have zero calls, we can complete the scale-in right
setTimeout(() => {
const calls = srf.locals.sessionTracker.count;
if (calls === 0) {
logger.info('scale-in can complete immediately as we have no calls in progress');
process.exit(0);
}
else {
logger.info(`${calls} calls in progress; scale-in will complete when they are done`);
}
}, 5000);
});
}
async function pingProxies(srf) {

View File

@@ -55,28 +55,11 @@ const extractSdpMedia = (sdp) => {
}
};
const getLeadingCodec = (sdp) => {
if (!sdp) {
return null;
}
const parsed = sdpTransform.parse(sdp);
const audio = parsed.media?.find((m) => m.type === 'audio');
if (!audio) {
return null;
}
return audio.rtp?.[0]?.codec || null;
};
module.exports = {
isOnhold,
mergeSdpMedia,
extractSdpMedia,
isOpusFirst,
makeOpusFirst,
removeVideoSdp,
getLeadingCodec
removeVideoSdp
};

View File

@@ -1,196 +0,0 @@
const { assert } = require('console');
const Emitter = require('events');
const {
VadDetection,
SileroVadDetection
} = require('../utils/constants.json');
class SttLatencyCalculator extends Emitter {
constructor({ logger, cs}) {
super();
this.logger = logger;
this.cs = cs;
this.isRunning = false;
this.isInTalkSpurt = false;
this.start_talking_time = 0;
this.talkspurts = [];
this.vendor = this.cs.vad?.vendor || 'silero';
this.stt_start_time = 0;
this.stt_stop_time = 0;
this.stt_on_transcription_time = 0;
}
set sttStartTime(time) {
this.stt_start_time = time;
}
get sttStartTime() {
return this.stt_start_time || 0;
}
set sttStopTime(time) {
this.stt_stop_time = time;
}
get sttStopTime() {
return this.stt_stop_time || 0;
}
set sttOnTranscriptionTime(time) {
this.stt_on_transcription_time = time;
}
get sttOnTranscriptionTime() {
return this.stt_on_transcription_time || 0;
}
_onVadDetected(_ep, _evt, fsEvent) {
if (fsEvent.getHeader('detected-event') === 'stop_talking') {
if (this.isInTalkSpurt) {
this.talkspurts.push({
start: this.start_talking_time,
stop: Date.now()
});
}
this.start_talking_time = 0;
this.isInTalkSpurt = false;
} else if (fsEvent.getHeader('detected-event') === 'start_talking') {
this.start_talking_time = Date.now();
this.isInTalkSpurt = true;
}
}
_startVad() {
assert(!this.isRunning, 'Latency calculator is already running');
assert(this.cs.ep, 'Callsession has no endpoint to start the latency calculator');
const ep = this.cs.ep;
if (!ep.sttLatencyVadHandler) {
ep.sttLatencyVadHandler = this._onVadDetected.bind(this, ep);
if (this.vendor === 'silero') {
ep.addCustomEventListener(SileroVadDetection.Detection, ep.sttLatencyVadHandler);
} else {
ep.addCustomEventListener(VadDetection.Detection, ep.sttLatencyVadHandler);
}
}
this.stop_talking_time = 0;
this.start_talking_time = 0;
this.vad = {
...(this.cs.vad || {}),
strategy: 'continuous',
bugname: 'stt-latency-calculator-vad',
vendor: this.vendor
};
ep.startVadDetection(this.vad);
this.isRunning = true;
}
_stopVad() {
if (this.isRunning) {
this.logger.warn('Latency calculator is still running, stopping VAD detection');
const ep = this.cs.ep;
ep.stopVadDetection(this.vad);
if (ep.sttLatencyVadHandler) {
if (this.vendor === 'silero') {
this.ep?.removeCustomEventListener(SileroVadDetection.Detection, ep.sttLatencyVadHandler);
} else {
this.ep?.removeCustomEventListener(VadDetection.Detection, ep.sttLatencyVadHandler);
}
ep.sttLatencyVadHandler = null;
}
this.isRunning = false;
this.logger.info('STT Latency Calculator stopped');
}
}
start() {
if (this.isRunning) {
this.logger.warn('Latency calculator is already running');
return;
}
if (!this.cs.ep) {
this.logger.error('Callsession has no endpoint to start the latency calculator');
return;
}
this._startVad();
this.logger.debug('STT Latency Calculator started');
}
stop() {
this._stopVad();
}
toUnixTimestamp(date) {
return Math.floor(date / 1000);
}
calculateLatency() {
if (!this.isRunning) {
return null;
}
const stt_stop_time = this.stt_stop_time || Date.now();
if (this.isInTalkSpurt) {
this.talkspurts.push({
start: this.start_talking_time,
stop: stt_stop_time
});
this.isInTalkSpurt = false;
this.start_talking_time = 0;
}
const stt_on_transcription_time = this.stt_on_transcription_time || stt_stop_time;
const start_talking_time = this.talkspurts[0]?.start;
let lastIdx = this.talkspurts.length - 1;
lastIdx = lastIdx < 0 ? 0 : lastIdx;
const stop_talking_time = this.talkspurts[lastIdx]?.stop || stt_stop_time;
return {
stt_start_time: this.toUnixTimestamp(this.stt_start_time),
stt_stop_time: this.toUnixTimestamp(stt_stop_time),
start_talking_time: this.toUnixTimestamp(start_talking_time),
stop_talking_time: this.toUnixTimestamp(stop_talking_time),
stt_latency: parseFloat((Math.abs(stt_on_transcription_time - stop_talking_time)) / 1000).toFixed(2),
stt_latency_ms: Math.abs(stt_on_transcription_time - stop_talking_time),
stt_usage: parseFloat((stt_stop_time - this.stt_start_time) / 1000).toFixed(2),
talkspurts: this.talkspurts.map((ts) =>
([this.toUnixTimestamp(ts.start || 0), this.toUnixTimestamp(ts.stop || 0)]))
};
}
resetTime() {
if (!this.isRunning) {
return;
}
this.stt_start_time = Date.now();
this.stt_stop_time = 0;
this.stt_on_transcription_time = 0;
this.clearTalkspurts();
this.logger.info('STT Latency Calculator reset');
}
onTranscriptionReceived() {
if (!this.isRunning) {
return;
}
this.stt_on_transcription_time = Date.now();
this.logger.debug(`CallSession:on-transcription set to ${this.stt_on_transcription_time}`);
}
onTranscribeStop() {
if (!this.isRunning) {
return;
}
this.stt_stop_time = Date.now();
this.logger.debug(`CallSession:transcribe-stop set to ${this.stt_stop_time}`);
}
clearTalkspurts() {
this.talkspurts = [];
if (!this.isInTalkSpurt) {
this.start_talking_time = 0;
}
}
}
module.exports = SttLatencyCalculator;

View File

@@ -110,10 +110,6 @@ const stickyVars = {
voxist: [
'VOXIST_API_KEY',
],
cartesia: [
'CARTESIA_API_KEY',
'CARTESIA_MODEL_ID'
],
speechmatics: [
'SPEECHMATICS_API_KEY',
'SPEECHMATICS_HOST',
@@ -131,43 +127,6 @@ const stickyVars = {
'OPENAI_TURN_DETECTION_PREFIX_PADDING_MS',
'OPENAI_TURN_DETECTION_SILENCE_DURATION_MS',
],
houndify: [
'HOUNDIFY_CLIENT_ID',
'HOUNDIFY_CLIENT_KEY',
'HOUNDIFY_USER_ID',
'HOUNDIFY_MAX_SILENCE_SECONDS',
'HOUNDIFY_MAX_SILENCE_AFTER_FULL_QUERY_SECONDS',
'HOUNDIFY_MAX_SILENCE_AFTER_PARTIAL_QUERY_SECONDS',
'HOUNDIFY_VAD_SENSITIVITY',
'HOUNDIFY_VAD_TIMEOUT',
'HOUNDIFY_VAD_MODE',
'HOUNDIFY_VAD_VOICE_MS',
'HOUNDIFY_VAD_SILENCE_MS',
'HOUNDIFY_VAD_DEBUG',
'HOUNDIFY_AUDIO_FORMAT',
'HOUNDIFY_ENABLE_NOISE_REDUCTION',
'HOUNDIFY_AUDIO_ENDPOINT',
'HOUNDIFY_ENABLE_PROFANITY_FILTER',
'HOUNDIFY_ENABLE_PUNCTUATION',
'HOUNDIFY_ENABLE_CAPITALIZATION',
'HOUNDIFY_CONFIDENCE_THRESHOLD',
'HOUNDIFY_ENABLE_DISFLUENCY_FILTER',
'HOUNDIFY_MAX_RESULTS',
'HOUNDIFY_ENABLE_WORD_TIMESTAMPS',
'HOUNDIFY_MAX_ALTERNATIVES',
'HOUNDIFY_PARTIAL_TRANSCRIPT_INTERVAL',
'HOUNDIFY_SESSION_TIMEOUT',
'HOUNDIFY_CONNECTION_TIMEOUT',
'HOUNDIFY_LATITUDE',
'HOUNDIFY_LONGITUDE',
'HOUNDIFY_CITY',
'HOUNDIFY_STATE',
'HOUNDIFY_COUNTRY',
'HOUNDIFY_TIMEZONE',
'HOUNDIFY_DOMAIN',
'HOUNDIFY_CUSTOM_VOCABULARY',
'HOUNDIFY_LANGUAGE_MODEL'
],
};
/**
@@ -376,61 +335,6 @@ const normalizeDeepgram = (evt, channel, language, shortUtterance) => {
};
};
const normalizeGladia = (evt, channel, language, shortUtterance) => {
const copy = JSON.parse(JSON.stringify(evt));
// Handle Gladia transcript format
if (evt.type === 'transcript' && evt.data && evt.data.utterance) {
const utterance = evt.data.utterance;
const alternatives = [{
confidence: utterance.confidence || 0,
transcript: utterance.text || '',
}];
return {
language_code: utterance.language || language,
channel_tag: channel,
is_final: evt.data.is_final || false,
alternatives,
vendor: {
name: 'gladia',
evt: copy
}
};
}
};
const normalizeDeepgramFlux = (evt, channel, language) => {
const copy = JSON.parse(JSON.stringify(evt));
let turnTakingEvent;
if (['StartOfTurn', 'EagerEndOfTurn', 'TurnResumed', 'EndOfTurn'].includes(evt.event)) {
turnTakingEvent = evt.event;
}
/* calculate total confidence based on word-level confidence */
const realWords = (evt.words || [])
.filter((w) => ![',.!?;'].includes(w.word));
const confidence = realWords.length > 0 ? realWords.reduce((acc, w) => acc + w.confidence, 0) / realWords.length : 0;
return {
language_code: language,
channel_tag: channel,
is_final: evt.event === 'EndOfTurn',
alternatives: [
{
confidence,
end_of_turn_confidence: evt.end_of_turn_confidence,
transcript: evt.transcript,
...(turnTakingEvent && {turn_taking_event: turnTakingEvent})
}
],
vendor: {
name: 'deepgramflux',
evt: copy
}
};
};
const normalizeNvidia = (evt, channel, language) => {
const copy = JSON.parse(JSON.stringify(evt));
const alternatives = (evt.alternatives || [])
@@ -615,27 +519,16 @@ const normalizeAws = (evt, channel, language) => {
const normalizeAssemblyAi = (evt, channel, language) => {
const copy = JSON.parse(JSON.stringify(evt));
const alternatives = [];
let is_final = false;
if (evt.type && evt.type === 'Turn') {
// v3 is here
alternatives.push({
confidence: evt.end_of_turn_confidence,
transcript: evt.transcript,
});
is_final = evt.end_of_turn;
} else {
alternatives.push({
confidence: evt.confidence,
transcript: evt.text,
});
is_final = evt.message_type === 'FinalTranscript';
}
return {
language_code: language,
channel_tag: channel,
is_final,
alternatives,
is_final: evt.message_type === 'FinalTranscript',
alternatives: [
{
confidence: evt.confidence,
transcript: evt.text,
}
],
vendor: {
name: 'assemblyai',
evt: copy
@@ -643,30 +536,6 @@ const normalizeAssemblyAi = (evt, channel, language) => {
};
};
const normalizeHoundify = (evt, channel, language) => {
const copy = JSON.parse(JSON.stringify(evt));
const alternatives = [];
const is_final = evt.ResultsAreFinal && evt.ResultsAreFinal[0] === true;
if (evt.Disambiguation && evt.Disambiguation.ChoiceData && evt.Disambiguation.ChoiceData.length > 0) {
// Handle Houndify Voice Search Result format
const choiceData = evt.Disambiguation.ChoiceData[0];
alternatives.push({
confidence: choiceData.ConfidenceScore || choiceData.ASRConfidence || 0.0,
transcript: choiceData.FormattedTranscription || choiceData.Transcription || '',
});
}
return {
language_code: language,
channel_tag: channel,
is_final,
alternatives,
vendor: {
name: 'houndify',
evt: copy
}
};
};
const normalizeVoxist = (evt, channel, language) => {
const copy = JSON.parse(JSON.stringify(evt));
return {
@@ -686,25 +555,6 @@ const normalizeVoxist = (evt, channel, language) => {
};
};
const normalizeCartesia = (evt, channel, language) => {
const copy = JSON.parse(JSON.stringify(evt));
return {
language_code: language,
channel_tag: channel,
is_final: evt.is_final,
alternatives: [
{
confidence: 1.00,
transcript: evt.text,
}
],
vendor: {
name: 'cartesia',
evt: copy
}
};
};
const normalizeSpeechmatics = (evt, channel, language) => {
const copy = JSON.parse(JSON.stringify(evt));
const is_final = evt.message === 'AddTranscript';
@@ -766,10 +616,6 @@ module.exports = (logger) => {
switch (vendor) {
case 'deepgram':
return normalizeDeepgram(evt, channel, language, shortUtterance);
case 'gladia':
return normalizeGladia(evt, channel, language, shortUtterance);
case 'deepgramflux':
return normalizeDeepgramFlux(evt, channel, language, shortUtterance);
case 'microsoft':
return normalizeMicrosoft(evt, channel, language, punctuation);
case 'google':
@@ -788,12 +634,8 @@ module.exports = (logger) => {
return normalizeCobalt(evt, channel, language);
case 'assemblyai':
return normalizeAssemblyAi(evt, channel, language, shortUtterance);
case 'houndify':
return normalizeHoundify(evt, channel, language, shortUtterance);
case 'voxist':
return normalizeVoxist(evt, channel, language);
case 'cartesia':
return normalizeCartesia(evt, channel, language);
case 'verbio':
return normalizeVerbio(evt, channel, language);
case 'speechmatics':
@@ -886,15 +728,12 @@ module.exports = (logger) => {
AWS_ACCESS_KEY_ID: sttCredentials.accessKeyId,
AWS_SECRET_ACCESS_KEY: sttCredentials.secretAccessKey,
AWS_REGION: sttCredentials.region,
AWS_SECURITY_TOKEN: sttCredentials.securityToken,
AWS_SESSION_TOKEN: sttCredentials.sessionToken ? sttCredentials.sessionToken : sttCredentials.securityToken
AWS_SECURITY_TOKEN: sttCredentials.securityToken
}),
...(awsOptions.accessKey && {AWS_ACCESS_KEY_ID: awsOptions.accessKey}),
...(awsOptions.secretKey && {AWS_SECRET_ACCESS_KEY: awsOptions.secretKey}),
...(awsOptions.region && {AWS_REGION: awsOptions.region}),
...(awsOptions.securityToken && {AWS_SECURITY_TOKEN: awsOptions.securityToken}),
...(awsOptions.sessionToken && {AWS_SESSION_TOKEN: awsOptions.sessionToken ?
awsOptions.sessionToken : awsOptions.securityToken}),
...(awsOptions.languageModelName && {AWS_LANGUAGE_MODEL_NAME: awsOptions.languageModelName}),
...(awsOptions.piiEntityTypes?.length && {AWS_PII_ENTITY_TYPES: awsOptions.piiEntityTypes.join(',')}),
...(awsOptions.piiIdentifyEntities && {AWS_PII_IDENTIFY_ENTITIES: true}),
@@ -920,7 +759,7 @@ module.exports = (logger) => {
...(rOpts.initialSpeechTimeoutMs > 0 &&
{AZURE_INITIAL_SPEECH_TIMEOUT_MS: rOpts.initialSpeechTimeoutMs}),
...(rOpts.requestSnr && {AZURE_REQUEST_SNR: 1}),
...(azureOptions.audioLogging && {AZURE_AUDIO_LOGGING: 1}),
...(rOpts.audioLogging && {AZURE_AUDIO_LOGGING: 1}),
...{AZURE_USE_OUTPUT_FORMAT_DETAILED: 1},
...(azureOptions.speechSegmentationSilenceTimeoutMs &&
{AZURE_SPEECH_SEGMENTATION_SILENCE_TIMEOUT_MS: azureOptions.speechSegmentationSilenceTimeoutMs}),
@@ -995,14 +834,6 @@ module.exports = (logger) => {
const deepgramUri = deepgramOptions.deepgramSttUri || sttCredentials.deepgram_stt_uri;
const useTls = deepgramOptions.deepgramSttUseTls || sttCredentials.deepgram_stt_use_tls;
// DH (2025-08-11) entity_prompt is currently limited to 100 words
const entityPrompt = deepgramOptions.entityPrompt ?
deepgramOptions.entityPrompt
.split(/\s+/)
.slice(0, 100)
.join(' ')
: undefined;
/* default to a sensible model if not supplied */
if (!model) {
model = selectDefaultDeepgramModel(task, language);
@@ -1061,35 +892,7 @@ module.exports = (logger) => {
...(deepgramOptions.fillerWords) &&
{DEEPGRAM_SPEECH_FILLER_WORDS: deepgramOptions.fillerWords},
...((Array.isArray(deepgramOptions.keyterms) && deepgramOptions.keyterms.length > 0) &&
{DEEPGRAM_SPEECH_KEYTERMS: deepgramOptions.keyterms.join(',')}),
...(deepgramOptions.mipOptOut && {DEEPGRAM_SPEECH_MIP_OPT_OUT: deepgramOptions.mipOptOut}),
...(entityPrompt && {DEEPGRAM_SPEECH_ENTITY_PROMPT: entityPrompt}),
};
}
else if ('deepgramflux' === vendor) {
const {
eotThreshold,
eotTimeoutMs,
mipOptOut,
model,
eagerEotThreshold,
keyterms
} = rOpts.deepgramOptions || {};
opts = {
DEEPGRAMFLUX_API_KEY: sttCredentials.api_key,
DEEPGRAMFLUX_SPEECH_MODEL: model || 'flux-general-en',
...(eotThreshold && {DEEPGRAMFLUX_SPEECH_EOT_THRESHOLD: eotThreshold}),
...(eotTimeoutMs && {DEEPGRAMFLUX_SPEECH_EOT_TIMEOUT_MS: eotTimeoutMs}),
...(mipOptOut && {DEEPGRAMFLUX_SPEECH_MIP_OPT_OUT: mipOptOut}),
...(eagerEotThreshold && {DEEPGRAMFLUX_SPEECH_EAGER_EOT_THRESHOLD: eagerEotThreshold}),
...(keyterms && keyterms.length > 0 && {DEEPGRAMFLUX_SPEECH_KEYTERMS: keyterms.join(',')}),
};
}
else if ('gladia' === vendor) {
const {host, path} = sttCredentials;
opts = {
GLADIA_SPEECH_HOST: host,
GLADIA_SPEECH_PATH: path,
{DEEPGRAM_SPEECH_KEYTERMS: deepgramOptions.keyterms.join(',')})
};
}
else if ('soniox' === vendor) {
@@ -1190,89 +993,14 @@ module.exports = (logger) => {
};
}
else if ('assemblyai' === vendor) {
const serviceVersion = rOpts.assemblyAiOptions?.serviceVersion || sttCredentials.service_version || 'v2';
const {
formatTurns,
endOfTurnConfidenceThreshold,
minEndOfTurnSilenceWhenConfident,
maxTurnSilence
} = rOpts.assemblyAiOptions || {};
opts = {
...opts,
ASSEMBLYAI_API_VERSION: serviceVersion,
...(serviceVersion === 'v3' && {
...(formatTurns && {
ASSEMBLYAI_FORMAT_TURNS: formatTurns
}),
...(endOfTurnConfidenceThreshold && {
ASSEMBLYAI_END_OF_TURN_CONFIDENCE_THRESHOLD: endOfTurnConfidenceThreshold
}),
ASSEMBLYAI_MIN_END_OF_TURN_SILENCE_WHEN_CONFIDENT: minEndOfTurnSilenceWhenConfident || 500,
...(maxTurnSilence && {
ASSEMBLYAI_MAX_TURN_SILENCE: maxTurnSilence
}),
}),
...(sttCredentials.api_key) &&
{ASSEMBLYAI_API_KEY: sttCredentials.api_key},
...(rOpts.hints?.length > 0 &&
{ASSEMBLYAI_WORD_BOOST: JSON.stringify(rOpts.hints)})
};
}
else if ('houndify' === vendor) {
const {
latitude, longitude, city, state, country, timeZone, domain, audioEndpoint,
maxSilenceSeconds, maxSilenceAfterFullQuerySeconds, maxSilenceAfterPartialQuerySeconds,
vadSensitivity, vadTimeout, vadMode, vadVoiceMs, vadSilenceMs, vadDebug,
audioFormat, enableNoiseReduction, enableProfanityFilter, enablePunctuation,
enableCapitalization, confidenceThreshold, enableDisfluencyFilter,
maxResults, enableWordTimestamps, maxAlternatives, partialTranscriptInterval,
sessionTimeout, connectionTimeout, customVocabulary, languageModel,
requestInfo, sampleRate
} = rOpts.houndifyOptions || {};
const audioEndpointUri = audioEndpoint || sttCredentials.houndify_server_uri;
opts = {
...opts,
HOUNDIFY_CLIENT_ID: sttCredentials.client_id,
HOUNDIFY_CLIENT_KEY: sttCredentials.client_key,
HOUNDIFY_USER_ID: sttCredentials.user_id,
HOUNDIFY_MAX_SILENCE_SECONDS: maxSilenceSeconds || 5,
HOUNDIFY_MAX_SILENCE_AFTER_FULL_QUERY_SECONDS: maxSilenceAfterFullQuerySeconds || 1,
HOUNDIFY_MAX_SILENCE_AFTER_PARTIAL_QUERY_SECONDS: maxSilenceAfterPartialQuerySeconds || 1.5,
...(vadSensitivity && {HOUNDIFY_VAD_SENSITIVITY: vadSensitivity}),
...(vadTimeout && {HOUNDIFY_VAD_TIMEOUT: vadTimeout}),
...(vadMode && {HOUNDIFY_VAD_MODE: vadMode}),
...(vadVoiceMs && {HOUNDIFY_VAD_VOICE_MS: vadVoiceMs}),
...(vadSilenceMs && {HOUNDIFY_VAD_SILENCE_MS: vadSilenceMs}),
...(vadDebug && {HOUNDIFY_VAD_DEBUG: vadDebug}),
...(audioFormat && {HOUNDIFY_AUDIO_FORMAT: audioFormat}),
...(enableNoiseReduction && {HOUNDIFY_ENABLE_NOISE_REDUCTION: enableNoiseReduction}),
...(enableProfanityFilter && {HOUNDIFY_ENABLE_PROFANITY_FILTER: enableProfanityFilter}),
...(enablePunctuation && {HOUNDIFY_ENABLE_PUNCTUATION: enablePunctuation}),
...(enableCapitalization && {HOUNDIFY_ENABLE_CAPITALIZATION: enableCapitalization}),
...(confidenceThreshold && {HOUNDIFY_CONFIDENCE_THRESHOLD: confidenceThreshold}),
...(enableDisfluencyFilter && {HOUNDIFY_ENABLE_DISFLUENCY_FILTER: enableDisfluencyFilter}),
...(maxResults && {HOUNDIFY_MAX_RESULTS: maxResults}),
...(enableWordTimestamps && {HOUNDIFY_ENABLE_WORD_TIMESTAMPS: enableWordTimestamps}),
...(maxAlternatives && {HOUNDIFY_MAX_ALTERNATIVES: maxAlternatives}),
...(partialTranscriptInterval && {HOUNDIFY_PARTIAL_TRANSCRIPT_INTERVAL: partialTranscriptInterval}),
...(sessionTimeout && {HOUNDIFY_SESSION_TIMEOUT: sessionTimeout}),
...(connectionTimeout && {HOUNDIFY_CONNECTION_TIMEOUT: connectionTimeout}),
...(latitude && {HOUNDIFY_LATITUDE: latitude}),
...(longitude && {HOUNDIFY_LONGITUDE: longitude}),
...(city && {HOUNDIFY_CITY: city}),
...(state && {HOUNDIFY_STATE: state}),
...(country && {HOUNDIFY_COUNTRY: country}),
...(timeZone && {HOUNDIFY_TIMEZONE: timeZone}),
...(domain && {HOUNDIFY_DOMAIN: domain}),
...(audioEndpointUri && {HOUNDIFY_AUDIO_ENDPOINT: audioEndpointUri}),
...(customVocabulary && {HOUNDIFY_CUSTOM_VOCABULARY:
Array.isArray(customVocabulary) ? customVocabulary.join(',') : customVocabulary}),
...(languageModel && {HOUNDIFY_LANGUAGE_MODEL: languageModel}),
...(requestInfo && {HOUNDIFY_REQUEST_INFO: JSON.stringify(requestInfo)}),
...(sampleRate && {HOUNDIFY_SAMPLING_RATE: sampleRate}),
};
}
else if ('voxist' === vendor) {
opts = {
...opts,
@@ -1280,16 +1008,6 @@ module.exports = (logger) => {
{VOXIST_API_KEY: sttCredentials.api_key},
};
}
else if ('cartesia' === vendor) {
opts = {
...opts,
...(sttCredentials.api_key &&
{CARTESIA_API_KEY: sttCredentials.api_key}),
...(sttCredentials.stt_model_id && {
CARTESIA_MODEL_ID: sttCredentials.stt_model_id
})
};
}
else if ('openai' === vendor) {
const {openaiOptions = {}} = rOpts;
const model = openaiOptions.model || rOpts.model || sttCredentials.model_id || 'whisper-1';

View File

@@ -80,7 +80,7 @@ class TtsStreamingBuffer extends Emitter {
clearTimeout(this.timer);
this.removeCustomEventListeners();
if (this.ep) {
this._api(this.ep, [this.ep.uuid, 'stop'])
this._api(this.ep, [this.ep.uuid, 'close'])
.catch((err) =>
this.logger.info({ err }, 'TtsStreamingBuffer:stop Error closing TTS streaming')
);
@@ -163,6 +163,7 @@ class TtsStreamingBuffer extends Emitter {
}
clear() {
this.logger.debug('TtsStreamingBuffer:clear');
if (this._connectionStatus !== TtsStreamingConnectionStatus.Connected) return;
clearTimeout(this.timer);
this._api(this.ep, [this.ep.uuid, 'clear']).catch((err) =>
@@ -192,7 +193,10 @@ class TtsStreamingBuffer extends Emitter {
this.logger.debug('TtsStreamingBuffer:_feedQueue TTS stream is not open or no endpoint available');
return;
}
if (this._connectionStatus !== TtsStreamingConnectionStatus.Connected) {
if (
this._connectionStatus === TtsStreamingConnectionStatus.NotConnected ||
this._connectionStatus === TtsStreamingConnectionStatus.Failed
) {
this.logger.debug('TtsStreamingBuffer:_feedQueue TTS stream is not connected');
return;
}
@@ -220,8 +224,7 @@ class TtsStreamingBuffer extends Emitter {
this.queue.shift();
}
// Immediately send all accumulated text (ignoring sentence boundaries).
// Skip sending if flushText is only whitespace.
if (flushText.length > 0 && !isWhitespace(flushText)) {
if (flushText.length > 0) {
const modifiedFlushText = flushText.replace(/\n\n/g, '\n \n');
try {
await this._api(this.ep, [this.ep.uuid, 'send', modifiedFlushText]);
@@ -275,14 +278,6 @@ class TtsStreamingBuffer extends Emitter {
}
const chunk = combinedText.slice(0, chunkEnd);
// Check if the chunk is only whitespace before processing the queue
// If so, wait for more meaningful text
if (isWhitespace(chunk)) {
this.logger.debug('TtsStreamingBuffer:_feedQueue chunk is only whitespace, waiting for more text');
this._setTimerIfNeeded();
return;
}
// Now we iterate over the queue items
// and deduct their lengths until we've accounted for chunkEnd characters.
let remaining = chunkEnd;
@@ -306,14 +301,6 @@ class TtsStreamingBuffer extends Emitter {
this.bufferedLength -= chunkEnd;
const modifiedChunk = chunk.replace(/\n\n/g, '\n \n');
if (isWhitespace(modifiedChunk)) {
this.logger.debug('TtsStreamingBuffer:_feedQueue modified chunk is only whitespace, restoring queue');
this.queue.unshift({ type: 'text', value: chunk });
this.bufferedLength += chunkEnd;
this._setTimerIfNeeded();
return;
}
this.logger.debug(`TtsStreamingBuffer:_feedQueue sending chunk to tts: ${modifiedChunk}`);
try {
@@ -362,7 +349,6 @@ class TtsStreamingBuffer extends Emitter {
if (this.queue.length > 0) {
await this._feedQueue();
}
this.emit(TtsStreamingEvents.Connected, { vendor });
}
_onConnectFailure(vendor) {
@@ -413,7 +399,6 @@ class TtsStreamingBuffer extends Emitter {
removeCustomEventListeners() {
this.eventHandlers.forEach((h) => h.ep.removeCustomEventListener(h.event, h.handler));
this.eventHandlers.length = 0;
}
_initHandlers(ep) {
@@ -437,15 +422,7 @@ class TtsStreamingBuffer extends Emitter {
const findSentenceBoundary = (text, limit) => {
// Look for punctuation or double newline that signals sentence end.
// Includes:
// - ASCII: . ! ?
// - Arabic: ؟ (question mark), ۔ (full stop)
// - Japanese: 。 (full stop), , (full-width exclamation/question)
//
// For languages that use spaces between sentences, we still require
// whitespace or end-of-string after the mark. For Japanese (no spaces),
// we treat the punctuation itself as a boundary regardless of following char.
const sentenceEndRegex = /[.!?؟۔](?=\s|$)|[。!?]|\n\n/g;
const sentenceEndRegex = /[.!?](?=\s|$)|\n\n/g;
let lastSentenceBoundary = -1;
let match;
while ((match = sentenceEndRegex.exec(text)) && match.index < limit) {

View File

@@ -293,7 +293,7 @@ class WsRequestor extends BaseRequestor {
/* send the message */
this.ws.send(JSON.stringify(obj), async() => {
if (obj.type !== 'llm:event') this.logger.debug({obj}, `WsRequestor:request websocket: sent (${url})`);
this.logger.debug({obj}, `WsRequestor:request websocket: sent (${url})`);
// If session:reconnect is waiting for ack, hold here until ack to send queuedMsgs
if (this._reconnectPromise) {
try {

5847
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "jambonz-feature-server",
"version": "0.9.5",
"version": "0.9.4",
"main": "app.js",
"engines": {
"node": ">= 18.x"
@@ -27,14 +27,14 @@
"dependencies": {
"@aws-sdk/client-auto-scaling": "^3.549.0",
"@aws-sdk/client-sns": "^3.549.0",
"@jambonz/db-helpers": "^0.9.18",
"@jambonz/db-helpers": "^0.9.12",
"@jambonz/http-health-check": "^0.0.1",
"@jambonz/mw-registrar": "^0.2.7",
"@jambonz/realtimedb-helpers": "^0.8.15",
"@jambonz/speech-utils": "^0.2.26",
"@jambonz/realtimedb-helpers": "^0.8.13",
"@jambonz/speech-utils": "^0.2.12",
"@jambonz/stats-collector": "^0.1.10",
"@jambonz/time-series": "^0.2.15",
"@jambonz/verb-specifications": "^0.0.122",
"@jambonz/time-series": "^0.2.13",
"@jambonz/verb-specifications": "^0.0.105",
"@modelcontextprotocol/sdk": "^1.9.0",
"@opentelemetry/api": "^1.8.0",
"@opentelemetry/exporter-jaeger": "^1.23.0",
@@ -48,13 +48,13 @@
"bent": "^7.3.12",
"debug": "^4.3.4",
"deepcopy": "^2.1.0",
"drachtio-fsmrf": "^4.1.2",
"drachtio-srf": "^5.0.14",
"drachtio-fsmrf": "^4.0.3",
"drachtio-srf": "^5.0.5",
"express": "^4.19.2",
"express-validator": "^7.0.1",
"moment": "^2.30.1",
"parse-url": "^9.2.0",
"pino": "^10.1.0",
"pino": "^8.20.0",
"polly-ssml-split": "^0.1.0",
"sdp-transform": "^2.15.0",
"short-uuid": "^5.1.0",

View File

@@ -4,7 +4,6 @@ require('./ws-requestor-unit-test');
require('./http-requestor-retry-test');
require('./http-requestor-unit-test');
require('./unit-tests');
require('./tts-streaming-buffer-test');
require('./docker_start');
require('./create-test-db');
require('./account-validation-tests');
@@ -17,7 +16,6 @@ require('./sip-request-tests');
require('./create-call-test');
require('./play-tests');
require('./sip-refer-tests');
require('./sip-refer-handler-tests');
require('./listen-tests');
require('./config-test');
require('./queue-test');

View File

@@ -1,117 +0,0 @@
<?xml version="1.0" encoding="ISO-8859-1" ?>
<!DOCTYPE scenario SYSTEM "sipp.dtd">
<scenario name="UAS that accepts call and sends REFER">
<!-- Receive incoming INVITE -->
<recv request="INVITE" crlf="true">
<action>
<ereg regexp=".*" search_in="hdr" header="Subject:" assign_to="1" />
<ereg regexp=".*" search_in="hdr" header="From:" assign_to="2" />
</action>
</recv>
<!-- Send 180 Ringing -->
<send>
<![CDATA[
SIP/2.0 180 Ringing
[last_Via:]
[last_From:]
[last_To:];tag=[pid]SIPpTag01[call_number]
[last_Call-ID:]
[last_CSeq:]
[last_Record-Route:]
Subject:[$1]
Content-Length: 0
]]>
</send>
<!-- Send 200 OK with SDP -->
<send>
<![CDATA[
SIP/2.0 200 OK
[last_Via:]
[last_From:]
[last_To:];tag=[pid]SIPpTag01[call_number]
[last_Call-ID:]
[last_CSeq:]
[last_Record-Route:]
Subject:[$1]
Contact: <sip:[local_ip]:[local_port];transport=[transport]>
Content-Type: application/sdp
Content-Length: [len]
v=0
o=user1 53655765 2353687637 IN IP[local_ip_type] [local_ip]
s=-
c=IN IP[media_ip_type] [media_ip]
t=0 0
m=audio [media_port] RTP/AVP 0
a=rtpmap:0 PCMU/8000
]]>
</send>
<recv request="ACK" rtd="true" crlf="true">
<action>
<!-- Check if this is NOT the first call (tag ends with 012 or higher) -->
<ereg regexp="tag=1SIPpTag01[2-9]" search_in="hdr" header="To:" assign_to="3" />
<log message="Not first call check result: [$3]"/>
</action>
</recv>
<!-- Skip REFER if we found a non-first call tag -->
<nop next="skip_refer" test="3" value="" compare="not_equal">
<action>
<log message="Found non-first call tag [$3], skipping REFER"/>
</action>
</nop>
<!-- Wait a moment, then send REFER (only on first call) -->
<pause milliseconds="1000"/>
<nop>
<action>
<log message="Sending REFER for first call"/>
</action>
</nop>
<!-- Send REFER (only on first iteration) -->
<send retrans="500">
<![CDATA[
REFER sip:service@[remote_ip]:[remote_port] SIP/2.0
Via: SIP/2.0/[transport] [local_ip]:[local_port];branch=[branch]
From: <sip:[local_ip]:[local_port]>;tag=[pid]SIPpTag01[call_number]
To: [$2]
[last_Call-ID:]
CSeq: 2 REFER
Contact: <sip:[local_ip]:[local_port];transport=[transport]>
Max-Forwards: 70
X-Call-Number: [call_number]
Refer-To: <sip:+15551234567@example.com>
Referred-By: <sip:[local_ip]:[local_port]>
Content-Length: 0
]]>
</send>
<!-- Expect 202 Accepted (only on first iteration) -->
<recv response="202"/>
<label id="skip_refer"/>
<!-- Wait for BYE from feature server -->
<recv request="BYE"/>
<!-- Send 200 OK to BYE -->
<send>
<![CDATA[
SIP/2.0 200 OK
[last_Via:]
[last_From:]
[last_To:]
[last_Call-ID:]
[last_CSeq:]
Contact: <sip:[local_ip]:[local_port];transport=[transport]>
Content-Length: 0
]]>
</send>
</scenario>

View File

@@ -1,90 +0,0 @@
const test = require('tape');
const { sippUac } = require('./sipp')('test_fs');
const bent = require('bent');
const getJSON = bent('json')
const clearModule = require('clear-module');
const {provisionCallHook} = require('./utils');
const { sleepFor } = require('../lib/utils/helpers');
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});
function connect(connectable) {
return new Promise((resolve, reject) => {
connectable.on('connect', () => {
return resolve();
});
});
}
test('when parent leg recvs REFER it should end the dial after adulting child leg', async(t) => {
clearModule.all();
const {srf, disconnect} = require('../app');
try {
await connect(srf);
// wait for fs connected to drachtio server.
await sleepFor(1000);
// GIVEN
const from = "dial_refer_handler";
let verbs = [
{
"verb": "dial",
"callerId": from,
"actionHook": "/actionHook",
"referHook": "/referHook",
"anchorMedia": true,
"target": [
{
"type": "phone",
"number": "15083084809"
}
]
}
];
await provisionCallHook(from, verbs);
// THEN
//const p = sippUac('uas-dial.xml', '172.38.0.10', undefined, undefined, 2);
const p = sippUac('uas-dial-refer.xml', '172.38.0.10', undefined, undefined, 2);
await sleepFor(1000);
let account_sid = '622f62e4-303a-49f2-bbe0-eb1e1714e37a';
let post = bent('http://127.0.0.1:3000/', 'POST', 'json', 201);
post('v1/createCall', {
'account_sid':account_sid,
"call_hook": {
"url": "http://127.0.0.1:3100/",
"method": "POST",
},
"from": from,
"to": {
"type": "phone",
"number": "15583084808"
}});
await p;
// Verify that the referHook was called
const obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_referHook`);
t.ok(obj.body.from === from,
'dial-refer-handler: referHook was called with correct from');
t.ok(obj.body.refer_details && obj.body.refer_details.sip_refer_to,
'dial-refer-handler: refer_details included in referHook');
t.ok(obj.body.refer_details.refer_to_user === '+15551234567',
'dial-refer-handler: refer_to_user correctly parsed');
t.ok(obj.body.refer_details.referring_call_sid,
'dial-refer-handler: referring_call_sid included');
t.ok(obj.body.refer_details.referred_call_sid,
'dial-refer-handler: referred_call_sid included');
disconnect();
} catch (err) {
console.log(`error received: ${err}`);
disconnect();
t.error(err);
}
});

View File

@@ -58,46 +58,6 @@ test('\'refer\' tests w/202 and NOTIFY', {timeout: 25000}, async(t) => {
}
});
test('\'refer\' tests tel:', {timeout: 25000}, async(t) => {
clearModule.all();
const {srf, disconnect} = require('../app');
try {
await connect(srf);
// GIVEN
const verbs = [
{
verb: 'say',
text: 'silence_stream://100'
},
{
verb: 'sip:refer',
referTo: 'tel:+1234567890',
actionHook: '/actionHook'
}
];
const noVerbs = [];
const from = 'refer_with_tel';
await provisionCallHook(from, verbs);
await provisionActionHook(from, noVerbs)
// THEN
await sippUac('uac-refer-with-notify.xml', '172.38.0.10', from);
t.pass('refer: successfully received 202 Accepted');
await sleepFor(1000);
const obj = await getJSON(`http:127.0.0.1:3100/lastRequest/${from}_actionHook`);
t.ok(obj.body.final_referred_call_status === 200, 'refer: successfully received NOTIFY with 200 OK');
// console.log(`obj: ${JSON.stringify(obj)}`);
disconnect();
} catch (err) {
console.log(`error received: ${err}`);
disconnect();
t.error(err);
}
});
test('\'refer\' tests w/202 but no NOTIFY', {timeout: 25000}, async(t) => {
clearModule.all();
const {srf, disconnect} = require('../app');

View File

@@ -1,177 +0,0 @@
const test = require('tape');
const sinon = require('sinon');
const noop = () => {};
const logger = {
error: noop,
info: noop,
debug: noop
};
const {
TtsStreamingConnectionStatus
} = require('../lib/utils/constants.json');
const TtsStreamingBuffer = require('../lib/utils/tts-streaming-buffer');
// Helper to create a mock CallSession
function createMockCs(options = {}) {
const mockEp = {
uuid: 'test-uuid-1234',
api: sinon.stub().resolves({ body: '+OK' }),
addCustomEventListener: sinon.stub(),
removeCustomEventListener: sinon.stub()
};
return {
logger,
ep: mockEp,
isTtsStreamOpen: options.isTtsStreamOpen !== undefined ? options.isTtsStreamOpen : true,
getTsStreamingVendor: () => options.vendor || 'deepgram'
};
}
/**
* BUG REPRODUCTION TEST
*
* This test reproduces the exact issue from production logs:
* {
* "args": ["uuid", "send", " "],
* "msg": "Error calling uuid_deepgram_tts_streaming: -USAGE: <uuid> connect|send|clear|close [tokens]"
* }
*
* Root cause: When multiple flushes are queued while connecting, and a space token
* gets buffered between flushes, Phase 1 of _feedQueue sends that space to the TTS vendor.
*
* Sequence:
* 1. bufferTokens('Hello.') while connecting
* 2. flush() while connecting
* 3. bufferTokens(' ') while connecting (passes because bufferedLength > 0)
* 4. flush() while connecting
* 5. Connection completes, _feedQueue processes: [text:Hello., flush, text:" ", flush]
* 6. First flush sends "Hello." - OK
* 7. Second flush sends " " - BUG!
*/
test('TtsStreamingBuffer: multiple flushes while connecting - space token sent to TTS vendor', async(t) => {
const cs = createMockCs();
const buffer = new TtsStreamingBuffer(cs);
buffer._connectionStatus = TtsStreamingConnectionStatus.Connecting;
buffer.vendor = 'deepgram';
const apiCalls = [];
const originalApi = buffer._api.bind(buffer);
buffer._api = async function(ep, args) {
apiCalls.push({ args: [...args] });
return originalApi(ep, args);
};
// First batch while connecting
await buffer.bufferTokens('Hello.');
buffer.flush();
// Second batch - just a space (passes because bufferedLength > 0)
await buffer.bufferTokens(' ');
buffer.flush();
// Verify queue state before connect
t.equal(buffer.queue.length, 4, 'queue should have 4 items: [text, flush, text, flush]');
t.equal(buffer.queue[0].type, 'text', 'first item should be text');
t.equal(buffer.queue[0].value, 'Hello.', 'first text should be "Hello."');
t.equal(buffer.queue[1].type, 'flush', 'second item should be flush');
t.equal(buffer.queue[2].type, 'text', 'third item should be text');
t.equal(buffer.queue[2].value, ' ', 'third item should be space');
t.equal(buffer.queue[3].type, 'flush', 'fourth item should be flush');
// Connect - triggers _feedQueue
buffer._connectionStatus = TtsStreamingConnectionStatus.Connected;
await buffer._feedQueue();
// Check API calls
const sendCalls = apiCalls.filter(call => call.args[1] === 'send');
// This assertion will FAIL until the bug is fixed
const whitespaceOnlySends = sendCalls.filter(call => /^\s*$/.test(call.args[2]));
t.equal(whitespaceOnlySends.length, 0,
`should not send whitespace-only tokens, but sent: ${whitespaceOnlySends.map(c => JSON.stringify(c.args[2])).join(', ')}`);
t.end();
});
/**
* Additional test: Verify text with trailing space in same flush is OK
*/
test('TtsStreamingBuffer: text with trailing space in same flush should work', async(t) => {
const cs = createMockCs();
const buffer = new TtsStreamingBuffer(cs);
buffer._connectionStatus = TtsStreamingConnectionStatus.Connecting;
buffer.vendor = 'deepgram';
const apiCalls = [];
const originalApi = buffer._api.bind(buffer);
buffer._api = async function(ep, args) {
apiCalls.push({ args: [...args] });
return originalApi(ep, args);
};
// Buffer text with trailing space, then flush
await buffer.bufferTokens('Hello.');
await buffer.bufferTokens(' ');
buffer.flush();
// Connect
buffer._connectionStatus = TtsStreamingConnectionStatus.Connected;
await buffer._feedQueue();
const sendCalls = apiCalls.filter(call => call.args[1] === 'send');
t.equal(sendCalls.length, 1, 'should have one send call');
t.equal(sendCalls[0].args[2], 'Hello. ', 'should send "Hello. " (text with trailing space)');
t.end();
});
/**
* Test: Leading whitespace should be discarded when buffer is empty
*/
test('TtsStreamingBuffer: leading whitespace discarded when buffer empty', async(t) => {
const cs = createMockCs();
const buffer = new TtsStreamingBuffer(cs);
buffer._connectionStatus = TtsStreamingConnectionStatus.Connected;
buffer.vendor = 'deepgram';
// Try to buffer whitespace when buffer is empty
const result = await buffer.bufferTokens(' ');
t.equal(result.status, 'ok', 'should return ok status');
t.equal(buffer.bufferedLength, 0, 'buffer should remain empty');
t.equal(buffer.queue.length, 0, 'queue should remain empty');
t.end();
});
/**
* Test: Whitespace can be buffered when buffer has content
*/
test('TtsStreamingBuffer: whitespace accepted when buffer has content', async(t) => {
const cs = createMockCs();
const buffer = new TtsStreamingBuffer(cs);
buffer._connectionStatus = TtsStreamingConnectionStatus.Connecting;
buffer.vendor = 'deepgram';
// Buffer real text first
await buffer.bufferTokens('Hello');
// Now buffer whitespace (should pass because bufferedLength > 0)
const result = await buffer.bufferTokens(' ');
t.equal(result.status, 'ok', 'should return ok status');
t.equal(buffer.bufferedLength, 6, 'buffer should have 6 chars');
t.equal(buffer.queue.length, 2, 'queue should have 2 items');
t.end();
});

View File

@@ -99,24 +99,6 @@ app.post('/actionHook', (req, res) => {
return res.sendStatus(200);
});
/*
* referHook
*/
app.post('/referHook', (req, res) => {
console.log({payload: req.body}, 'POST /referHook');
let key = req.body.from + "_referHook"
addRequestToMap(key, req, hook_mapping);
return res.json([{"verb": "pause", "length": 2}]);
});
/*
* adultingHook
*/
app.post('/adulting', (req, res) => {
console.log({payload: req.body}, 'POST /adulting');
return res.sendStatus(200);
});
/*
* customHook
* For the hook to return

View File

@@ -83,8 +83,7 @@ test('invalid jambonz json create alert tests', async(t) => {
{account_sid: 'bb845d4b-83a9-4cde-a6e9-50f3743bab3f', page: 1, page_size: 25, days: 7});
let checked = false;
for (let i = 0; i < data.total; i++) {
checked = data.data[i].message === 'malformed jambonz payload: must be array';
if (checked) break;
checked = data.data[i].message === 'malformed jambonz payload: must be array'
}
t.ok(checked, 'alert is raised as expected');
disconnect();