mirror of
https://github.com/jambonz/jambonz-feature-server.git
synced 2026-02-11 00:39:56 +00:00
Compare commits
1 Commits
v0.9.2-4
...
snyk-fix-b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3902548b59 |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -42,5 +42,4 @@ ecosystem.config.js
|
||||
test/credentials/*.json
|
||||
run-tests.sh
|
||||
run-coverage.sh
|
||||
.vscode
|
||||
.env
|
||||
.vscode
|
||||
7
app.js
7
app.js
@@ -100,13 +100,8 @@ createHttpListener(logger, srf)
|
||||
});
|
||||
|
||||
|
||||
setInterval(async() => {
|
||||
setInterval(() => {
|
||||
srf.locals.stats.gauge('fs.sip.calls.count', sessionTracker.count);
|
||||
// Checking system log level
|
||||
const systemInformation = await srf.locals.dbHelpers.lookupSystemInformation();
|
||||
if (systemInformation && systemInformation.log_level) {
|
||||
logger.level = systemInformation.log_level;
|
||||
}
|
||||
}, 20000);
|
||||
|
||||
const disconnect = () => {
|
||||
|
||||
@@ -108,8 +108,6 @@ const DEEPGRAM_API_KEY = process.env.DEEPGRAM_API_KEY;
|
||||
const ANCHOR_MEDIA_ALWAYS = process.env.ANCHOR_MEDIA_ALWAYS;
|
||||
const VMD_HINTS_FILE = process.env.VMD_HINTS_FILE;
|
||||
|
||||
const JAMBONES_AWS_TRANSCRIBE_USE_GRPC = process.env.JAMBONES_AWS_TRANSCRIBE_USE_GRPC;
|
||||
|
||||
/* security, secrets */
|
||||
const LEGACY_CRYPTO = !!process.env.LEGACY_CRYPTO;
|
||||
const JWT_SECRET = process.env.JWT_SECRET;
|
||||
@@ -192,7 +190,6 @@ module.exports = {
|
||||
ANCHOR_MEDIA_ALWAYS,
|
||||
VMD_HINTS_FILE,
|
||||
JAMBONES_FREESWITCH_MAX_CALL_DURATION_MINS,
|
||||
JAMBONES_AWS_TRANSCRIBE_USE_GRPC,
|
||||
|
||||
LEGACY_CRYPTO,
|
||||
JWT_SECRET,
|
||||
|
||||
@@ -258,8 +258,6 @@ router.post('/',
|
||||
callId: inviteReq.get('Call-ID'),
|
||||
accountSid,
|
||||
traceId: rootSpan.traceId
|
||||
}, {
|
||||
...(account.enable_debug_log && {level: 'debug'})
|
||||
});
|
||||
app.requestor.logger = app.notifier.logger = sipLogger;
|
||||
const callInfo = new CallInfo({
|
||||
@@ -293,8 +291,6 @@ router.post('/',
|
||||
},
|
||||
cbProvisional: (prov) => {
|
||||
const callStatus = prov.body ? CallStatus.EarlyMedia : CallStatus.Ringing;
|
||||
// Update call-id for sbc outbound INVITE
|
||||
cs.callInfo.sbcCallid = prov.get('X-CID');
|
||||
if ([180, 183].includes(prov.status) && prov.body) connectStream(prov.body);
|
||||
restDial.emit('callStatus', prov.status, !!prov.body);
|
||||
cs.emit('callStatusChange', {callStatus, sipStatus: prov.status});
|
||||
|
||||
@@ -187,20 +187,14 @@ module.exports = function(srf, logger) {
|
||||
|
||||
const {span} = rootSpan.startChildSpan('lookupAccountDetails');
|
||||
try {
|
||||
const accountDetail = await lookupAccountDetails(account_sid);
|
||||
const account = accountDetail?.account;
|
||||
req.locals.accountInfo = accountDetail;
|
||||
req.locals.service_provider_sid = account?.service_provider_sid;
|
||||
req.locals.accountInfo = await lookupAccountDetails(account_sid);
|
||||
req.locals.service_provider_sid = req.locals.accountInfo?.account?.service_provider_sid;
|
||||
span.end();
|
||||
if (!account?.is_active) {
|
||||
if (!req.locals.accountInfo.account.is_active) {
|
||||
logger.info(`Account is inactive or suspended ${account_sid}`);
|
||||
// TODO: alert
|
||||
return res.send(503, {headers: {'X-Reason': 'Account exists but is inactive'}});
|
||||
}
|
||||
// Change the default log level to debug
|
||||
if (account?.enable_debug_log) {
|
||||
req.locals.logger.level = 'debug';
|
||||
}
|
||||
logger.debug({accountInfo: req.locals?.accountInfo?.account}, `retrieved account info for ${account_sid}`);
|
||||
next();
|
||||
} catch (err) {
|
||||
@@ -385,7 +379,7 @@ module.exports = function(srf, logger) {
|
||||
const {rootSpan, siprec, application:app} = req.locals;
|
||||
let span;
|
||||
try {
|
||||
if (app.tasks && app.tasks?.length > 0 && !JAMBONES_MYSQL_REFRESH_TTL) {
|
||||
if (app.tasks && !JAMBONES_MYSQL_REFRESH_TTL) {
|
||||
app.tasks = normalizeJambones(logger, app.tasks).map((tdata) => makeTask(logger, tdata));
|
||||
if (0 === app.tasks.length) throw new Error('no application provided');
|
||||
return next();
|
||||
|
||||
@@ -32,7 +32,6 @@ class CallInfo {
|
||||
this.sipStatus = 100;
|
||||
this.sipReason = 'Trying';
|
||||
this.callStatus = CallStatus.Trying;
|
||||
this.sbcCallid = req.get('X-CID');
|
||||
this.originatingSipIp = req.get('X-Forwarded-For');
|
||||
this.originatingSipTrunkName = req.get('X-Originating-Carrier');
|
||||
const {siprec} = req.locals;
|
||||
@@ -130,7 +129,6 @@ class CallInfo {
|
||||
from: this.from,
|
||||
to: this.to,
|
||||
callId: this.callId,
|
||||
sbcCallid: this.sbcCallid,
|
||||
sipStatus: this.sipStatus,
|
||||
sipReason: this.sipReason,
|
||||
callStatus: this.callStatus,
|
||||
|
||||
@@ -20,7 +20,6 @@ const listTaskNames = require('../utils/summarize-tasks');
|
||||
const HttpRequestor = require('../utils/http-requestor');
|
||||
const WsRequestor = require('../utils/ws-requestor');
|
||||
const ActionHookDelayProcessor = require('../utils/action-hook-delay');
|
||||
const {parseUri} = require('drachtio-srf');
|
||||
const {
|
||||
JAMBONES_INJECT_CONTENT,
|
||||
JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
|
||||
@@ -340,18 +339,6 @@ class CallSession extends Emitter {
|
||||
this.application.fallback_speech_recognizer_language = language;
|
||||
}
|
||||
|
||||
/**
|
||||
* global referHook
|
||||
*/
|
||||
|
||||
set referHook(hook) {
|
||||
this._referHook = hook;
|
||||
}
|
||||
|
||||
get referHook() {
|
||||
return this._referHook;
|
||||
}
|
||||
|
||||
/**
|
||||
* Vad
|
||||
*/
|
||||
@@ -533,6 +520,15 @@ class CallSession extends Emitter {
|
||||
this._actionHookDelayRetries = e;
|
||||
}
|
||||
|
||||
// Getter/setter for current tts vendor
|
||||
get currentTtsVendor() {
|
||||
return this._currentTtsVendor;
|
||||
}
|
||||
|
||||
set currentTtsVendor(vendor) {
|
||||
this._currentTtsVendor = vendor;
|
||||
}
|
||||
|
||||
get actionHookDelayProcessor() {
|
||||
return this._actionHookDelayProcessor;
|
||||
}
|
||||
@@ -557,17 +553,6 @@ class CallSession extends Emitter {
|
||||
this.wakeupResolver = null;
|
||||
}
|
||||
});
|
||||
this._actionHookDelayProcessor.on('giveupWithTasks', (tasks) => {
|
||||
this.logger.info('CallSession: ActionHookDelayProcessor: giveupWithTasks event');
|
||||
const giveUpTasks = normalizeJambones(this.logger, tasks).map((tdata) => makeTask(this.logger, tdata));
|
||||
this.logger.info({tasks: listTaskNames(giveUpTasks)}, 'CallSession:giveupWithTasks task list');
|
||||
|
||||
// we need to clear the ahd, as we do not want to execute actionHookDelay actions again
|
||||
this.clearActionHookDelayProcessor();
|
||||
|
||||
// replace the application with giveUpTasks
|
||||
this.replaceApplication(giveUpTasks);
|
||||
});
|
||||
} catch (err) {
|
||||
this.logger.error({err}, 'CallSession: Error creating ActionHookDelayProcessor');
|
||||
}
|
||||
@@ -963,56 +948,42 @@ class CallSession extends Emitter {
|
||||
speech_credential_sid: credential.speech_credential_sid,
|
||||
cobalt_server_uri: credential.cobalt_server_uri
|
||||
};
|
||||
}
|
||||
else if ('elevenlabs' === vendor) {
|
||||
} else if ('elevenlabs' === vendor) {
|
||||
return {
|
||||
api_key: credential.api_key,
|
||||
model_id: credential.model_id,
|
||||
options: credential.options
|
||||
};
|
||||
}
|
||||
else if ('playht' === vendor) {
|
||||
} else if ('playht' === vendor) {
|
||||
return {
|
||||
api_key: credential.api_key,
|
||||
user_id: credential.user_id,
|
||||
voice_engine: credential.voice_engine,
|
||||
options: credential.options
|
||||
};
|
||||
}
|
||||
else if ('rimelabs' === vendor) {
|
||||
} else if ('rimelabs' === vendor) {
|
||||
return {
|
||||
api_key: credential.api_key,
|
||||
model_id: credential.model_id,
|
||||
options: credential.options
|
||||
};
|
||||
}
|
||||
else if ('assemblyai' === vendor) {
|
||||
} else if ('assemblyai' === vendor) {
|
||||
return {
|
||||
speech_credential_sid: credential.speech_credential_sid,
|
||||
api_key: credential.api_key
|
||||
};
|
||||
}
|
||||
else if ('whisper' === vendor) {
|
||||
} else if ('whisper' === vendor) {
|
||||
return {
|
||||
api_key: credential.api_key,
|
||||
model_id: credential.model_id
|
||||
};
|
||||
}
|
||||
else if ('verbio' === vendor) {
|
||||
} else if ('verbio' === vendor) {
|
||||
return {
|
||||
client_id: credential.client_id,
|
||||
client_secret: credential.client_secret,
|
||||
engine_version: credential.engine_version
|
||||
};
|
||||
}
|
||||
else if ('speechmatics' === vendor) {
|
||||
this.logger.info({credential}, 'CallSession:getSpeechCredentials - speechmatics credential');
|
||||
return {
|
||||
api_key: credential.api_key,
|
||||
speechmatics_stt_uri: credential.speechmatics_stt_uri,
|
||||
};
|
||||
}
|
||||
else if (vendor.startsWith('custom:')) {
|
||||
} else if (vendor.startsWith('custom:')) {
|
||||
return {
|
||||
speech_credential_sid: credential.speech_credential_sid,
|
||||
auth_token: credential.auth_token,
|
||||
@@ -1172,11 +1143,6 @@ class CallSession extends Emitter {
|
||||
this.wakeupResolver({reason: 'session ended'});
|
||||
this.wakeupResolver = null;
|
||||
}
|
||||
|
||||
if (this._maxCallDurationTimer) {
|
||||
clearTimeout(this._maxCallDurationTimer);
|
||||
this._maxCallDurationTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1594,29 +1560,6 @@ Duration=${duration} `
|
||||
this.logger.info({response}, '_lccBoostAudioSignal: response from freeswitch');
|
||||
}
|
||||
|
||||
_lccToolOutput(tool_call_id, opts, callSid) {
|
||||
// only valid if we are in an LLM verb
|
||||
const task = this.currentTask;
|
||||
if (!task || !task.name.startsWith('Llm')) {
|
||||
return this.logger.info('CallSession:_lccToolOutput - invalid command since we are not in an llm');
|
||||
}
|
||||
|
||||
task.processToolOutput(tool_call_id, opts, callSid)
|
||||
.catch((err) => this.logger.error(err, 'CallSession:_lccToolOutput'));
|
||||
}
|
||||
|
||||
|
||||
_lccLlmUpdate(opts, callSid) {
|
||||
// only valid if we are in an LLM verb
|
||||
const task = this.currentTask;
|
||||
if (!task || !task.name.startsWith('Llm')) {
|
||||
return this.logger.info('CallSession:_lccLlmUpdate - invalid command since we are not in an llm');
|
||||
}
|
||||
|
||||
task.processLlmUpdate(opts, callSid)
|
||||
.catch((err) => this.logger.error(err, 'CallSession:_lccLlmUpdate'));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* perform call hangup by jambonz
|
||||
@@ -1677,12 +1620,6 @@ Duration=${duration} `
|
||||
else if (opts.boostAudioSignal) {
|
||||
return this._lccBoostAudioSignal(opts, callSid);
|
||||
}
|
||||
else if (opts.llm_tool_output) {
|
||||
return this._lccToolOutput(opts.tool_call_id, opts.llm_tool_output, callSid);
|
||||
}
|
||||
else if (opts.llm_update) {
|
||||
return this._lccLlmUpdate(opts.llm_update, callSid);
|
||||
}
|
||||
|
||||
// whisper may be the only thing we are asked to do, or it may that
|
||||
// we are doing a whisper after having muted, paused recording etc..
|
||||
@@ -1879,7 +1816,7 @@ Duration=${duration} `
|
||||
this._jambonzHangup();
|
||||
}
|
||||
|
||||
async _onCommand({msgid, command, call_sid, queueCommand, tool_call_id, data}) {
|
||||
async _onCommand({msgid, command, call_sid, queueCommand, data}) {
|
||||
this.logger.info({msgid, command, queueCommand, data}, 'CallSession:_onCommand - received command');
|
||||
let resolution;
|
||||
switch (command) {
|
||||
@@ -1980,14 +1917,6 @@ Duration=${duration} `
|
||||
});
|
||||
break;
|
||||
|
||||
case 'llm:tool-output':
|
||||
this._lccToolOutput(tool_call_id, data, call_sid);
|
||||
break;
|
||||
|
||||
case 'llm:update':
|
||||
this._lccLlmUpdate(data, call_sid);
|
||||
break;
|
||||
|
||||
default:
|
||||
this.logger.info(`CallSession:_onCommand - invalid command ${command}`);
|
||||
}
|
||||
@@ -2303,59 +2232,17 @@ Duration=${duration} `
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle incoming REFER
|
||||
* Handle incoming REFER if we are in a dial task
|
||||
* @param {*} req
|
||||
* @param {*} res
|
||||
*/
|
||||
_onRefer(req, res) {
|
||||
const task = this.currentTask;
|
||||
const sd = task.sd;
|
||||
if (task && TaskName.Dial === task.name && sd && task.referHook) {
|
||||
if (task && TaskName.Dial === task.name && sd) {
|
||||
task.handleRefer(this, req, res);
|
||||
}
|
||||
else {
|
||||
this._handleRefer(req, res);
|
||||
}
|
||||
}
|
||||
|
||||
async _handleRefer(req, res) {
|
||||
if (this._referHook) {
|
||||
try {
|
||||
const to = parseUri(req.getParsedHeader('Refer-To').uri);
|
||||
const by = parseUri(req.getParsedHeader('Referred-By').uri);
|
||||
const b3 = this.b3;
|
||||
const httpHeaders = b3 && {b3};
|
||||
const json = await this.requestor.request('verb:hook', this._referHook, {
|
||||
...(this.callInfo.toJSON()),
|
||||
refer_details: {
|
||||
sip_refer_to: req.get('Refer-To'),
|
||||
sip_referred_by: req.get('Referred-By'),
|
||||
sip_user_agent: req.get('User-Agent'),
|
||||
refer_to_user: to.scheme === 'tel' ? to.number : to.user,
|
||||
referred_by_user: by.scheme === 'tel' ? by.number : by.user,
|
||||
referring_call_sid: this.callSid,
|
||||
referred_call_sid: null,
|
||||
}
|
||||
}, httpHeaders);
|
||||
|
||||
if (json && Array.isArray(json)) {
|
||||
const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
|
||||
if (tasks && tasks.length > 0) {
|
||||
this.logger.info('CallSession:handleRefer received REFER, get new tasks');
|
||||
this.replaceApplication(tasks);
|
||||
if (this.wakeupResolver) {
|
||||
this.wakeupResolver({reason: 'CallSession: referHook new taks'});
|
||||
this.wakeupResolver = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
res.send(202);
|
||||
this.logger.info('CallSession:handleRefer - sent 202 Accepted');
|
||||
} catch (err) {
|
||||
this.logger.error({err}, 'CallSession:handleRefer - error while asking referHook');
|
||||
res.send(err.statusCode || 501);
|
||||
}
|
||||
} else {
|
||||
res.send(501);
|
||||
}
|
||||
}
|
||||
@@ -2685,27 +2572,6 @@ Duration=${duration} `
|
||||
this.verbHookSpan = null;
|
||||
}
|
||||
}
|
||||
|
||||
async startMaxCallDurationTimer(timeLimit) {
|
||||
if (!this._maxCallDurationTimer && timeLimit > 0) {
|
||||
this.timeLimit = timeLimit;
|
||||
this._maxCallDurationTimer = setTimeout(this._onMaxCallDuration.bind(this), timeLimit * 1000);
|
||||
this.logger.debug(`CallSession:startMaxCallDurationTimer - started max call duration timer for ${timeLimit}s`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* _onMaxCallDuration - called when the call has reached the maximum duration
|
||||
*/
|
||||
_onMaxCallDuration() {
|
||||
this.logger.info(`callSession:_onMaxCallDuration tearing down call as it has reached ${this.timeLimit}s`);
|
||||
if (!this.dlg) {
|
||||
this.logger.debug('CallSession:_onMaxCallDuration - no dialog, call already gone');
|
||||
return;
|
||||
}
|
||||
this._jambonzHangup('Max Call Duration');
|
||||
this._maxCallDurationTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = CallSession;
|
||||
|
||||
@@ -72,8 +72,6 @@ class InboundCallSession extends CallSession {
|
||||
|
||||
_jambonzHangup() {
|
||||
this.dlg?.destroy();
|
||||
// kill current task or wakeup the call session.
|
||||
this._callReleased();
|
||||
}
|
||||
|
||||
_hangup(terminatedBy = 'jambonz') {
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
const CallSession = require('./call-session');
|
||||
const {CallStatus} = require('../utils/constants');
|
||||
const moment = require('moment');
|
||||
const {parseUri} = require('drachtio-srf');
|
||||
const { normalizeJambones } = require('@jambonz/verb-specifications');
|
||||
const makeTask = require('../tasks/make_task');
|
||||
|
||||
/**
|
||||
* @classdesc Subclass of CallSession. This represents a CallSession that is
|
||||
* created for an outbound call that is initiated via the REST API.
|
||||
@@ -45,6 +49,59 @@ class RestCallSession extends CallSession {
|
||||
dlg.on('modify', this._onReinvite.bind(this));
|
||||
this.wrapDialog(dlg);
|
||||
}
|
||||
|
||||
/**
|
||||
* global referHook
|
||||
*/
|
||||
|
||||
set referHook(hook) {
|
||||
this._referHook = hook;
|
||||
}
|
||||
|
||||
/**
|
||||
* This is invoked when the called party sends REFER to Jambonz.
|
||||
*/
|
||||
async _onRefer(req, res) {
|
||||
if (this._referHook) {
|
||||
try {
|
||||
const to = parseUri(req.getParsedHeader('Refer-To').uri);
|
||||
const by = parseUri(req.getParsedHeader('Referred-By').uri);
|
||||
const b3 = this.b3;
|
||||
const httpHeaders = b3 && {b3};
|
||||
const json = await this.requestor.request('verb:hook', this._referHook, {
|
||||
...(this.callInfo.toJSON()),
|
||||
refer_details: {
|
||||
sip_refer_to: req.get('Refer-To'),
|
||||
sip_referred_by: req.get('Referred-By'),
|
||||
sip_user_agent: req.get('User-Agent'),
|
||||
refer_to_user: to.scheme === 'tel' ? to.number : to.user,
|
||||
referred_by_user: by.scheme === 'tel' ? by.number : by.user,
|
||||
referring_call_sid: this.callSid,
|
||||
referred_call_sid: null,
|
||||
}
|
||||
}, httpHeaders);
|
||||
|
||||
if (json && Array.isArray(json)) {
|
||||
const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
|
||||
if (tasks && tasks.length > 0) {
|
||||
this.logger.info('RestCallSession:handleRefer received REFER, get new tasks');
|
||||
this.replaceApplication(tasks);
|
||||
if (this.wakeupResolver) {
|
||||
this.wakeupResolver({reason: 'RestCallSession: referHook new taks'});
|
||||
this.wakeupResolver = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
res.send(202);
|
||||
this.logger.info('RestCallSession:handleRefer - sent 202 Accepted');
|
||||
} catch (err) {
|
||||
this.logger.error({err}, 'RestCallSession:handleRefer - error while asking referHook');
|
||||
res.send(err.statusCode || 501);
|
||||
}
|
||||
} else {
|
||||
res.send(501);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* This is invoked when the called party hangs up, in order to calculate the call duration.
|
||||
*/
|
||||
|
||||
@@ -34,8 +34,7 @@ class TaskConfig extends Task {
|
||||
'finishOnKey', 'input', 'numDigits', 'minDigits', 'maxDigits',
|
||||
'interDigitTimeout', 'bargein', 'dtmfBargein', 'minBargeinWordCount', 'actionHook'
|
||||
].forEach((k) => {
|
||||
const val = this.bargeIn[k];
|
||||
if (val !== undefined && val !== null) this.gatherOpts[k] = val;
|
||||
if (this.bargeIn[k]) this.gatherOpts[k] = this.bargeIn[k];
|
||||
});
|
||||
}
|
||||
if (this.transcribe?.enable) {
|
||||
@@ -74,7 +73,6 @@ class TaskConfig extends Task {
|
||||
get hasDub() { return Object.keys(this.dub).length; }
|
||||
get hasVad() { return Object.keys(this.vad).length; }
|
||||
get hasFillerNoise() { return Object.keys(this.fillerNoise).length; }
|
||||
get hasReferHook() { return Object.keys(this.data).includes('referHook'); }
|
||||
|
||||
get summary() {
|
||||
const phrase = [];
|
||||
@@ -84,13 +82,13 @@ class TaskConfig extends Task {
|
||||
|
||||
if (this.bargeIn.enable) phrase.push('enable barge-in');
|
||||
if (this.hasSynthesizer) {
|
||||
const {vendor:v, language:l, voice, label} = this.synthesizer;
|
||||
const s = `{${v},${l},${voice},${label || 'None'}}`;
|
||||
const {vendor:v, language:l, voice} = this.synthesizer;
|
||||
const s = `{${v},${l},${voice}}`;
|
||||
phrase.push(`set synthesizer${s}`);
|
||||
}
|
||||
if (this.hasRecognizer) {
|
||||
const {vendor:v, language:l, label} = this.recognizer;
|
||||
const s = `{${v},${l},${label || 'None'}}`;
|
||||
const {vendor:v, language:l} = this.recognizer;
|
||||
const s = `{${v},${l}}`;
|
||||
phrase.push(`set recognizer${s}`);
|
||||
}
|
||||
if (this.hasRecording) phrase.push(this.record.action);
|
||||
@@ -105,7 +103,6 @@ class TaskConfig extends Task {
|
||||
if (this.notifyEvents) phrase.push(`event notification ${this.notifyEvents ? 'on' : 'off'}`);
|
||||
if (this.onHoldMusic) phrase.push(`onHoldMusic: ${this.onHoldMusic}`);
|
||||
if ('boostAudioSignal' in this.data) phrase.push(`setGain ${this.data.boostAudioSignal}`);
|
||||
if (this.hasReferHook) phrase.push('set referHook');
|
||||
return `${this.name}{${phrase.join(',')}}`;
|
||||
}
|
||||
|
||||
@@ -298,13 +295,9 @@ class TaskConfig extends Task {
|
||||
voiceMs: this.vad.voiceMs || 250,
|
||||
silenceMs: this.vad.silenceMs || 150,
|
||||
strategy: this.vad.strategy || 'one-shot',
|
||||
mode: (this.vad.mode !== undefined && this.vad.mode !== null) ? this.vad.mode : 2
|
||||
mode: this.vad.mod || 2
|
||||
};
|
||||
}
|
||||
|
||||
if (this.hasReferHook) {
|
||||
cs.referHook = this.data.referHook;
|
||||
}
|
||||
}
|
||||
|
||||
async kill(cs) {
|
||||
|
||||
@@ -82,8 +82,6 @@ function filterAndLimit(logger, tasks) {
|
||||
return unique;
|
||||
}
|
||||
|
||||
const sleepFor = (ms) => new Promise((resolve) => setTimeout(() => resolve(), ms));
|
||||
|
||||
class TaskDial extends Task {
|
||||
constructor(logger, opts) {
|
||||
super(logger, opts);
|
||||
@@ -205,16 +203,7 @@ class TaskDial extends Task {
|
||||
else {
|
||||
this.epOther = cs.ep;
|
||||
if (this.dialMusic && this.epOther && this.epOther.connected) {
|
||||
(async() => {
|
||||
do {
|
||||
try {
|
||||
await this.epOther.play(this.dialMusic);
|
||||
} catch (err) {
|
||||
this.logger.error(err, `TaskDial:exec error playing ${this.dialMusic}`);
|
||||
await sleepFor(1000);
|
||||
}
|
||||
} while (!this.killed || !this.bridged);
|
||||
})();
|
||||
this.epOther.play(this.dialMusic).catch((err) => {});
|
||||
}
|
||||
}
|
||||
if (!this.killed) await this._attemptCalls(cs);
|
||||
@@ -340,17 +329,15 @@ class TaskDial extends Task {
|
||||
|
||||
const to = parseUri(req.getParsedHeader('Refer-To').uri);
|
||||
const by = parseUri(req.getParsedHeader('Referred-By').uri);
|
||||
const referredBy = req.get('Referred-By');
|
||||
const userAgent = req.get('User-Agent');
|
||||
this.logger.info({to}, 'refer to parsed');
|
||||
const json = await cs.requestor.request('verb:hook', this.referHook, {
|
||||
...(callInfo.toJSON()),
|
||||
refer_details: {
|
||||
sip_refer_to: req.get('Refer-To'),
|
||||
sip_referred_by: req.get('Referred-By'),
|
||||
sip_user_agent: req.get('User-Agent'),
|
||||
refer_to_user: to.scheme === 'tel' ? to.number : to.user,
|
||||
...(referredBy && {sip_referred_by: referredBy}),
|
||||
...(userAgent && {sip_user_agent: userAgent}),
|
||||
...(by && {referred_by_user: by.scheme === 'tel' ? by.number : by.user}),
|
||||
referred_by_user: by.scheme === 'tel' ? by.number : by.user,
|
||||
referring_call_sid,
|
||||
referred_call_sid
|
||||
}
|
||||
@@ -381,7 +368,6 @@ class TaskDial extends Task {
|
||||
res.send(202);
|
||||
this.logger.info('DialTask:handleRefer - sent 202 Accepted');
|
||||
} catch (err) {
|
||||
this.logger.info({err}, 'DialTask:handleRefer - error processing incoming REFER');
|
||||
res.send(err.statusCode || 501);
|
||||
}
|
||||
}
|
||||
@@ -487,7 +473,7 @@ class TaskDial extends Task {
|
||||
}
|
||||
|
||||
async _attemptCalls(cs) {
|
||||
const {req, callInfo, direction, srf} = cs;
|
||||
const {req, srf} = cs;
|
||||
const {getSBC} = srf.locals;
|
||||
const {lookupTeamsByAccount, lookupAccountBySid} = srf.locals.dbHelpers;
|
||||
const {lookupCarrier, lookupCarrierByPhoneNumber} = dbUtils(this.logger, cs.srf);
|
||||
@@ -499,11 +485,8 @@ class TaskDial extends Task {
|
||||
this.headers = {
|
||||
'X-Account-Sid': cs.accountSid,
|
||||
...(req && req.has('X-CID') && {'X-CID': req.get('X-CID')}),
|
||||
...(direction === 'outbound' && callInfo.sbcCallid && {'X-CID': callInfo.sbcCallid}),
|
||||
...(!JAMBONZ_DISABLE_DIAL_PAI_HEADER && req && {
|
||||
...(req.has('P-Asserted-Identity') && {'P-Asserted-Identity': req.get('P-Asserted-Identity')}),
|
||||
...(req.has('Privacy') && {'Privacy': req.get('Privacy')}),
|
||||
}),
|
||||
...(req && req.has('P-Asserted-Identity') && !JAMBONZ_DISABLE_DIAL_PAI_HEADER &&
|
||||
{'P-Asserted-Identity': req.get('P-Asserted-Identity')}),
|
||||
...(req && req.has('X-Voip-Carrier-Sid') && {'X-Voip-Carrier-Sid': req.get('X-Voip-Carrier-Sid')}),
|
||||
// Put headers at the end to make sure opt.headers override all default behavior.
|
||||
...this.headers
|
||||
@@ -618,7 +601,6 @@ class TaskDial extends Task {
|
||||
dialCallStatus: obj.callStatus,
|
||||
dialSipStatus: obj.sipStatus,
|
||||
dialCallSid: sd.callSid,
|
||||
dialSbcCallid: sd.callInfo.sbcCallid
|
||||
});
|
||||
}
|
||||
switch (obj.callStatus) {
|
||||
|
||||
@@ -12,8 +12,7 @@ const {
|
||||
JambonzTranscriptionEvents,
|
||||
AssemblyAiTranscriptionEvents,
|
||||
VadDetection,
|
||||
VerbioTranscriptionEvents,
|
||||
SpeechmaticsTranscriptionEvents
|
||||
VerbioTranscriptionEvents
|
||||
} = require('../utils/constants.json');
|
||||
const {
|
||||
JAMBONES_GATHER_EARLY_HINTS_MATCH,
|
||||
@@ -23,7 +22,6 @@ const {
|
||||
const makeTask = require('./make_task');
|
||||
const assert = require('assert');
|
||||
const SttTask = require('./stt-task');
|
||||
const { SpeechCredentialError } = require('../utils/error');
|
||||
|
||||
class TaskGather extends SttTask {
|
||||
constructor(logger, opts, parentTask) {
|
||||
@@ -124,20 +122,7 @@ class TaskGather extends SttTask {
|
||||
return s;
|
||||
}
|
||||
|
||||
async exec(cs, obj) {
|
||||
try {
|
||||
await this.handling(cs, obj);
|
||||
} catch (error) {
|
||||
if (error instanceof SpeechCredentialError) {
|
||||
this.logger.info('Gather failed due to SpeechCredentialError, finished!');
|
||||
this.notifyTaskDone();
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async handling(cs, {ep}) {
|
||||
async exec(cs, {ep}) {
|
||||
this.logger.debug({options: this.data}, 'Gather:exec');
|
||||
await super.exec(cs, {ep});
|
||||
const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
|
||||
@@ -192,10 +177,8 @@ class TaskGather extends SttTask {
|
||||
this._startVad();
|
||||
|
||||
const startDtmfListener = () => {
|
||||
assert(!this._dtmfListenerStarted);
|
||||
if (this.input.includes('digits') || this.dtmfBargein || this.asrDtmfTerminationDigit) {
|
||||
ep.on('dtmf', this._onDtmf.bind(this, cs, ep));
|
||||
this._dtmfListenerStarted = true;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -298,8 +281,7 @@ class TaskGather extends SttTask {
|
||||
}
|
||||
}
|
||||
|
||||
// https://github.com/jambonz/jambonz-feature-server/issues/913
|
||||
if (this.listenDuringPrompt || (!this.sayTask && !this.playTask)) {
|
||||
if (this.listenDuringPrompt) {
|
||||
startDtmfListener();
|
||||
}
|
||||
|
||||
@@ -515,24 +497,6 @@ class TaskGather extends SttTask {
|
||||
this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.ConnectFailure,
|
||||
this._onVendorConnectFailure.bind(this, cs, ep));
|
||||
break;
|
||||
|
||||
case 'speechmatics':
|
||||
this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
|
||||
this.addCustomEventListener(
|
||||
ep, SpeechmaticsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
|
||||
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Info,
|
||||
this._onSpeechmaticsInfo.bind(this, cs, ep));
|
||||
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.RecognitionStarted,
|
||||
this._onSpeechmaticsRecognitionStarted.bind(this, cs, ep));
|
||||
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Connect,
|
||||
this._onVendorConnect.bind(this, cs, ep));
|
||||
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.ConnectFailure,
|
||||
this._onVendorConnectFailure.bind(this, cs, ep));
|
||||
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Error,
|
||||
this._onSpeechmaticsErrror.bind(this, cs, ep));
|
||||
|
||||
break;
|
||||
|
||||
default:
|
||||
if (this.vendor.startsWith('custom:')) {
|
||||
this.bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
|
||||
@@ -771,7 +735,7 @@ class TaskGather extends SttTask {
|
||||
|
||||
evt = this.normalizeTranscription(evt, this.vendor, 1, this.language,
|
||||
this.shortUtterance, this.data.recognizer.punctuation);
|
||||
this.logger.debug({evt, bugname, finished, vendor: this.vendor}, 'Gather:_onTranscription normalized transcript');
|
||||
//this.logger.debug({evt, bugname, finished, vendor: this.vendor}, 'Gather:_onTranscription normalized transcript');
|
||||
|
||||
if (evt.alternatives.length === 0) {
|
||||
this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, continue listening');
|
||||
@@ -1006,11 +970,10 @@ class TaskGather extends SttTask {
|
||||
if (code === 413 && error === 'Too much speech') return this._resolve('timeout');
|
||||
}
|
||||
this.logger.info({evt}, 'TaskGather:_onJambonzError');
|
||||
const errMessage = evt.error || evt.Message;
|
||||
writeAlerts({
|
||||
account_sid: cs.accountSid,
|
||||
alert_type: AlertType.STT_FAILURE,
|
||||
message: `Custom speech vendor ${this.vendor} error: ${errMessage}`,
|
||||
message: `Custom speech vendor ${this.vendor} error: ${evt.error}`,
|
||||
vendor: this.vendor,
|
||||
target_sid: cs.callSid
|
||||
}).catch((err) => this.logger.info({err}, 'Error generating alert for jambonz custom connection failure'));
|
||||
@@ -1026,25 +989,12 @@ class TaskGather extends SttTask {
|
||||
}
|
||||
}
|
||||
|
||||
async _onSpeechmaticsErrror(cs, _ep, evt) {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
const {message, ...e} = evt;
|
||||
this._onVendorError(cs, _ep, {error: JSON.stringify(e)});
|
||||
}
|
||||
|
||||
async _onVendorError(cs, _ep, evt) {
|
||||
super._onVendorError(cs, _ep, evt);
|
||||
if (!(await this._startFallback(cs, _ep, evt))) {
|
||||
this._resolve('stt-error', evt);
|
||||
}
|
||||
}
|
||||
async _onSpeechmaticsRecognitionStarted(_cs, _ep, evt) {
|
||||
this.logger.debug({evt}, 'TaskGather:_onSpeechmaticsRecognitionStarted');
|
||||
}
|
||||
|
||||
async _onSpeechmaticsInfo(_cs, _ep, evt) {
|
||||
this.logger.debug({evt}, 'TaskGather:_onSpeechmaticsInfo');
|
||||
}
|
||||
|
||||
_onVadDetected(cs, ep) {
|
||||
if (this.bargein && this.minBargeinWordCount === 0) {
|
||||
@@ -1103,7 +1053,6 @@ class TaskGather extends SttTask {
|
||||
|
||||
this.span.setAttributes({
|
||||
channel: 1,
|
||||
'stt.label': this.label || 'None',
|
||||
'stt.resolve': reason,
|
||||
'stt.result': JSON.stringify(evt)
|
||||
});
|
||||
@@ -1137,18 +1086,15 @@ class TaskGather extends SttTask {
|
||||
if (this.parentTask) this.parentTask.emit('dtmf', evt);
|
||||
else {
|
||||
this.emit('dtmf', evt);
|
||||
const payload = {digits: this.digitBuffer, reason: 'dtmfDetected'};
|
||||
this.logger.info({payload}, 'TaskGather:_resolve - sending dtmf event to app');
|
||||
returnedVerbs = await this.performAction(payload);
|
||||
returnedVerbs = await this.performAction({digits: this.digitBuffer, reason: 'dtmfDetected'});
|
||||
}
|
||||
}
|
||||
else if (reason.startsWith('speech')) {
|
||||
if (this.parentTask) this.parentTask.emit('transcription', evt);
|
||||
else {
|
||||
this.emit('transcription', evt);
|
||||
const payload = {speech: evt, reason: 'speechDetected'};
|
||||
this.logger.info({payload}, 'TaskGather:_resolve - sending speech transcript to app');
|
||||
returnedVerbs = await this.performAction(payload);
|
||||
this.logger.debug('TaskGather:_resolve - invoking performAction');
|
||||
returnedVerbs = await this.performAction({speech: evt, reason: 'speechDetected'});
|
||||
this.logger.debug({returnedVerbs}, 'TaskGather:_resolve - back from performAction');
|
||||
}
|
||||
}
|
||||
@@ -1156,7 +1102,6 @@ class TaskGather extends SttTask {
|
||||
if (this.parentTask) this.parentTask.emit('timeout', evt);
|
||||
else {
|
||||
this.emit('timeout', evt);
|
||||
this.logger.info('TaskGather:_resolve - sending timeout event to app');
|
||||
returnedVerbs = await this.performAction({reason: 'timeout'});
|
||||
}
|
||||
}
|
||||
@@ -1164,15 +1109,12 @@ class TaskGather extends SttTask {
|
||||
if (this.parentTask) this.parentTask.emit('stt-error', evt);
|
||||
else {
|
||||
this.emit('stt-error', evt);
|
||||
const payload = {reason: 'stt-error', details: evt.error};
|
||||
this.logger.info({payload}, 'TaskGather:_resolve - sending stt-error event to app');
|
||||
returnedVerbs = await this.performAction(payload);
|
||||
returnedVerbs = await this.performAction({reason: 'error', details: evt.error});
|
||||
}
|
||||
} else if (reason.startsWith('stt-low-confidence')) {
|
||||
if (this.parentTask) this.parentTask.emit('stt-low-confidence', evt);
|
||||
else {
|
||||
this.emit('stt-low-confidence', evt);
|
||||
this.logger.info('TaskGather:_resolve - sending stt-low-confidence event to app');
|
||||
returnedVerbs = await this.performAction({reason: 'stt-low-confidence'});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
const Task = require('../task');
|
||||
const {TaskPreconditions} = require('../../utils/constants');
|
||||
const TaskLlmOpenAI_S2S = require('./llms/openai_s2s');
|
||||
|
||||
class TaskLlm extends Task {
|
||||
constructor(logger, opts) {
|
||||
super(logger, opts);
|
||||
this.preconditions = TaskPreconditions.Endpoint;
|
||||
|
||||
['vendor', 'model', 'auth', 'connectOptions'].forEach((prop) => {
|
||||
this[prop] = this.data[prop];
|
||||
});
|
||||
|
||||
this.eventHandlers = [];
|
||||
|
||||
// delegate to the specific llm model
|
||||
this.llm = this.createSpecificLlm();
|
||||
}
|
||||
|
||||
get name() { return this.llm.name ; }
|
||||
|
||||
get toolHook() { return this.llm?.toolHook; }
|
||||
|
||||
get eventHook() { return this.llm?.eventHook; }
|
||||
|
||||
get ep() { return this.cs.ep; }
|
||||
|
||||
async exec(cs, {ep}) {
|
||||
await super.exec(cs, {ep});
|
||||
await this.llm.exec(cs, {ep});
|
||||
}
|
||||
|
||||
async kill(cs) {
|
||||
super.kill(cs);
|
||||
await this.llm.kill(cs);
|
||||
}
|
||||
|
||||
createSpecificLlm() {
|
||||
let llm;
|
||||
switch (this.vendor) {
|
||||
case 'openai':
|
||||
case 'microsoft':
|
||||
if (this.model.startsWith('gpt-4o-realtime')) {
|
||||
llm = new TaskLlmOpenAI_S2S(this.logger, this.data, this);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unsupported vendor ${this.vendor} for LLM`);
|
||||
}
|
||||
|
||||
if (!llm) {
|
||||
throw new Error(`Unsupported vendor:model ${this.vendor}:${this.model}`);
|
||||
}
|
||||
return llm;
|
||||
}
|
||||
|
||||
addCustomEventListener(ep, event, handler) {
|
||||
this.eventHandlers.push({ep, event, handler});
|
||||
ep.addCustomEventListener(event, handler);
|
||||
}
|
||||
|
||||
removeCustomEventListeners() {
|
||||
this.eventHandlers.forEach((h) => h.ep.removeCustomEventListener(h.event, h.handler));
|
||||
}
|
||||
|
||||
async sendEventHook(data) {
|
||||
await this.cs?.requestor.request('llm:event', this.eventHook, data);
|
||||
}
|
||||
|
||||
async sendToolHook(tool_call_id, data) {
|
||||
await this.cs?.requestor.request('llm:tool-call', this.toolHook, {tool_call_id, ...data});
|
||||
}
|
||||
|
||||
async processToolOutput(tool_call_id, data) {
|
||||
if (!this.ep.connected) {
|
||||
this.logger.info('TaskLlm:processToolOutput - no connected endpoint');
|
||||
return;
|
||||
}
|
||||
this.llm.processToolOutput(this.ep, tool_call_id, data);
|
||||
}
|
||||
|
||||
async processLlmUpdate(data, callSid) {
|
||||
if (this.ep.connected) {
|
||||
if (typeof this.llm.processLlmUpdate === 'function') {
|
||||
this.llm.processLlmUpdate(this.ep, data, callSid);
|
||||
}
|
||||
else {
|
||||
const {vendor, model} = this.llm;
|
||||
this.logger.info({data, callSid},
|
||||
`TaskLlm:_processLlmUpdate: LLM ${vendor}:${model} does not support llm:update`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = TaskLlm;
|
||||
@@ -1,357 +0,0 @@
|
||||
const Task = require('../../task');
|
||||
const TaskName = 'Llm_OpenAI_s2s';
|
||||
const {LlmEvents_OpenAI} = require('../../../utils/constants');
|
||||
const ClientEvent = 'client.event';
|
||||
const SessionDelete = 'session.delete';
|
||||
|
||||
const openai_server_events = [
|
||||
'error',
|
||||
'session.created',
|
||||
'session.updated',
|
||||
'conversation.created',
|
||||
'input_audio_buffer.committed',
|
||||
'input_audio_buffer.cleared',
|
||||
'input_audio_buffer.speech_started',
|
||||
'input_audio_buffer.speech_stopped',
|
||||
'conversation.item.created',
|
||||
'conversation.item.input_audio_transcription.completed',
|
||||
'conversation.item.input_audio_transcription.failed',
|
||||
'conversation.item.truncated',
|
||||
'conversation.item.deleted',
|
||||
'response.created',
|
||||
'response.done',
|
||||
'response.output_item.added',
|
||||
'response.output_item.done',
|
||||
'response.content_part.added',
|
||||
'response.content_part.done',
|
||||
'response.text.delta',
|
||||
'response.text.done',
|
||||
'response.audio_transcript.delta',
|
||||
'response.audio_transcript.done',
|
||||
'response.audio.delta',
|
||||
'response.audio.done',
|
||||
'response.function_call_arguments.delta',
|
||||
'response.function_call_arguments.done',
|
||||
'rate_limits.updated',
|
||||
'output_audio.playback_started',
|
||||
'output_audio.playback_stopped',
|
||||
];
|
||||
|
||||
const expandWildcards = (events) => {
|
||||
const expandedEvents = [];
|
||||
|
||||
events.forEach((evt) => {
|
||||
if (evt.endsWith('.*')) {
|
||||
const prefix = evt.slice(0, -2); // Remove the wildcard ".*"
|
||||
const matchingEvents = openai_server_events.filter((e) => e.startsWith(prefix));
|
||||
expandedEvents.push(...matchingEvents);
|
||||
} else {
|
||||
expandedEvents.push(evt);
|
||||
}
|
||||
});
|
||||
|
||||
return expandedEvents;
|
||||
};
|
||||
|
||||
class TaskLlmOpenAI_S2S extends Task {
|
||||
constructor(logger, opts, parentTask) {
|
||||
super(logger, opts, parentTask);
|
||||
this.parent = parentTask;
|
||||
|
||||
this.vendor = this.parent.vendor;
|
||||
this.model = this.parent.model;
|
||||
this.auth = this.parent.auth;
|
||||
this.connectionOptions = this.parent.connectOptions;
|
||||
|
||||
const {apiKey} = this.auth || {};
|
||||
if (!apiKey) throw new Error('auth.apiKey is required for OpenAI S2S');
|
||||
|
||||
if (['openai', 'microsoft'].indexOf(this.vendor) === -1) {
|
||||
throw new Error(`Invalid vendor ${this.vendor} for OpenAI S2S`);
|
||||
}
|
||||
|
||||
if ('microsoft' === this.vendor && !this.connectionOptions?.host) {
|
||||
throw new Error('connectionOptions.host is required for Microsoft OpenAI S2S');
|
||||
}
|
||||
|
||||
this.apiKey = apiKey;
|
||||
this.authType = 'microsoft' === this.vendor ? 'query' : 'bearer';
|
||||
this.actionHook = this.data.actionHook;
|
||||
this.eventHook = this.data.eventHook;
|
||||
this.toolHook = this.data.toolHook;
|
||||
const {response_create, session_update} = this.data.llmOptions;
|
||||
|
||||
if (typeof response_create !== 'object') {
|
||||
throw new Error('llmOptions with an initial response.create is required for OpenAI S2S');
|
||||
}
|
||||
|
||||
this.response_create = response_create;
|
||||
this.session_update = session_update;
|
||||
|
||||
this.results = {
|
||||
completionReason: 'normal conversation end'
|
||||
};
|
||||
|
||||
/**
|
||||
* only one of these will have items,
|
||||
* if includeEvents, then these are the events to include
|
||||
* if excludeEvents, then these are the events to exclude
|
||||
*/
|
||||
this.includeEvents = [];
|
||||
this.excludeEvents = [];
|
||||
|
||||
/* default to all events if user did not specify */
|
||||
this._populateEvents(this.data.events || openai_server_events);
|
||||
|
||||
this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
|
||||
this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
|
||||
}
|
||||
|
||||
get name() { return TaskName; }
|
||||
|
||||
get host() {
|
||||
const {host} = this.connectionOptions || {};
|
||||
return host || (this.vendor === 'openai' ? 'api.openai.com' : void 0);
|
||||
}
|
||||
|
||||
get path() {
|
||||
const {path} = this.connectionOptions || {};
|
||||
if (path) return path;
|
||||
|
||||
switch (this.vendor) {
|
||||
case 'openai':
|
||||
return 'v1/realtime?model=gpt-4o-realtime-preview-2024-10-01';
|
||||
case 'microsoft':
|
||||
return 'openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview-1001&';
|
||||
}
|
||||
}
|
||||
|
||||
async _api(ep, args) {
|
||||
const res = await ep.api('uuid_openai_s2s', `^^|${args.join('|')}`);
|
||||
if (!res.body?.startsWith('+OK')) {
|
||||
throw new Error({args}, `Error calling uuid_openai_s2s: ${res.body}`);
|
||||
}
|
||||
}
|
||||
|
||||
async exec(cs, {ep}) {
|
||||
await super.exec(cs);
|
||||
|
||||
await this._startListening(cs, ep);
|
||||
|
||||
await this.awaitTaskDone();
|
||||
|
||||
/* note: the parent llm verb started the span, which is why this is necessary */
|
||||
await this.parent.performAction(this.results);
|
||||
|
||||
this._unregisterHandlers();
|
||||
}
|
||||
|
||||
async kill(cs) {
|
||||
super.kill(cs);
|
||||
|
||||
this._api(cs.ep, [cs.ep.uuid, SessionDelete])
|
||||
.catch((err) => this.logger.info({err}, 'TaskLlmOpenAI_S2S:kill - error deleting session'));
|
||||
|
||||
this.notifyTaskDone();
|
||||
}
|
||||
|
||||
/**
|
||||
* Send function call output to the OpenAI server in the form of conversation.item.create
|
||||
* per https://platform.openai.com/docs/guides/realtime/function-calls
|
||||
*/
|
||||
async processToolOutput(ep, tool_call_id, data) {
|
||||
try {
|
||||
this.logger.debug({tool_call_id, data}, 'TaskLlmOpenAI_S2S:processToolOutput');
|
||||
|
||||
if (!data.type || data.type !== 'conversation.item.create') {
|
||||
this.logger.info({data},
|
||||
'TaskLlmOpenAI_S2S:processToolOutput - invalid tool output, must be conversation.item.create');
|
||||
}
|
||||
else {
|
||||
await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
|
||||
|
||||
// spec also recommends to send immediate response.create
|
||||
await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify({type: 'response.create'})]);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.info({err}, 'TaskLlmOpenAI_S2S:processToolOutput');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a session.update to the OpenAI server
|
||||
* Note: creating and deleting conversation items also supported as well as interrupting the assistant
|
||||
*/
|
||||
async processLlmUpdate(ep, data, _callSid) {
|
||||
try {
|
||||
this.logger.debug({data, _callSid}, 'TaskLlmOpenAI_S2S:processLlmUpdate');
|
||||
|
||||
if (!data.type || ![
|
||||
'session.update',
|
||||
'conversation.item.create',
|
||||
'conversation.item.delete',
|
||||
'response.cancel'
|
||||
].includes(data.type)) {
|
||||
this.logger.info({data}, 'TaskLlmOpenAI_S2S:processLlmUpdate - invalid mid-call request');
|
||||
}
|
||||
else {
|
||||
await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.info({err}, 'TaskLlmOpenAI_S2S:processLlmUpdate');
|
||||
}
|
||||
}
|
||||
|
||||
async _startListening(cs, ep) {
|
||||
this._registerHandlers(ep);
|
||||
|
||||
try {
|
||||
const args = [ep.uuid, 'session.create', this.host, this.path, this.authType, this.apiKey];
|
||||
await this._api(ep, args);
|
||||
} catch (err) {
|
||||
this.logger.error({err}, 'TaskLlmOpenAI_S2S:_startListening');
|
||||
this.notifyTaskDone();
|
||||
}
|
||||
}
|
||||
|
||||
async _sendClientEvent(ep, obj) {
|
||||
let ok = true;
|
||||
this.logger.debug({obj}, 'TaskLlmOpenAI_S2S:_sendClientEvent');
|
||||
try {
|
||||
const args = [ep.uuid, ClientEvent, JSON.stringify(obj)];
|
||||
await this._api(ep, args);
|
||||
} catch (err) {
|
||||
ok = false;
|
||||
this.logger.error({err}, 'TaskLlmOpenAI_S2S:_sendClientEvent - Error');
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
async _sendInitialMessage(ep) {
|
||||
let obj = {type: 'response.create', response: this.response_create};
|
||||
if (!await this._sendClientEvent(ep, obj)) {
|
||||
this.notifyTaskDone();
|
||||
}
|
||||
|
||||
/* send immediate session.update if present */
|
||||
else if (this.session_update) {
|
||||
obj = {type: 'session.update', session: this.session_update};
|
||||
this.logger.debug({obj}, 'TaskLlmOpenAI_S2S:_sendInitialMessage - sending session.update');
|
||||
if (!await this._sendClientEvent(ep, obj)) {
|
||||
this.notifyTaskDone();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_registerHandlers(ep) {
|
||||
this.addCustomEventListener(ep, LlmEvents_OpenAI.Connect, this._onConnect.bind(this, ep));
|
||||
this.addCustomEventListener(ep, LlmEvents_OpenAI.ConnectFailure, this._onConnectFailure.bind(this, ep));
|
||||
this.addCustomEventListener(ep, LlmEvents_OpenAI.Disconnect, this._onDisconnect.bind(this, ep));
|
||||
this.addCustomEventListener(ep, LlmEvents_OpenAI.ServerEvent, this._onServerEvent.bind(this, ep));
|
||||
}
|
||||
|
||||
_unregisterHandlers() {
|
||||
this.removeCustomEventListeners();
|
||||
}
|
||||
|
||||
_onError(ep, evt) {
|
||||
this.logger.info({evt}, 'TaskLlmOpenAI_S2S:_onError');
|
||||
this.notifyTaskDone();
|
||||
}
|
||||
|
||||
_onConnect(ep) {
|
||||
this.logger.debug('TaskLlmOpenAI_S2S:_onConnect');
|
||||
this._sendInitialMessage(ep);
|
||||
}
|
||||
_onConnectFailure(_ep, evt) {
|
||||
this.logger.info(evt, 'TaskLlmOpenAI_S2S:_onConnectFailure');
|
||||
this.results = {completionReason: 'connection failure'};
|
||||
this.notifyTaskDone();
|
||||
}
|
||||
_onDisconnect(_ep, evt) {
|
||||
this.logger.info(evt, 'TaskLlmOpenAI_S2S:_onConnectFailure');
|
||||
this.results = {completionReason: 'disconnect from remote end'};
|
||||
this.notifyTaskDone();
|
||||
}
|
||||
async _onServerEvent(ep, evt) {
|
||||
let endConversation = false;
|
||||
const type = evt.type;
|
||||
this.logger.info({evt}, 'TaskLlmOpenAI_S2S:_onServerEvent');
|
||||
|
||||
/* check for failures, such as rate limit exceeded, that should terminate the conversation */
|
||||
if (type === 'response.done' && evt.response.status === 'failed') {
|
||||
endConversation = true;
|
||||
this.results = {
|
||||
completionReason: 'server failure',
|
||||
error: evt.response.status_details?.error
|
||||
};
|
||||
}
|
||||
|
||||
/* server errors of some sort */
|
||||
else if (type === 'error') {
|
||||
endConversation = true;
|
||||
this.results = {
|
||||
completionReason: 'server error',
|
||||
error: evt.error
|
||||
};
|
||||
}
|
||||
|
||||
/* tool calls */
|
||||
else if (type === 'response.output_item.done' && evt.item?.type === 'function_call') {
|
||||
this.logger.debug({evt}, 'TaskLlmOpenAI_S2S:_onServerEvent - function_call');
|
||||
if (!this.toolHook) {
|
||||
this.logger.warn({evt}, 'TaskLlmOpenAI_S2S:_onServerEvent - no toolHook defined!');
|
||||
}
|
||||
else {
|
||||
const {name, call_id} = evt.item;
|
||||
const args = JSON.parse(evt.item.arguments);
|
||||
|
||||
try {
|
||||
await this.parent.sendToolHook(call_id, {name, args});
|
||||
} catch (err) {
|
||||
this.logger.info({err, evt}, 'TaskLlmOpenAI - error calling function');
|
||||
this.results = {
|
||||
completionReason: 'client error calling function',
|
||||
error: err
|
||||
};
|
||||
endConversation = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* check whether we should notify on this event */
|
||||
if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
|
||||
this.parent.sendEventHook(evt)
|
||||
.catch((err) => this.logger.info({err}, 'TaskLlmOpenAI_S2S:_onServerEvent - error sending event hook'));
|
||||
}
|
||||
|
||||
if (endConversation) {
|
||||
this.logger.info({results: this.results}, 'TaskLlmOpenAI_S2S:_onServerEvent - ending conversation due to error');
|
||||
this.notifyTaskDone();
|
||||
}
|
||||
}
|
||||
|
||||
_populateEvents(events) {
|
||||
if (events.includes('all')) {
|
||||
/* work by excluding specific events */
|
||||
const exclude = events
|
||||
.filter((evt) => evt.startsWith('-'))
|
||||
.map((evt) => evt.slice(1));
|
||||
if (exclude.length === 0) this.includeEvents = openai_server_events;
|
||||
else this.excludeEvents = expandWildcards(exclude);
|
||||
}
|
||||
else {
|
||||
/* work by including specific events */
|
||||
const include = events
|
||||
.filter((evt) => !evt.startsWith('-'));
|
||||
this.includeEvents = expandWildcards(include);
|
||||
}
|
||||
|
||||
this.logger.debug({
|
||||
includeEvents: this.includeEvents,
|
||||
excludeEvents: this.excludeEvents
|
||||
}, 'TaskLlmOpenAI_S2S:_populateEvents');
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = TaskLlmOpenAI_S2S;
|
||||
@@ -62,9 +62,6 @@ function makeTask(logger, obj, parent) {
|
||||
case TaskName.Message:
|
||||
const TaskMessage = require('./message');
|
||||
return new TaskMessage(logger, data, parent);
|
||||
case TaskName.Llm:
|
||||
const TaskLlm = require('./llm');
|
||||
return new TaskLlm(logger, data, parent);
|
||||
case TaskName.Rasa:
|
||||
const TaskRasa = require('./rasa');
|
||||
return new TaskRasa(logger, data, parent);
|
||||
|
||||
@@ -12,7 +12,6 @@ class TaskRestDial extends Task {
|
||||
|
||||
this.from = this.data.from;
|
||||
this.callerName = this.data.callerName;
|
||||
this.timeLimit = this.data.timeLimit;
|
||||
this.fromHost = this.data.fromHost;
|
||||
this.to = this.data.to;
|
||||
this.call_hook = this.data.call_hook;
|
||||
@@ -67,9 +66,6 @@ class TaskRestDial extends Task {
|
||||
const cs = this.callSession;
|
||||
cs.setDialog(dlg);
|
||||
cs.referHook = this.referHook;
|
||||
if (this.timeLimit) {
|
||||
cs.startMaxCallDurationTimer(this.timeLimit);
|
||||
}
|
||||
this.logger.debug('TaskRestDial:_onConnect - call connected');
|
||||
if (this.sipRequestWithinDialogHook) this._initSipRequestWithinDialogHandler(cs, dlg);
|
||||
try {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
const TtsTask = require('./tts-task');
|
||||
const {TaskName, TaskPreconditions} = require('../utils/constants');
|
||||
const pollySSMLSplit = require('polly-ssml-split');
|
||||
const { SpeechCredentialError } = require('../utils/error');
|
||||
|
||||
const breakLengthyTextIfNeeded = (logger, text) => {
|
||||
const chunkSize = 1000;
|
||||
@@ -62,23 +61,7 @@ class TaskSay extends TtsTask {
|
||||
}
|
||||
}
|
||||
|
||||
async exec(cs, obj) {
|
||||
try {
|
||||
await this.handling(cs, obj);
|
||||
this.emit('playDone');
|
||||
} catch (error) {
|
||||
if (error instanceof SpeechCredentialError) {
|
||||
// if say failed due to speech credentials, alarm is writtern and error notification is sent
|
||||
// finished this say to move to next task.
|
||||
this.logger.info('Say failed due to SpeechCredentialError, finished!');
|
||||
this.emit('playDone');
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async handling(cs, {ep}) {
|
||||
async exec(cs, {ep}) {
|
||||
const {srf, accountSid:account_sid, callSid:target_sid} = cs;
|
||||
const {writeAlerts, AlertType} = srf.locals;
|
||||
const {addFileToCache} = srf.locals.dbHelpers;
|
||||
@@ -134,7 +117,7 @@ class TaskSay extends TtsTask {
|
||||
} else {
|
||||
this.notifyError(
|
||||
{ msg: 'TTS error', details:`TTS vendor ${vendor} error: ${error}`, failover: 'not available'});
|
||||
throw new SpeechCredentialError(error.message);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
let filepath;
|
||||
@@ -177,22 +160,32 @@ class TaskSay extends TtsTask {
|
||||
account_sid,
|
||||
alert_type: AlertType.TTS_FAILURE,
|
||||
vendor,
|
||||
detail: evt.variable_tts_error,
|
||||
target_sid
|
||||
detail: evt.variable_tts_error
|
||||
}).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
|
||||
}
|
||||
if (evt.variable_tts_cache_filename && !this.killed) {
|
||||
const text = parseTextFromSayString(this.text[segment]);
|
||||
addFileToCache(evt.variable_tts_cache_filename, {
|
||||
account_sid,
|
||||
vendor,
|
||||
language,
|
||||
voice,
|
||||
engine,
|
||||
text
|
||||
}).catch((err) => this.logger.info({err}, 'Error adding file to cache'));
|
||||
else {
|
||||
this.logger.debug({evt}, 'Say got playback-stop');
|
||||
if (evt.variable_tts_error) {
|
||||
writeAlerts({
|
||||
account_sid,
|
||||
alert_type: AlertType.TTS_FAILURE,
|
||||
vendor,
|
||||
detail: evt.variable_tts_error,
|
||||
target_sid
|
||||
}).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
|
||||
}
|
||||
if (evt.variable_tts_cache_filename && !this.killed) {
|
||||
const text = parseTextFromSayString(this.text[segment]);
|
||||
addFileToCache(evt.variable_tts_cache_filename, {
|
||||
account_sid,
|
||||
vendor,
|
||||
language,
|
||||
voice,
|
||||
engine,
|
||||
text
|
||||
}).catch((err) => this.logger.info({err}, 'Error adding file to cache'));
|
||||
}
|
||||
}
|
||||
|
||||
if (this._playResolve) {
|
||||
evt.variable_tts_error ? this._playReject(new Error(evt.variable_tts_error)) : this._playResolve();
|
||||
}
|
||||
@@ -213,7 +206,6 @@ class TaskSay extends TtsTask {
|
||||
continue;
|
||||
} catch (err) {
|
||||
this.logger.info({err}, 'Error waiting for playback-stop event');
|
||||
throw err;
|
||||
}
|
||||
} finally {
|
||||
this._playPromise = null;
|
||||
@@ -231,6 +223,7 @@ class TaskSay extends TtsTask {
|
||||
segment++;
|
||||
}
|
||||
}
|
||||
this.emit('playDone');
|
||||
}
|
||||
|
||||
async kill(cs) {
|
||||
|
||||
@@ -94,10 +94,7 @@ class TaskSipRefer extends Task {
|
||||
}
|
||||
if (status >= 200) {
|
||||
this.referSpan.setAttributes({'refer.finalNotify': status});
|
||||
await this.performAction({refer_status: 202, final_referred_call_status: status})
|
||||
.catch((err) => {
|
||||
this.logger.error(err, 'TaskSipRefer:exec - error performing action finalNotify');
|
||||
});
|
||||
await this.performAction({refer_status: 202, final_referred_call_status: status});
|
||||
this.notifyTaskDone();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,8 +2,6 @@ const Task = require('./task');
|
||||
const assert = require('assert');
|
||||
const crypto = require('crypto');
|
||||
const { TaskPreconditions, CobaltTranscriptionEvents } = require('../utils/constants');
|
||||
const { SpeechCredentialError } = require('../utils/error');
|
||||
const {JAMBONES_AWS_TRANSCRIBE_USE_GRPC} = require('../config');
|
||||
|
||||
class SttTask extends Task {
|
||||
|
||||
@@ -18,14 +16,12 @@ class SttTask extends Task {
|
||||
normalizeTranscription,
|
||||
setSpeechCredentialsAtRuntime,
|
||||
compileSonioxTranscripts,
|
||||
consolidateTranscripts,
|
||||
updateSpeechmaticsPayload
|
||||
consolidateTranscripts
|
||||
} = require('../utils/transcription-utils')(logger);
|
||||
this.setChannelVarsForStt = setChannelVarsForStt;
|
||||
this.normalizeTranscription = normalizeTranscription;
|
||||
this.compileSonioxTranscripts = compileSonioxTranscripts;
|
||||
this.consolidateTranscripts = consolidateTranscripts;
|
||||
this.updateSpeechmaticsPayload = updateSpeechmaticsPayload;
|
||||
this.eventHandlers = [];
|
||||
this.isHandledByPrimaryProvider = true;
|
||||
/**
|
||||
@@ -194,7 +190,7 @@ class SttTask extends Task {
|
||||
target_sid: cs.callSid
|
||||
}).catch((err) => this.logger.info({err}, 'Error generating alert for no stt'));
|
||||
// the ASR might have fallback configuration, should not done task here.
|
||||
throw new SpeechCredentialError(`No speech-to-text service credentials for ${vendor} have been configured`);
|
||||
throw new Error(`No speech-to-text service credentials for ${vendor} have been configured`);
|
||||
}
|
||||
|
||||
if (vendor === 'nuance' && credentials.client_id) {
|
||||
@@ -218,25 +214,14 @@ class SttTask extends Task {
|
||||
region,
|
||||
roleArn
|
||||
});
|
||||
this.logger.debug({roleArn}, `(roleArn) got aws access token ${servedFromCache ? 'from cache' : ''}`);
|
||||
this.logger.debug({roleArn}, `got aws access token ${servedFromCache ? 'from cache' : ''}`);
|
||||
credentials = {...credentials, accessKeyId, secretAccessKey, sessionToken};
|
||||
}
|
||||
else if (vendor === 'verbio' && credentials.client_id && credentials.client_secret) {
|
||||
} else if (vendor === 'verbio' && credentials.client_id && credentials.client_secret) {
|
||||
const {access_token, servedFromCache} = await getVerbioAccessToken(credentials);
|
||||
this.logger.debug({client_id: credentials.client_id},
|
||||
`got verbio access token ${servedFromCache ? 'from cache' : ''}`);
|
||||
credentials.access_token = access_token;
|
||||
}
|
||||
else if (vendor == 'aws' && !JAMBONES_AWS_TRANSCRIBE_USE_GRPC) {
|
||||
/* get AWS access token */
|
||||
const {accessKeyId, secretAccessKey, securityToken, region } = credentials;
|
||||
if (!securityToken) {
|
||||
const { servedFromCache, ...newCredentials} = await getAwsAuthToken({accessKeyId, secretAccessKey, region});
|
||||
this.logger.debug({newCredentials}, `got aws security token ${servedFromCache ? 'from cache' : ''}`);
|
||||
credentials = {...newCredentials, region};
|
||||
}
|
||||
}
|
||||
|
||||
return credentials;
|
||||
}
|
||||
|
||||
|
||||
@@ -12,13 +12,10 @@ const {
|
||||
NvidiaTranscriptionEvents,
|
||||
JambonzTranscriptionEvents,
|
||||
TranscribeStatus,
|
||||
AssemblyAiTranscriptionEvents,
|
||||
VerbioTranscriptionEvents,
|
||||
SpeechmaticsTranscriptionEvents
|
||||
AssemblyAiTranscriptionEvents
|
||||
} = require('../utils/constants.json');
|
||||
const { normalizeJambones } = require('@jambonz/verb-specifications');
|
||||
const SttTask = require('./stt-task');
|
||||
const { SpeechCredentialError } = require('../utils/error');
|
||||
|
||||
const STT_LISTEN_SPAN_NAME = 'stt-listen';
|
||||
|
||||
@@ -76,20 +73,7 @@ class TaskTranscribe extends SttTask {
|
||||
return this.channel === 2 || this.separateRecognitionPerChannel && this.ep2;
|
||||
}
|
||||
|
||||
async exec(cs, obj) {
|
||||
try {
|
||||
await this.handling(cs, obj);
|
||||
} catch (error) {
|
||||
if (error instanceof SpeechCredentialError) {
|
||||
this.logger.info('Transcribe failed due to SpeechCredentialError, finished!');
|
||||
this.notifyTaskDone();
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async handling(cs, {ep, ep2}) {
|
||||
async exec(cs, {ep, ep2}) {
|
||||
await super.exec(cs, {ep, ep2});
|
||||
|
||||
if (this.data.recognizer.vendor === 'nuance') {
|
||||
@@ -239,13 +223,6 @@ class TaskTranscribe extends SttTask {
|
||||
this.addCustomEventListener(ep, SonioxTranscriptionEvents.Transcription,
|
||||
this._onTranscription.bind(this, cs, ep, channel));
|
||||
break;
|
||||
|
||||
case 'verbio':
|
||||
this.bugname = `${this.bugname_prefix}verbio_transcribe`;
|
||||
this.addCustomEventListener(
|
||||
ep, VerbioTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
|
||||
break;
|
||||
|
||||
case 'cobalt':
|
||||
this.bugname = `${this.bugname_prefix}cobalt_transcribe`;
|
||||
this.addCustomEventListener(ep, CobaltTranscriptionEvents.Transcription,
|
||||
@@ -303,22 +280,6 @@ class TaskTranscribe extends SttTask {
|
||||
this._onVendorConnectFailure.bind(this, cs, ep, channel));
|
||||
break;
|
||||
|
||||
case 'speechmatics':
|
||||
this.bugname = `${this.bugname_prefix}speechmatics_transcribe`;
|
||||
this.addCustomEventListener(
|
||||
ep, SpeechmaticsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
|
||||
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Info,
|
||||
this._onSpeechmaticsInfo.bind(this, cs, ep));
|
||||
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.RecognitionStarted,
|
||||
this._onSpeechmaticsRecognitionStarted.bind(this, cs, ep));
|
||||
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Connect,
|
||||
this._onVendorConnect.bind(this, cs, ep));
|
||||
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.ConnectFailure,
|
||||
this._onVendorConnectFailure.bind(this, cs, ep));
|
||||
this.addCustomEventListener(ep, SpeechmaticsTranscriptionEvents.Error,
|
||||
this._onSpeechmaticsError.bind(this, cs, ep));
|
||||
break;
|
||||
|
||||
default:
|
||||
if (this.vendor.startsWith('custom:')) {
|
||||
this.bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
|
||||
@@ -502,7 +463,6 @@ class TaskTranscribe extends SttTask {
|
||||
if (this.childSpan[channel - 1] && this.childSpan[channel - 1].span) {
|
||||
this.childSpan[channel - 1].span.setAttributes({
|
||||
channel,
|
||||
'stt.label': this.label || 'None',
|
||||
'stt.resolve': 'transcript',
|
||||
'stt.result': JSON.stringify(evt)
|
||||
});
|
||||
@@ -556,8 +516,7 @@ class TaskTranscribe extends SttTask {
|
||||
if (this.childSpan[channel - 1] && this.childSpan[channel - 1].span) {
|
||||
this.childSpan[channel - 1].span.setAttributes({
|
||||
channel,
|
||||
'stt.resolve': 'timeout',
|
||||
'stt.label': this.label || 'None',
|
||||
'stt.resolve': 'timeout'
|
||||
});
|
||||
this.childSpan[channel - 1].span.end();
|
||||
}
|
||||
@@ -574,8 +533,7 @@ class TaskTranscribe extends SttTask {
|
||||
if (this.childSpan[channel - 1] && this.childSpan[channel - 1].span) {
|
||||
this.childSpan[channel - 1].span.setAttributes({
|
||||
channel,
|
||||
'stt.resolve': 'max duration exceeded',
|
||||
'stt.label': this.label || 'None',
|
||||
'stt.resolve': 'max duration exceeded'
|
||||
});
|
||||
this.childSpan[channel - 1].span.end();
|
||||
}
|
||||
@@ -659,8 +617,7 @@ class TaskTranscribe extends SttTask {
|
||||
if (this.childSpan[channel - 1] && this.childSpan[channel - 1].span) {
|
||||
this.childSpan[channel - 1].span.setAttributes({
|
||||
channel,
|
||||
'stt.resolve': 'connection failure',
|
||||
'stt.label': this.label || 'None',
|
||||
'stt.resolve': 'connection failure'
|
||||
});
|
||||
this.childSpan[channel - 1].span.end();
|
||||
}
|
||||
@@ -669,20 +626,6 @@ class TaskTranscribe extends SttTask {
|
||||
}
|
||||
}
|
||||
|
||||
async _onSpeechmaticsRecognitionStarted(_cs, _ep, evt) {
|
||||
this.logger.debug({evt}, 'TaskGather:_onSpeechmaticsRecognitionStarted');
|
||||
}
|
||||
|
||||
async _onSpeechmaticsInfo(_cs, _ep, evt) {
|
||||
this.logger.debug({evt}, 'TaskGather:_onSpeechmaticsInfo');
|
||||
}
|
||||
|
||||
async _onSpeechmaticsErrror(cs, _ep, evt) {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
const {message, ...e} = evt;
|
||||
this._onVendorError(cs, _ep, {error: JSON.stringify(e)});
|
||||
}
|
||||
|
||||
_startAsrTimer(channel) {
|
||||
if (this.vendor === 'deepgram') return; // no need
|
||||
assert(this.isContinuousAsr);
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
const Task = require('./task');
|
||||
const { TaskPreconditions } = require('../utils/constants');
|
||||
const { SpeechCredentialError } = require('../utils/error');
|
||||
|
||||
class TtsTask extends Task {
|
||||
|
||||
@@ -53,8 +52,7 @@ class TtsTask extends Task {
|
||||
|
||||
let credentials = cs.getSpeechCredentials(vendor, 'tts', label);
|
||||
if (!credentials) {
|
||||
throw new SpeechCredentialError(
|
||||
`No text-to-speech service credentials for ${vendor} with labels: ${label} have been configured`);
|
||||
throw new Error(`No text-to-speech service credentials for ${vendor} with labels: ${label} have been configured`);
|
||||
}
|
||||
/* parse Nuance voices into name and model */
|
||||
let model;
|
||||
@@ -83,39 +81,16 @@ class TtsTask extends Task {
|
||||
credentials.optimize_streaming_latency = this.options.optimize_streaming_latency
|
||||
|| credentials.optimize_streaming_latency;
|
||||
voice = this.options.voice_id || voice;
|
||||
} else if (vendor === 'rimelabs') {
|
||||
credentials = credentials || {};
|
||||
credentials.model_id = this.options.model_id || credentials.model_id;
|
||||
} else if (vendor === 'whisper') {
|
||||
credentials = credentials || {};
|
||||
credentials.model_id = this.options.model_id || credentials.model_id;
|
||||
} else if (vendor === 'verbio') {
|
||||
credentials = credentials || {};
|
||||
credentials.engine_version = this.options.engine_version || credentials.engine_version;
|
||||
} else if (vendor === 'playht') {
|
||||
credentials = credentials || {};
|
||||
credentials.voice_engine = this.options.voice_engine || credentials.voice_engine;
|
||||
}
|
||||
|
||||
/**
|
||||
* note on cache_speech_handles. This was found to be risky.
|
||||
* It can cause a crash in the following sequence on a single call:
|
||||
* 1. Stream tts on vendor A with cache_speech_handles=1, then
|
||||
* 2. Stream tts on vendor B with cache_speech_handles=1
|
||||
*
|
||||
* we previously tried to track when vendors were switched and manage the flag accordingly,
|
||||
* but it difficult to track all the scenarios and the benefit (slightly faster start to tts playout)
|
||||
* is probably minimal. DH.
|
||||
*/
|
||||
ep.set({
|
||||
tts_engine: vendor.startsWith('custom:') ? 'custom' : vendor,
|
||||
tts_voice: voice,
|
||||
//cache_speech_handles: !cs.currentTtsVendor || cs.currentTtsVendor === vendor ? 1 : 0,
|
||||
cache_speech_handles: 0,
|
||||
cache_speech_handles: !cs.currentTtsVendor || cs.currentTtsVendor === vendor ? 1 : 0,
|
||||
}).catch((err) => this.logger.info({err}, 'Error setting tts_engine on endpoint'));
|
||||
// set the current vendor on the call session
|
||||
// If vendor is changed from the previous one, then reset the cache_speech_handles flag
|
||||
//cs.currentTtsVendor = vendor;
|
||||
cs.currentTtsVendor = vendor;
|
||||
|
||||
if (!preCache && !this._disableTracing) this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
|
||||
try {
|
||||
@@ -126,7 +101,7 @@ class TtsTask extends Task {
|
||||
vendor,
|
||||
target_sid: cs.callSid
|
||||
}).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
|
||||
throw new SpeechCredentialError('no provisioned speech credentials for TTS');
|
||||
throw new Error('no provisioned speech credentials for TTS');
|
||||
}
|
||||
// synthesize all of the text elements
|
||||
let lastUpdated = false;
|
||||
@@ -141,8 +116,7 @@ class TtsTask extends Task {
|
||||
const {span} = this.startChildSpan('tts-generation', {
|
||||
'tts.vendor': vendor,
|
||||
'tts.language': language,
|
||||
'tts.voice': voice,
|
||||
'tts.label': label || 'None',
|
||||
'tts.voice': voice
|
||||
});
|
||||
this.otelSpan = span;
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ const makeTask = require('../tasks/make_task');
|
||||
const Emitter = require('events');
|
||||
const { normalizeJambones } = require('@jambonz/verb-specifications');
|
||||
const {TaskName} = require('../utils/constants');
|
||||
const assert = require('assert');
|
||||
|
||||
/**
|
||||
* ActionHookDelayProcessor
|
||||
@@ -24,12 +25,10 @@ class ActionHookDelayProcessor extends Emitter {
|
||||
this._active = false;
|
||||
|
||||
const enabled = this.init(opts);
|
||||
if (enabled && this.noResponseTimeout &&
|
||||
(!this.actions || !Array.isArray(this.actions) || this.actions.length === 0)) {
|
||||
if (enabled && (!this.actions || !Array.isArray(this.actions) || this.actions.length === 0)) {
|
||||
throw new Error('ActionHookDelayProcessor: no actions specified');
|
||||
}
|
||||
else if (enabled && this.actions &&
|
||||
this.actions.some((a) => !a.verb || ![TaskName.Say, TaskName.Play].includes(a.verb))) {
|
||||
else if (enabled && this.actions.some((a) => !a.verb || ![TaskName.Say, TaskName.Play].includes(a.verb))) {
|
||||
throw new Error(`ActionHookDelayProcessor: invalid actions specified: ${JSON.stringify(this.actions)}`);
|
||||
}
|
||||
}
|
||||
@@ -52,9 +51,8 @@ class ActionHookDelayProcessor extends Emitter {
|
||||
|
||||
this.actions = opts.actions;
|
||||
this.retries = opts.retries || 0;
|
||||
this.noResponseTimeout = opts.noResponseTimeout;
|
||||
this.noResponseTimeout = opts.noResponseTimeout || 0;
|
||||
this.noResponseGiveUpTimeout = opts.noResponseGiveUpTimeout;
|
||||
this.giveUpActions = opts.giveUpActions;
|
||||
|
||||
// return false if these options actually disable the ahdp
|
||||
return ('enable' in opts && opts.enable === true) ||
|
||||
@@ -68,16 +66,11 @@ class ActionHookDelayProcessor extends Emitter {
|
||||
this.logger.debug('ActionHookDelayProcessor#start: already started due to prior gather which is continuing');
|
||||
return;
|
||||
}
|
||||
assert(!this._noResponseTimer);
|
||||
this._active = true;
|
||||
this._retryCount = 0;
|
||||
if (this.noResponseTimeout > 0) {
|
||||
const timeoutMs = this.noResponseTimeout * 1000;
|
||||
this._noResponseTimer = setTimeout(this._onNoResponseTimer.bind(this), timeoutMs);
|
||||
} else {
|
||||
this.logger.debug(
|
||||
'ActionHookDelayProcessor#start: noResponseTimeout is 0 or undefined hence not calling _onNoResponseTimer'
|
||||
);
|
||||
}
|
||||
const timeoutMs = this.noResponseTimeout === 0 ? 1 : this.noResponseTimeout * 1000;
|
||||
this._noResponseTimer = setTimeout(this._onNoResponseTimer.bind(this), timeoutMs);
|
||||
|
||||
if (this.noResponseGiveUpTimeout > 0) {
|
||||
const timeoutMs = this.noResponseGiveUpTimeout * 1000;
|
||||
@@ -137,9 +130,7 @@ class ActionHookDelayProcessor extends Emitter {
|
||||
this.logger.debug({evt}, 'got playback-start');
|
||||
if (!this._active) {
|
||||
this.logger.info({evt}, 'ActionHookDelayProcessor#_onNoResponseTimer: killing audio immediately');
|
||||
|
||||
/* note: in race condition we may have just hung up and cs.ep cleared */
|
||||
this.ep?.api('uuid_break', this.ep?.uuid)
|
||||
this.ep.api('uuid_break', this.ep.uuid)
|
||||
.catch((err) => this.logger.info(err,
|
||||
'ActionHookDelayProcessor#_onNoResponseTimer Error killing audio'));
|
||||
}
|
||||
@@ -168,14 +159,9 @@ class ActionHookDelayProcessor extends Emitter {
|
||||
|
||||
_onNoResponseGiveUpTimer() {
|
||||
this._active = false;
|
||||
if (!this.giveUpActions) {
|
||||
this.logger.info('ActionHookDelayProcessor#_onNoResponseGiveUpTimer');
|
||||
this.stop().catch((err) => {});
|
||||
this.emit('giveup');
|
||||
} else {
|
||||
this.logger.info('ActionHookDelayProcessor#_onNoResponseGiveUpTimer - giveUpActions');
|
||||
this.emit('giveupWithTasks', this.giveUpActions);
|
||||
}
|
||||
this.logger.info('ActionHookDelayProcessor#_onNoResponseGiveUpTimer');
|
||||
this.stop().catch((err) => {});
|
||||
this.emit('giveup');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
"Leave": "leave",
|
||||
"Lex": "lex",
|
||||
"Listen": "listen",
|
||||
"Llm": "llm",
|
||||
"Message": "message",
|
||||
"Pause": "pause",
|
||||
"Play": "play",
|
||||
@@ -127,14 +126,6 @@
|
||||
"NoSpeechDetected": "azure_transcribe::no_speech_detected",
|
||||
"VadDetected": "azure_transcribe::vad_detected"
|
||||
},
|
||||
"SpeechmaticsTranscriptionEvents": {
|
||||
"Transcription": "speechmatics_transcribe::transcription",
|
||||
"Info": "speechmatics_transcribe::info",
|
||||
"RecognitionStarted": "speechmatics_transcribe::recognition_started",
|
||||
"ConnectFailure": "speechmatics_transcribe::connect_failed",
|
||||
"Connect": "speechmatics_transcribe::connect",
|
||||
"Error": "speechmatics_transcribe::error"
|
||||
},
|
||||
"JambonzTranscriptionEvents": {
|
||||
"Transcription": "jambonz_transcribe::transcription",
|
||||
"ConnectFailure": "jambonz_transcribe::connect_failed",
|
||||
@@ -167,13 +158,6 @@
|
||||
"StandbyEnter": "standby-enter",
|
||||
"StandbyExit": "standby-exit"
|
||||
},
|
||||
"LlmEvents_OpenAI": {
|
||||
"Error": "error",
|
||||
"Connect": "openai_s2s::connect",
|
||||
"ConnectFailure": "openai_s2s::connect_failed",
|
||||
"Disconnect": "openai_s2s::disconnect",
|
||||
"ServerEvent": "openai_s2s::server_event"
|
||||
},
|
||||
"QueueResults": {
|
||||
"Bridged": "bridged",
|
||||
"Error": "error",
|
||||
@@ -200,8 +184,6 @@
|
||||
"dial:confirm",
|
||||
"verb:hook",
|
||||
"verb:status",
|
||||
"llm:event",
|
||||
"llm:tool-call",
|
||||
"jambonz:error"
|
||||
],
|
||||
"RecordState": {
|
||||
|
||||
@@ -91,47 +91,35 @@ const speechMapper = (cred) => {
|
||||
else if ('cobalt' === obj.vendor) {
|
||||
const o = JSON.parse(decrypt(credential));
|
||||
obj.cobalt_server_uri = o.cobalt_server_uri;
|
||||
}
|
||||
else if ('elevenlabs' === obj.vendor) {
|
||||
} else if ('elevenlabs' === obj.vendor) {
|
||||
const o = JSON.parse(decrypt(credential));
|
||||
obj.api_key = o.api_key;
|
||||
obj.model_id = o.model_id;
|
||||
obj.options = o.options;
|
||||
}
|
||||
else if ('playht' === obj.vendor) {
|
||||
} else if ('playht' === obj.vendor) {
|
||||
const o = JSON.parse(decrypt(credential));
|
||||
obj.api_key = o.api_key;
|
||||
obj.user_id = o.user_id;
|
||||
obj.voice_engine = o.voice_engine;
|
||||
obj.options = o.options;
|
||||
}
|
||||
else if ('rimelabs' === obj.vendor) {
|
||||
} else if ('rimelabs' === obj.vendor) {
|
||||
const o = JSON.parse(decrypt(credential));
|
||||
obj.api_key = o.api_key;
|
||||
obj.model_id = o.model_id;
|
||||
obj.options = o.options;
|
||||
}
|
||||
else if ('assemblyai' === obj.vendor) {
|
||||
} else if ('assemblyai' === obj.vendor) {
|
||||
const o = JSON.parse(decrypt(credential));
|
||||
obj.api_key = o.api_key;
|
||||
}
|
||||
else if ('whisper' === obj.vendor) {
|
||||
} else if ('whisper' === obj.vendor) {
|
||||
const o = JSON.parse(decrypt(credential));
|
||||
obj.api_key = o.api_key;
|
||||
obj.model_id = o.model_id;
|
||||
}
|
||||
else if ('verbio' === obj.vendor) {
|
||||
} else if ('verbio' === obj.vendor) {
|
||||
const o = JSON.parse(decrypt(credential));
|
||||
obj.client_id = o.client_id;
|
||||
obj.client_secret = o.client_secret;
|
||||
obj.engine_version = o.engine_version;
|
||||
}
|
||||
else if ('speechmatics' === obj.vendor) {
|
||||
const o = JSON.parse(decrypt(credential));
|
||||
obj.api_key = o.api_key;
|
||||
obj.speechmatics_stt_uri = o.speechmatics_stt_uri;
|
||||
}
|
||||
else if (obj.vendor.startsWith('custom:')) {
|
||||
} else if (obj.vendor.startsWith('custom:')) {
|
||||
const o = JSON.parse(decrypt(credential));
|
||||
obj.auth_token = o.auth_token;
|
||||
obj.custom_stt_url = o.custom_stt_url;
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
class SpeechCredentialError extends Error {
|
||||
constructor(msg) {
|
||||
super(msg);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
SpeechCredentialError
|
||||
};
|
||||
@@ -1,5 +1,5 @@
|
||||
const Mrf = require('drachtio-fsmrf');
|
||||
const os = require('os');
|
||||
const ip = require('ip');
|
||||
const {
|
||||
JAMBONES_MYSQL_HOST,
|
||||
JAMBONES_MYSQL_USER,
|
||||
@@ -18,19 +18,6 @@ const {
|
||||
const Registrar = require('@jambonz/mw-registrar');
|
||||
const assert = require('assert');
|
||||
|
||||
function getLocalIp() {
|
||||
const interfaces = os.networkInterfaces();
|
||||
for (const interfaceName in interfaces) {
|
||||
const interface = interfaces[interfaceName];
|
||||
for (const iface of interface) {
|
||||
if (iface.family === 'IPv4' && !iface.internal) {
|
||||
return iface.address;
|
||||
}
|
||||
}
|
||||
}
|
||||
return '127.0.0.1'; // Fallback to localhost if no suitable interface found
|
||||
}
|
||||
|
||||
function initMS(logger, wrapper, ms) {
|
||||
Object.assign(wrapper, {ms, active: true, connects: 1});
|
||||
logger.info(`connected to freeswitch at ${ms.address}`);
|
||||
@@ -152,8 +139,7 @@ function installSrfLocals(srf, logger) {
|
||||
lookupAccountBySid,
|
||||
lookupAccountCapacitiesBySid,
|
||||
lookupSmppGateways,
|
||||
lookupClientByAccountAndUsername,
|
||||
lookupSystemInformation
|
||||
lookupClientByAccountAndUsername
|
||||
} = require('@jambonz/db-helpers')({
|
||||
host: JAMBONES_MYSQL_HOST,
|
||||
user: JAMBONES_MYSQL_USER,
|
||||
@@ -208,8 +194,8 @@ function installSrfLocals(srf, logger) {
|
||||
|
||||
let localIp;
|
||||
try {
|
||||
// Either use the configured IP address or discover it
|
||||
localIp = HTTP_IP || getLocalIp();
|
||||
// Either use the configured IP address or call ip.address() to find it
|
||||
localIp = HTTP_IP || ip.address();
|
||||
} catch (err) {
|
||||
logger.error({err}, 'installSrfLocals - error detecting local ipv4 address');
|
||||
}
|
||||
@@ -229,7 +215,6 @@ function installSrfLocals(srf, logger) {
|
||||
lookupAccountCapacitiesBySid,
|
||||
lookupSmppGateways,
|
||||
lookupClientByAccountAndUsername,
|
||||
lookupSystemInformation,
|
||||
updateCallStatus,
|
||||
retrieveCall,
|
||||
listCalls,
|
||||
|
||||
@@ -213,8 +213,6 @@ class SingleDialer extends Emitter {
|
||||
},
|
||||
cbProvisional: (prov) => {
|
||||
const status = {sipStatus: prov.status, sipReason: prov.reason};
|
||||
// Update call-id for sbc outbound INVITE
|
||||
this.callInfo.sbcCallid = prov.get('X-CID');
|
||||
if ([180, 183].includes(prov.status) && prov.body) {
|
||||
if (status.callStatus !== CallStatus.EarlyMedia) {
|
||||
status.callStatus = CallStatus.EarlyMedia;
|
||||
@@ -449,8 +447,6 @@ class SingleDialer extends Emitter {
|
||||
});
|
||||
|
||||
cs.req = this.req;
|
||||
// fixed hangup an adulting session does not send status callback Completed
|
||||
cs.wrapDialog(this.dlg);
|
||||
cs.exec().catch((err) => newLogger.error({err}, 'doAdulting: error executing session'));
|
||||
return cs;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
const {TaskName} = require('./constants.json');
|
||||
const {
|
||||
TaskName,
|
||||
} = require('./constants.json');
|
||||
|
||||
const stickyVars = {
|
||||
google: [
|
||||
'GOOGLE_SPEECH_HINTS',
|
||||
@@ -48,13 +51,7 @@ const stickyVars = {
|
||||
aws: [
|
||||
'AWS_VOCABULARY_NAME',
|
||||
'AWS_VOCABULARY_FILTER_METHOD',
|
||||
'AWS_VOCABULARY_FILTER_NAME',
|
||||
'AWS_LANGUAGE_MODEL_NAME',
|
||||
'AWS_ACCESS_KEY_ID',
|
||||
'AWS_SECRET_ACCESS_KEY',
|
||||
'AWS_REGION',
|
||||
'AWS_SECURITY_TOKEN',
|
||||
'AWS_PII_ENTITY_TYPES',
|
||||
'AWS_VOCABULARY_FILTER_NAME'
|
||||
],
|
||||
nuance: [
|
||||
'NUANCE_ACCESS_TOKEN',
|
||||
@@ -103,12 +100,6 @@ const stickyVars = {
|
||||
assemblyai: [
|
||||
'ASSEMBLYAI_API_KEY',
|
||||
'ASSEMBLYAI_WORD_BOOST'
|
||||
],
|
||||
speechmatics: [
|
||||
'SPEECHMATICS_API_KEY',
|
||||
'SPEECHMATICS_HOST',
|
||||
'SPEECHMATICS_PATH',
|
||||
'SPEECHMATICS_SPEECH_HINTS',
|
||||
]
|
||||
};
|
||||
|
||||
@@ -161,12 +152,7 @@ const selectDefaultDeepgramModel = (task, language) => {
|
||||
|
||||
const optimalGoogleModels = {
|
||||
'v1' : {
|
||||
'en-IN':['telephony', 'telephony'],
|
||||
'es-DO':['default', 'default'],
|
||||
'es-MX':['default', 'default'],
|
||||
'en-AU':['telephony', 'telephony'],
|
||||
'en-GB':['telephony', 'telephony'],
|
||||
'en-NZ':['telephony', 'telephony']
|
||||
'en-IN':['telephony', 'latest_long']
|
||||
},
|
||||
'v2' : {
|
||||
'en-IN':['telephony', 'long']
|
||||
@@ -455,41 +441,16 @@ const normalizeMicrosoft = (evt, channel, language, punctuation = true) => {
|
||||
|
||||
const normalizeAws = (evt, channel, language) => {
|
||||
const copy = JSON.parse(JSON.stringify(evt));
|
||||
const isGrpcPayload = Array.isArray(evt);
|
||||
if (isGrpcPayload) {
|
||||
/* legacy grpc api */
|
||||
return {
|
||||
language_code: language,
|
||||
channel_tag: channel,
|
||||
is_final: evt[0].is_final,
|
||||
alternatives: evt[0].alternatives,
|
||||
vendor: {
|
||||
name: 'aws',
|
||||
evt: copy
|
||||
}
|
||||
};
|
||||
}
|
||||
else {
|
||||
/* websocket api */
|
||||
const alternatives = evt.Transcript?.Results[0]?.Alternatives.map((alt) => {
|
||||
const items = alt.Items.filter((item) => item.Type === 'pronunciation' && 'Confidence' in item);
|
||||
const confidence = items.reduce((acc, item) => acc + item.Confidence, 0) / items.length;
|
||||
return {
|
||||
transcript: alt.Transcript,
|
||||
confidence
|
||||
};
|
||||
});
|
||||
return {
|
||||
language_code: language,
|
||||
channel_tag: channel,
|
||||
is_final: evt.Transcript?.Results[0].IsPartial === false,
|
||||
alternatives,
|
||||
vendor: {
|
||||
name: 'aws',
|
||||
evt: copy
|
||||
}
|
||||
};
|
||||
}
|
||||
return {
|
||||
language_code: language,
|
||||
channel_tag: channel,
|
||||
is_final: evt[0].is_final,
|
||||
alternatives: evt[0].alternatives,
|
||||
vendor: {
|
||||
name: 'aws',
|
||||
evt: copy
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
const normalizeAssemblyAi = (evt, channel, language) => {
|
||||
@@ -505,37 +466,12 @@ const normalizeAssemblyAi = (evt, channel, language) => {
|
||||
}
|
||||
],
|
||||
vendor: {
|
||||
name: 'assemblyai',
|
||||
name: 'ASSEMBLYAI',
|
||||
evt: copy
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
const normalizeSpeechmatics = (evt, channel, language) => {
|
||||
const copy = JSON.parse(JSON.stringify(evt));
|
||||
const is_final = evt.message === 'AddTranscript';
|
||||
const words = evt.results?.filter((r) => r.type === 'word') || [];
|
||||
const confidence = words.length > 0 ?
|
||||
words.reduce((acc, word) => acc + word.alternatives[0].confidence, 0) / words.length :
|
||||
0;
|
||||
|
||||
const alternative = {
|
||||
confidence,
|
||||
transcript: evt.metadata?.transcript
|
||||
};
|
||||
const obj = {
|
||||
language_code: language,
|
||||
channel_tag: channel,
|
||||
is_final,
|
||||
alternatives: [alternative],
|
||||
vendor: {
|
||||
name: 'speechmatics',
|
||||
evt: copy
|
||||
}
|
||||
};
|
||||
return obj;
|
||||
};
|
||||
|
||||
module.exports = (logger) => {
|
||||
const normalizeTranscription = (evt, vendor, channel, language, shortUtterance, punctuation) => {
|
||||
|
||||
@@ -563,8 +499,6 @@ module.exports = (logger) => {
|
||||
return normalizeAssemblyAi(evt, channel, language, shortUtterance);
|
||||
case 'verbio':
|
||||
return normalizeVerbio(evt, channel, language);
|
||||
case 'speechmatics':
|
||||
return normalizeSpeechmatics(evt, channel, language);
|
||||
default:
|
||||
if (vendor.startsWith('custom:')) {
|
||||
return normalizeCustom(evt, channel, language, vendor);
|
||||
@@ -638,29 +572,17 @@ module.exports = (logger) => {
|
||||
};
|
||||
}
|
||||
else if (['aws', 'polly'].includes(vendor)) {
|
||||
const {awsOptions = {}} = rOpts;
|
||||
const vocabularyName = awsOptions.vocabularyName || rOpts.vocabularyName;
|
||||
const vocabularyFilterName = awsOptions.vocabularyFilterName || rOpts.vocabularyFilterName;
|
||||
const filterMethod = awsOptions.vocabularyFilterMethod || rOpts.filterMethod;
|
||||
opts = {
|
||||
...opts,
|
||||
...(vocabularyName && {AWS_VOCABULARY_NAME: vocabularyName}),
|
||||
...(vocabularyFilterName && {AWS_VOCABULARY_FILTER_NAME: vocabularyFilterName}),
|
||||
...(filterMethod && {AWS_VOCABULARY_FILTER_METHOD: filterMethod}),
|
||||
...(rOpts.vocabularyName && {AWS_VOCABULARY_NAME: rOpts.vocabularyName}),
|
||||
...(rOpts.vocabularyFilterName && {AWS_VOCABULARY_FILTER_NAME: rOpts.vocabularyFilterName}),
|
||||
...(rOpts.filterMethod && {AWS_VOCABULARY_FILTER_METHOD: rOpts.filterMethod}),
|
||||
...(sttCredentials && {
|
||||
AWS_ACCESS_KEY_ID: sttCredentials.accessKeyId,
|
||||
AWS_SECRET_ACCESS_KEY: sttCredentials.secretAccessKey,
|
||||
...(sttCredentials.accessKeyId && {AWS_ACCESS_KEY_ID: sttCredentials.accessKeyId}),
|
||||
...(sttCredentials.secretAccessKey && {AWS_SECRET_ACCESS_KEY: sttCredentials.secretAccessKey}),
|
||||
AWS_REGION: sttCredentials.region,
|
||||
AWS_SECURITY_TOKEN: sttCredentials.securityToken
|
||||
...(sttCredentials.sessionToken && {AWS_SESSION_TOKEN: sttCredentials.sessionToken}),
|
||||
}),
|
||||
...(awsOptions.accessKey && {AWS_ACCESS_KEY_ID: awsOptions.accessKey}),
|
||||
...(awsOptions.secretKey && {AWS_SECRET_ACCESS_KEY: awsOptions.secretKey}),
|
||||
...(awsOptions.region && {AWS_REGION: awsOptions.region}),
|
||||
...(awsOptions.securityToken && {AWS_SECURITY_TOKEN: awsOptions.securityToken}),
|
||||
...(awsOptions.languageModelName && {AWS_LANGUAGE_MODEL_NAME: awsOptions.languageModelName}),
|
||||
...(awsOptions.piiEntityTypes?.length && {AWS_PII_ENTITY_TYPES: awsOptions.piiEntityTypes.join(',')}),
|
||||
...(awsOptions.piiIdentifyEntities && {AWS_PII_IDENTIFY_ENTITIES: true}),
|
||||
...(awsOptions.languageModelName && {AWS_LANGUAGE_MODEL_NAME: awsOptions.languageModelName}),
|
||||
};
|
||||
}
|
||||
else if ('microsoft' === vendor) {
|
||||
@@ -906,8 +828,7 @@ module.exports = (logger) => {
|
||||
...(cobaltOptions.enableConfusionNetwork && {COBALT_ENABLE_CONFUSION_NETWORK: 1}),
|
||||
...(cobaltOptions.compiledContextData && {COBALT_COMPILED_CONTEXT_DATA: cobaltOptions.compiledContextData}),
|
||||
};
|
||||
}
|
||||
else if ('assemblyai' === vendor) {
|
||||
} else if ('assemblyai' === vendor) {
|
||||
opts = {
|
||||
...opts,
|
||||
...(sttCredentials.api_key) &&
|
||||
@@ -915,8 +836,7 @@ module.exports = (logger) => {
|
||||
...(rOpts.hints?.length > 0 &&
|
||||
{ASSEMBLYAI_WORD_BOOST: JSON.stringify(rOpts.hints)})
|
||||
};
|
||||
}
|
||||
else if ('verbio' === vendor) {
|
||||
} else if ('verbio' === vendor) {
|
||||
const {verbioOptions = {}} = rOpts;
|
||||
opts = {
|
||||
...opts,
|
||||
@@ -935,16 +855,7 @@ module.exports = (logger) => {
|
||||
...(verbioOptions.speech_incomplete_timeout &&
|
||||
{VERBIO_SPEECH_INCOMPLETE_TIMEOUT: verbioOptions.speech_incomplete_timeout}),
|
||||
};
|
||||
}
|
||||
else if ('speechmatics' === vendor) {
|
||||
opts = {
|
||||
...opts,
|
||||
...(sttCredentials.api_key) && {SPEECHMATICS_API_KEY: sttCredentials.api_key},
|
||||
...(sttCredentials.speechmatics_stt_uri) && {SPEECHMATICS_HOST: sttCredentials.speechmatics_stt_uri},
|
||||
...(rOpts.hints?.length > 0 && {SPEECHMATICS_SPEECH_HINTS: rOpts.hints.join(',')}),
|
||||
};
|
||||
}
|
||||
else if (vendor.startsWith('custom:')) {
|
||||
} else if (vendor.startsWith('custom:')) {
|
||||
let {options = {}} = rOpts.customOptions || {};
|
||||
const {sampleRate} = rOpts.customOptions || {};
|
||||
const {auth_token, custom_stt_url} = sttCredentials;
|
||||
@@ -1012,6 +923,6 @@ module.exports = (logger) => {
|
||||
setChannelVarsForStt,
|
||||
setSpeechCredentialsAtRuntime,
|
||||
compileSonioxTranscripts,
|
||||
consolidateTranscripts,
|
||||
consolidateTranscripts
|
||||
};
|
||||
};
|
||||
|
||||
@@ -44,7 +44,7 @@ class WsRequestor extends BaseRequestor {
|
||||
async request(type, hook, params, httpHeaders = {}) {
|
||||
assert(HookMsgTypes.includes(type));
|
||||
const url = hook.url || hook;
|
||||
const wantsAck = !['call:status', 'verb:status', 'jambonz:error', 'llm:event', 'llm:tool-call'].includes(type);
|
||||
const wantsAck = !['call:status', 'verb:status', 'jambonz:error'].includes(type);
|
||||
|
||||
if (this.maliciousClient) {
|
||||
this.logger.info({url: this.url}, 'WsRequestor:request - discarding msg to malicious client');
|
||||
@@ -132,7 +132,7 @@ class WsRequestor extends BaseRequestor {
|
||||
type,
|
||||
msgid,
|
||||
call_sid: this.call_sid,
|
||||
hook: ['verb:hook', 'session:redirect', 'llm:event', 'llm:tool-call'].includes(type) ? url : undefined,
|
||||
hook: ['verb:hook', 'session:redirect'].includes(type) ? url : undefined,
|
||||
data: {...payload},
|
||||
...b3
|
||||
};
|
||||
@@ -392,9 +392,8 @@ class WsRequestor extends BaseRequestor {
|
||||
/* messages must be JSON format */
|
||||
try {
|
||||
const obj = JSON.parse(content);
|
||||
this.logger.debug({obj}, 'WsRequestor:_onMessage - received message');
|
||||
//const {type, msgid, command, call_sid = this.call_sid, queueCommand = false, data} = obj;
|
||||
const {type, msgid, command, queueCommand = false, tool_call_id, data} = obj;
|
||||
const {type, msgid, command, queueCommand = false, data} = obj;
|
||||
const call_sid = obj.callSid || this.call_sid;
|
||||
|
||||
//this.logger.debug({obj}, 'WsRequestor:request websocket: received');
|
||||
@@ -408,8 +407,8 @@ class WsRequestor extends BaseRequestor {
|
||||
|
||||
case 'command':
|
||||
assert.ok(command, 'command property not supplied');
|
||||
assert.ok(data || command === 'llm:tool-output', 'data property not supplied');
|
||||
this._recvCommand(msgid, command, call_sid, queueCommand, tool_call_id, data);
|
||||
assert.ok(data, 'data property not supplied');
|
||||
this._recvCommand(msgid, command, call_sid, queueCommand, data);
|
||||
break;
|
||||
|
||||
default:
|
||||
@@ -433,10 +432,10 @@ class WsRequestor extends BaseRequestor {
|
||||
success && success(data);
|
||||
}
|
||||
|
||||
_recvCommand(msgid, command, call_sid, queueCommand, tool_call_id, data) {
|
||||
_recvCommand(msgid, command, call_sid, queueCommand, data) {
|
||||
// TODO: validate command
|
||||
this.logger.debug({msgid, command, call_sid, queueCommand, data}, 'received command');
|
||||
this.emit('command', {msgid, command, call_sid, queueCommand, tool_call_id, data});
|
||||
this.emit('command', {msgid, command, call_sid, queueCommand, data});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
782
package-lock.json
generated
782
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
11
package.json
11
package.json
@@ -31,10 +31,10 @@
|
||||
"@jambonz/http-health-check": "^0.0.1",
|
||||
"@jambonz/mw-registrar": "^0.2.7",
|
||||
"@jambonz/realtimedb-helpers": "^0.8.8",
|
||||
"@jambonz/speech-utils": "^0.1.20",
|
||||
"@jambonz/speech-utils": "^0.1.15",
|
||||
"@jambonz/stats-collector": "^0.1.10",
|
||||
"@jambonz/verb-specifications": "^0.0.91",
|
||||
"@jambonz/time-series": "^0.2.13",
|
||||
"@jambonz/time-series": "^0.2.9",
|
||||
"@jambonz/verb-specifications": "^0.0.76",
|
||||
"@opentelemetry/api": "^1.8.0",
|
||||
"@opentelemetry/exporter-jaeger": "^1.23.0",
|
||||
"@opentelemetry/exporter-trace-otlp-http": "^0.50.0",
|
||||
@@ -49,8 +49,9 @@
|
||||
"deepcopy": "^2.1.0",
|
||||
"drachtio-fsmrf": "^3.0.45",
|
||||
"drachtio-srf": "^4.5.35",
|
||||
"express": "^4.19.2",
|
||||
"express": "^4.21.0",
|
||||
"express-validator": "^7.0.1",
|
||||
"ip": "^2.0.1",
|
||||
"moment": "^2.30.1",
|
||||
"parse-url": "^9.2.0",
|
||||
"pino": "^8.20.0",
|
||||
@@ -60,7 +61,7 @@
|
||||
"short-uuid": "^5.1.0",
|
||||
"sinon": "^17.0.1",
|
||||
"to-snake-case": "^1.0.0",
|
||||
"undici": "^6.20.0",
|
||||
"undici": "^6.19.4",
|
||||
"uuid-random": "^1.3.2",
|
||||
"verify-aws-sns-signature": "^0.1.0",
|
||||
"ws": "^8.18.0",
|
||||
|
||||
@@ -222,62 +222,3 @@ test('test create-call app_json', async(t) => {
|
||||
t.error(err);
|
||||
}
|
||||
});
|
||||
|
||||
test('test create-call timeLimit', async(t) => {
|
||||
clearModule.all();
|
||||
const {srf, disconnect} = require('../app');
|
||||
|
||||
try {
|
||||
await connect(srf);
|
||||
|
||||
|
||||
// GIVEN
|
||||
let from = 'create-call-app-json';
|
||||
let account_sid = 'bb845d4b-83a9-4cde-a6e9-50f3743bab3f';
|
||||
|
||||
// Give UAS app time to come up
|
||||
const p = sippUac('uas.xml', '172.38.0.10', from);
|
||||
await waitFor(1000);
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
const app_json = `[
|
||||
{
|
||||
"verb": "pause",
|
||||
"length": 7
|
||||
}
|
||||
]`;
|
||||
|
||||
const post = bent('http://127.0.0.1:3000/', 'POST', 'json', 201);
|
||||
post('v1/createCall', {
|
||||
'account_sid':account_sid,
|
||||
"call_hook": {
|
||||
"url": "http://127.0.0.1:3100/",
|
||||
"method": "POST",
|
||||
"username": "username",
|
||||
"password": "password"
|
||||
},
|
||||
app_json,
|
||||
"from": from,
|
||||
"to": {
|
||||
"type": "phone",
|
||||
"number": "15583084809"
|
||||
},
|
||||
"timeLimit": 1,
|
||||
"speech_recognizer_vendor": "google",
|
||||
"speech_recognizer_language": "en"
|
||||
});
|
||||
|
||||
//THEN
|
||||
await p;
|
||||
const endTime = Date.now();
|
||||
|
||||
t.ok(endTime - startTime < 2000, 'create-call: timeLimit is respected');
|
||||
|
||||
disconnect();
|
||||
} catch (err) {
|
||||
console.log(`error received: ${err}`);
|
||||
disconnect();
|
||||
t.error(err);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
/* SQLEditor (MySQL (2))*/
|
||||
|
||||
SET FOREIGN_KEY_CHECKS=0;
|
||||
|
||||
DROP TABLE IF EXISTS account_static_ips;
|
||||
@@ -54,8 +53,6 @@ DROP TABLE IF EXISTS signup_history;
|
||||
|
||||
DROP TABLE IF EXISTS smpp_addresses;
|
||||
|
||||
DROP TABLE IF EXISTS google_custom_voices;
|
||||
|
||||
DROP TABLE IF EXISTS speech_credentials;
|
||||
|
||||
DROP TABLE IF EXISTS system_information;
|
||||
@@ -139,9 +136,6 @@ account_sid CHAR(36) NOT NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT 1,
|
||||
username VARCHAR(64),
|
||||
password VARCHAR(1024),
|
||||
allow_direct_app_calling BOOLEAN NOT NULL DEFAULT 1,
|
||||
allow_direct_queue_calling BOOLEAN NOT NULL DEFAULT 1,
|
||||
allow_direct_user_calling BOOLEAN NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY (client_sid)
|
||||
);
|
||||
|
||||
@@ -344,23 +338,11 @@ label VARCHAR(64),
|
||||
PRIMARY KEY (speech_credential_sid)
|
||||
);
|
||||
|
||||
CREATE TABLE google_custom_voices
|
||||
(
|
||||
google_custom_voice_sid CHAR(36) NOT NULL UNIQUE ,
|
||||
speech_credential_sid CHAR(36) NOT NULL,
|
||||
model VARCHAR(512) NOT NULL,
|
||||
reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
|
||||
name VARCHAR(64) NOT NULL,
|
||||
PRIMARY KEY (google_custom_voice_sid)
|
||||
);
|
||||
|
||||
CREATE TABLE system_information
|
||||
(
|
||||
domain_name VARCHAR(255),
|
||||
sip_domain_name VARCHAR(255),
|
||||
monitoring_domain_name VARCHAR(255),
|
||||
private_network_cidr VARCHAR(8192),
|
||||
log_level ENUM('info', 'debug') NOT NULL DEFAULT 'info'
|
||||
monitoring_domain_name VARCHAR(255)
|
||||
);
|
||||
|
||||
CREATE TABLE users
|
||||
@@ -455,14 +437,11 @@ CREATE TABLE sip_gateways
|
||||
sip_gateway_sid CHAR(36),
|
||||
ipv4 VARCHAR(128) NOT NULL COMMENT 'ip address or DNS name of the gateway. For gateways providing inbound calling service, ip address is required.',
|
||||
netmask INTEGER NOT NULL DEFAULT 32,
|
||||
port INTEGER COMMENT 'sip signaling port',
|
||||
port INTEGER NOT NULL DEFAULT 5060 COMMENT 'sip signaling port',
|
||||
inbound BOOLEAN NOT NULL COMMENT 'if true, whitelist this IP to allow inbound calls from the gateway',
|
||||
outbound BOOLEAN NOT NULL COMMENT 'if true, include in least-cost routing when placing calls to the PSTN',
|
||||
voip_carrier_sid CHAR(36) NOT NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT 1,
|
||||
send_options_ping BOOLEAN NOT NULL DEFAULT 0,
|
||||
use_sips_scheme BOOLEAN NOT NULL DEFAULT 0,
|
||||
pad_crypto BOOLEAN NOT NULL DEFAULT 0,
|
||||
protocol ENUM('udp','tcp','tls', 'tls/srtp') DEFAULT 'udp' COMMENT 'Outbound call protocol',
|
||||
PRIMARY KEY (sip_gateway_sid)
|
||||
) COMMENT='A whitelisted sip gateway used for origination/termination';
|
||||
@@ -499,19 +478,11 @@ messaging_hook_sid CHAR(36) COMMENT 'webhook to call for inbound SMS/MMS ',
|
||||
app_json TEXT,
|
||||
speech_synthesis_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
|
||||
speech_synthesis_language VARCHAR(12) NOT NULL DEFAULT 'en-US',
|
||||
speech_synthesis_voice VARCHAR(256),
|
||||
speech_synthesis_voice VARCHAR(64),
|
||||
speech_synthesis_label VARCHAR(64),
|
||||
speech_recognizer_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
|
||||
speech_recognizer_language VARCHAR(64) NOT NULL DEFAULT 'en-US',
|
||||
speech_recognizer_label VARCHAR(64),
|
||||
use_for_fallback_speech BOOLEAN DEFAULT false,
|
||||
fallback_speech_synthesis_vendor VARCHAR(64),
|
||||
fallback_speech_synthesis_language VARCHAR(12),
|
||||
fallback_speech_synthesis_voice VARCHAR(256),
|
||||
fallback_speech_synthesis_label VARCHAR(64),
|
||||
fallback_speech_recognizer_vendor VARCHAR(64),
|
||||
fallback_speech_recognizer_language VARCHAR(64),
|
||||
fallback_speech_recognizer_label VARCHAR(64),
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
record_all_calls BOOLEAN NOT NULL DEFAULT false,
|
||||
PRIMARY KEY (application_sid)
|
||||
@@ -554,7 +525,6 @@ siprec_hook_sid CHAR(36),
|
||||
record_all_calls BOOLEAN NOT NULL DEFAULT false,
|
||||
record_format VARCHAR(16) NOT NULL DEFAULT 'mp3',
|
||||
bucket_credential VARCHAR(8192) COMMENT 'credential used to authenticate with storage service',
|
||||
enable_debug_log BOOLEAN NOT NULL DEFAULT false,
|
||||
PRIMARY KEY (account_sid)
|
||||
) COMMENT='An enterprise that uses the platform for comm services';
|
||||
|
||||
@@ -649,10 +619,6 @@ ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (ser
|
||||
CREATE INDEX account_sid_idx ON speech_credentials (account_sid);
|
||||
ALTER TABLE speech_credentials ADD FOREIGN KEY account_sid_idxfk_8 (account_sid) REFERENCES accounts (account_sid);
|
||||
|
||||
CREATE INDEX google_custom_voice_sid_idx ON google_custom_voices (google_custom_voice_sid);
|
||||
CREATE INDEX speech_credential_sid_idx ON google_custom_voices (speech_credential_sid);
|
||||
ALTER TABLE google_custom_voices ADD FOREIGN KEY speech_credential_sid_idxfk (speech_credential_sid) REFERENCES speech_credentials (speech_credential_sid) ON DELETE CASCADE;
|
||||
|
||||
CREATE INDEX user_sid_idx ON users (user_sid);
|
||||
CREATE INDEX email_idx ON users (email);
|
||||
CREATE INDEX phone_idx ON users (phone);
|
||||
@@ -738,5 +704,4 @@ ALTER TABLE accounts ADD FOREIGN KEY queue_event_hook_sid_idxfk (queue_event_hoo
|
||||
ALTER TABLE accounts ADD FOREIGN KEY device_calling_application_sid_idxfk (device_calling_application_sid) REFERENCES applications (application_sid);
|
||||
|
||||
ALTER TABLE accounts ADD FOREIGN KEY siprec_hook_sid_idxfk (siprec_hook_sid) REFERENCES applications (application_sid);
|
||||
|
||||
SET FOREIGN_KEY_CHECKS=1;
|
||||
SET FOREIGN_KEY_CHECKS=1;
|
||||
Reference in New Issue
Block a user