Compare commits

...

54 Commits

Author SHA1 Message Date
Dave Horton
119a49725a avoid a pre-caching attempt if synth settings change 2024-01-13 12:21:54 -05:00
Dave Horton
5b1c2d3052 more fixes 2024-01-12 16:22:23 -05:00
Dave Horton
21f03dd13a fix for establishing vendor etc 2024-01-12 16:19:59 -05:00
Dave Horton
c2eef50512 wip 2024-01-12 16:13:36 -05:00
dependabot[bot]
d3d494191f Bump follow-redirects from 1.15.2 to 1.15.4 (#603)
Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.2 to 1.15.4.
- [Release notes](https://github.com/follow-redirects/follow-redirects/releases)
- [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.2...v1.15.4)

---
updated-dependencies:
- dependency-name: follow-redirects
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2024-01-10 09:24:36 -05:00
Paulo Telles
859e816a8e issue #605 (#606)
Co-authored-by: p.souza <p.souza@cognigy.com>
2024-01-10 09:23:09 -05:00
Hoan Luu Huu
29bbcf1be0 add user-agent to http and ws requestor (#602)
* add user-agent to http and ws requestor

* wip

* fix review comment
2024-01-10 08:54:46 -05:00
Dave Horton
6f6d7a06b0 Revert "Fix/dual legs transcribe race condition (#593)" (#600)
This reverts commit 9d70ed96a1.
2024-01-09 08:23:13 -05:00
Anton Voylenko
a2ba80a9a3 Snake case customer data for refer (#598)
* update envs

* fix refer customer data

* use data from function
2024-01-08 19:15:08 -05:00
Hoan Luu Huu
9d70ed96a1 Fix/dual legs transcribe race condition (#593)
* fs only stop one of bugname when transcribe is used for dual legs

* wip

* fix review comment

* wip

* wip
2024-01-06 19:12:51 -05:00
Hoan Luu Huu
8173a306f7 fix stt default vendor cannot be mapped to correct value (#588) 2024-01-04 07:34:30 -05:00
Hoan Luu Huu
2e69630544 fix siprec to remap sdp base on participant label (#587)
* fix siprec to remap sdp base on participant label

* fix
2024-01-03 11:10:31 -05:00
Hoan Luu Huu
15829139c1 fix hangup headers (#583)
* fix hangup headers

* no need for callback

* fix test failure

---------

Co-authored-by: Dave Horton <daveh@beachdognet.com>
2023-12-28 14:59:59 -05:00
Dave Horton
2c48083c26 fix to be more precise about removing custom event handlers so that w… (#580)
* fix to be more precise about removing custom event handlers so that when we stop a gather we dont also inadvertently stop a background transcribe as well

* test fixes

* fix: endpointing=false was being ignored for Deepgram
2023-12-28 11:00:27 -05:00
Hoan Luu Huu
9d8291f892 Transcribe background task (#576)
* first draft

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* update verb-specification

* fix comment reviews

* provide bugname when stopping transcription, otherwise it will continue

---------

Co-authored-by: Dave Horton <daveh@beachdognet.com>
2023-12-26 21:50:51 -05:00
Hoan Luu Huu
3e8474867f support deepgram (#579)
* support deepgram

* update speech utils
2023-12-26 07:46:35 -05:00
Hoan Luu Huu
9eb315ecd6 fix config and stt task for altLanguages (#575)
* fix config and stt task for altLanguages

* clear freeswitch channel var when altLanguages is empty list
2023-12-25 22:21:34 -05:00
Hoan Luu Huu
2ec1460b4e nuance transcribe should have utteranceDetectionMode=multiple (#574)
* nuance transcribe should have utteranceDetectionMode=multiple

* nuance transcribe should have utteranceDetectionMode=multiple
2023-12-20 18:44:08 -05:00
Hoan Luu Huu
e30782ea7b fix riva transcribe issue (#570) 2023-12-20 10:25:17 -05:00
Hoan Luu Huu
83c1c07eb0 fix enqueue waithook on ws hold the session (#572)
* fix enqueue waithook on ws hold the session

* wip
2023-12-19 09:42:20 -05:00
Dave Horton
47fbc1a4a4 allow custom speech with no auth token (#571) 2023-12-18 14:51:34 -05:00
Dave Horton
7474a359a4 update to drachtio-fsmrf@3.0.33 2023-12-18 14:28:25 -05:00
Hoan Luu Huu
30977b309c punctuation for microsoft (#566)
* punctuation for microsoft

* wip
2023-12-18 08:38:05 -05:00
Hoan Luu Huu
bcb4bf43bf fix altLanguages (#567)
* fix altLanguages

* adding testcase
2023-12-16 08:35:09 -05:00
Dave Horton
077460d0e2 Feat/multiset envs (#569)
* update to drachtio-fsmrf@3.0.31

* fix prev commits
2023-12-15 15:39:15 -05:00
Matt Hertogs
6629b45671 Fix typo in contributing.md (#555) 2023-12-11 08:37:36 -05:00
Hoan Luu Huu
353a9c1917 Feat/complete adult call (#563)
* feat client direct call from registration webhook (#562)

* feat client direct call from registration webhook

* feat client direct call from registration webhook

* wip

* fix cannot lcc complete adult call

* wip

* wip
2023-12-11 08:37:01 -05:00
Hoan Luu Huu
230fe9ea11 feat client direct call from registration webhook (#562)
* feat client direct call from registration webhook

* feat client direct call from registration webhook

* wip
2023-12-11 08:36:19 -05:00
Dave Horton
bb81f9f3da minor logging 2023-12-07 13:00:25 -05:00
Hoan Luu Huu
a7673c1819 update speech utils 0.0.31 (#560) 2023-12-05 20:47:16 -05:00
Hoan Luu Huu
59248c7638 fix config verb can override recognizer *Options (#558)
* fix config verb can override recognizer *Options

* wip

* clean up gather and transcribe

* fix failing testcase

* fix failing testcase

* fix failing testcase
2023-12-05 20:27:55 -05:00
Markus Frindt
46755f909c do not escape req.body.tag (#559)
Co-authored-by: Markus Frindt <m.frindt@cognigy.com>
2023-12-05 10:06:04 -05:00
Hoan Luu Huu
4273196447 Fix/interdigit timeout (#554)
* fix interdtmf timeout

* wip

* wip

* wip

* remove all speech listener if dtmf is received
2023-12-04 08:16:42 -05:00
Dave Horton
e5b60ca9b0 trying to pass redis client to speech utils causing restart (#557) 2023-12-01 18:26:33 -05:00
Markus Frindt
86a14daf79 Update say task and add possibility to use elevenlabs options (#556)
* Update say task and add possibility to use elevenlabs options from synthesizer

* revert ms change

* fix contdition for alerting

---------

Co-authored-by: Markus Frindt <m.frindt@cognigy.com>
2023-12-01 09:51:22 -05:00
Hoan Luu Huu
c66ad39001 sending DTMF inband from updateCall/command (#536)
* sending DTMF inband from updateCall/command

* wip

* wip
2023-12-01 09:05:14 -05:00
Hoan Luu Huu
0a0cbd57ba support elevenlabs options (#553)
* support elevenlabs options

* elevenlabs options from synthezier

* wip

* fix
2023-11-30 09:28:12 -05:00
Hoan Luu Huu
eb2d90ffaa fix release freeswitch media properly (#550)
* fix release freeswitch media properly

* if a leg is opus, modify b leg offer opus first

* if a leg is opus, modify b leg offer opus first

* wip

* wip

* fix review comments

* fix review comments

* fix review comments
2023-11-29 10:17:15 -05:00
Anton Voylenko
454ff7d1b8 update envs (#551) 2023-11-28 14:54:39 -05:00
Markus Frindt
7e349fe4e5 Add a custom sanitization function for the "Tag" property (#546)
* Add a custom sanitization function for the tag property in create call body

* remove tag.* property from schema

* do not escape URLs but validate them

* apply suggestions from PR review

---------

Co-authored-by: Markus Frindt <m.frindt@cognigy.com>
2023-11-27 13:03:47 -05:00
Hoan Luu Huu
9478f3a1b8 fix disable gather timeout if interDigitTimeout enabled. (#538) 2023-11-21 19:09:09 -06:00
Hoan Luu Huu
a3c241b569 update drachtio-fsmrf new version for fixing amd and transcribe run parallel (#543) 2023-11-21 11:12:45 -06:00
Anton Voylenko
5a68563f96 live call control for tag (#539) 2023-11-17 08:48:17 -05:00
Hoan Luu Huu
1cdd0cf611 feat Audult call session should have its own requestor (#535)
* feat Audult call session should have its own requestor

* fix eslint

* fix eslint

* if user authenticate from http server instead of databse

* wip

* fix custom STT

* fix custom STT

* fix custom STT
2023-11-16 08:22:35 -05:00
Dave Horton
9ae4b04fc5 update badge 2023-11-14 10:27:16 -05:00
Anton Voylenko
170c3c7ec4 rest:dial: Support sipindialog hook (#531)
* support sipindialog hook

* rename sipindialog method
2023-11-14 09:16:08 -05:00
Dave Horton
7c36a08852 Feat/dial via sip proxy (#532)
* add support for dialing via proxy with target.proxy

* update verb-specifications with support for target.proxy
2023-11-14 09:06:43 -05:00
Hoan Luu Huu
633237da1b let realtimedb-help build configuration from env vars (#526)
* let realtimedb-help build configuration from env vars

* update speech-utils version
2023-11-14 08:57:50 -05:00
Dave Horton
708c2c661f 0.8.5 2023-11-09 12:36:04 -05:00
Hoan Luu Huu
87632c549e feat support Whisper TTS (#523)
* feat support openai

* update speech utils version
2023-11-09 09:51:20 -05:00
Hoan Luu Huu
31559cbb3b user restriction (#520) 2023-11-08 12:39:56 -05:00
Dave Horton
1156bae2de fix for #521 - allow pause in confirmHook applications (#522) 2023-11-07 09:46:41 -05:00
Dave Horton
c6c599ab99 fix dialogflow tts bug (tts not working due to 'default' being assigned to label) and update to drachtio-srf with fix for parsing sip:1234@feature-server (#518) 2023-11-06 15:05:17 -05:00
Dave Horton
4d0f0fe75f prevent exception referencing user.uri (#517) 2023-11-06 13:42:37 -05:00
38 changed files with 3820 additions and 725 deletions

View File

@@ -1,4 +1,4 @@
# jambonz-feature-server ![Build Status](https://github.com/jambonz/jambonz-feature-server/workflows/CI/badge.svg)
# jambonz-feature-server [![CI](https://github.com/jambonz/jambonz-feature-server/actions/workflows/build.yml/badge.svg)](https://github.com/jambonz/jambonz-feature-server/actions/workflows/build.yml)
This application implements the core feature server of the jambones platform.
@@ -40,6 +40,8 @@ Configuration is provided via environment variables:
|JAMBONZ_RECORD_WS_BASE_URL| recording websocket URL to send the recording audio|no|
|JAMBONZ_RECORD_WS_USERNAME| recording websocket username|no|
|JAMBONZ_RECORD_WS_PASSWORD| recording websocket password|no|
|ANCHOR_MEDIA_ALWAYS| keep media on media server|no|
|JAMBONZ_DISABLE_DIAL_PAI_HEADER| control P-Asserted-Identity header on B-Leg|no|
### running under pm2
Typically, this application runs under [pm2](https://pm2.io) using an [ecosystem.config.js](https://pm2.keymetrics.io/docs/usage/application-declaration/) file similar to this:

2
app.js
View File

@@ -109,7 +109,7 @@ const disconnect = () => {
httpServer?.on('close', resolve);
httpServer?.close();
srf.disconnect();
srf.locals.mediaservers.forEach((ms) => ms.disconnect());
srf.locals.mediaservers?.forEach((ms) => ms.disconnect());
});
};

0
bin/k8s-pre-stop-hook.js Executable file → Normal file
View File

View File

@@ -45,7 +45,7 @@ The GCP credential is the JSON service key in stringified format.
#### Install Docker
The test suite ralso equires [Docker](https://www.docker.com/) and docker-compose to be installed on your laptop. Docker is used to set up a network with all of the elements required to test the jambonz-feature-server in a black-box type of fashion.
The test suite also requires [Docker](https://www.docker.com/) and docker-compose to be installed on your laptop. Docker is used to set up a network with all of the elements required to test the jambonz-feature-server in a black-box type of fashion.
Once you have docker installed, you can optionally make sure everything Docker-wise is working properly by running this command from the project folder:

View File

@@ -28,10 +28,6 @@ const JAMBONES_MYSQL_PORT = parseInt(process.env.JAMBONES_MYSQL_PORT, 10) || 330
const JAMBONES_MYSQL_REFRESH_TTL = parseInt(process.env.JAMBONES_MYSQL_REFRESH_TTL, 10) || 0;
const JAMBONES_MYSQL_CONNECTION_LIMIT = parseInt(process.env.JAMBONES_MYSQL_CONNECTION_LIMIT, 10) || 10;
/* redis */
const JAMBONES_REDIS_HOST = process.env.JAMBONES_REDIS_HOST;
const JAMBONES_REDIS_PORT = parseInt(process.env.JAMBONES_REDIS_PORT, 10) || 6379;
/* gather and hints */
const JAMBONES_GATHER_EARLY_HINTS_MATCH = process.env.JAMBONES_GATHER_EARLY_HINTS_MATCH;
const JAMBONZ_GATHER_EARLY_HINTS_MATCH = process.env.JAMBONZ_GATHER_EARLY_HINTS_MATCH;
@@ -124,35 +120,18 @@ const HTTP_TIMEOUT = 10000;
const HTTP_PROXY_IP = process.env.JAMBONES_HTTP_PROXY_IP;
const HTTP_PROXY_PORT = process.env.JAMBONES_HTTP_PROXY_PORT;
const HTTP_PROXY_PROTOCOL = process.env.JAMBONES_HTTP_PROXY_PROTOCOL || 'http';
const HTTP_USER_AGENT_HEADER = process.env.JAMBONES_HTTP_USER_AGENT_HEADER || 'jambonz';
const OPTIONS_PING_INTERVAL = parseInt(process.env.OPTIONS_PING_INTERVAL, 10) || 30000;
const JAMBONES_REDIS_SENTINELS = process.env.JAMBONES_REDIS_SENTINELS ? {
sentinels: process.env.JAMBONES_REDIS_SENTINELS.split(',').map((sentinel) => {
let host, port = 26379;
if (sentinel.includes(':')) {
const arr = sentinel.split(':');
host = arr[0];
port = parseInt(arr[1], 10);
} else {
host = sentinel;
}
return {host, port};
}),
name: process.env.JAMBONES_REDIS_SENTINEL_MASTER_NAME,
...(process.env.JAMBONES_REDIS_SENTINEL_PASSWORD && {
password: process.env.JAMBONES_REDIS_SENTINEL_PASSWORD
}),
...(process.env.JAMBONES_REDIS_SENTINEL_USERNAME && {
username: process.env.JAMBONES_REDIS_SENTINEL_USERNAME
})
} : null;
const JAMBONZ_RECORD_WS_BASE_URL = process.env.JAMBONZ_RECORD_WS_BASE_URL || process.env.JAMBONES_RECORD_WS_BASE_URL;
const JAMBONZ_RECORD_WS_USERNAME = process.env.JAMBONZ_RECORD_WS_USERNAME || process.env.JAMBONES_RECORD_WS_USERNAME;
const JAMBONZ_RECORD_WS_PASSWORD = process.env.JAMBONZ_RECORD_WS_PASSWORD || process.env.JAMBONES_RECORD_WS_PASSWORD;
const JAMBONZ_DISABLE_DIAL_PAI_HEADER = process.env.JAMBONZ_DISABLE_DIAL_PAI_HEADER || false;
const JAMBONES_DISABLE_DIRECT_P2P_CALL = process.env.JAMBONES_DISABLE_DIRECT_P2P_CALL || false;
const JAMBONES_EAGERLY_PRE_CACHE_AUDIO = process.env.JAMBONES_EAGERLY_PRE_CACHE_AUDIO;
module.exports = {
JAMBONES_MYSQL_HOST,
JAMBONES_MYSQL_USER,
@@ -170,14 +149,12 @@ module.exports = {
JAMBONZ_GATHER_EARLY_HINTS_MATCH,
JAMBONES_GATHER_CLEAR_GLOBAL_HINTS_ON_EMPTY_HINTS,
JAMBONES_FREESWITCH,
JAMBONES_REDIS_HOST,
JAMBONES_REDIS_PORT,
JAMBONES_REDIS_SENTINELS,
SMPP_URL,
JAMBONES_NETWORK_CIDR,
JAMBONES_API_BASE_URL,
JAMBONES_TIME_SERIES_HOST,
JAMBONES_INJECT_CONTENT,
JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
JAMBONES_ESL_LISTEN_ADDRESS,
JAMBONES_SBCS,
JAMBONES_OTEL_ENABLED,
@@ -220,6 +197,7 @@ module.exports = {
HTTP_PROXY_IP,
HTTP_PROXY_PORT,
HTTP_PROXY_PROTOCOL,
HTTP_USER_AGENT_HEADER,
OPTIONS_PING_INTERVAL,
RESPONSE_TIMEOUT_MS,
JAMBONES_WS_HANDSHAKE_TIMEOUT_MS,

View File

@@ -5,7 +5,7 @@ const CallInfo = require('../../session/call-info');
const {CallDirection, CallStatus} = require('../../utils/constants');
const uuidv4 = require('uuid-random');
const SipError = require('drachtio-srf').SipError;
const { validationResult } = require('express-validator');
const { validationResult, body } = require('express-validator');
const { validate } = require('@jambonz/verb-specifications');
const sysError = require('./error');
const HttpRequestor = require('../../utils/http-requestor');
@@ -13,7 +13,7 @@ const WsRequestor = require('../../utils/ws-requestor');
const RootSpan = require('../../utils/call-tracer');
const dbUtils = require('../../utils/db-utils');
const { mergeSdpMedia, extractSdpMedia } = require('../../utils/sdp-utils');
const { createCallSchema } = require('../schemas/create-call');
const { createCallSchema, customSanitizeFunction } = require('../schemas/create-call');
const removeNullProperties = (obj) => (Object.keys(obj).forEach((key) => obj[key] === null && delete obj[key]), obj);
const removeNulls = (req, res, next) => {
@@ -24,6 +24,12 @@ const removeNulls = (req, res, next) => {
router.post('/',
removeNulls,
createCallSchema,
body('tag').custom((value) => {
if (value) {
customSanitizeFunction(value);
}
return true;
}),
async(req, res) => {
const {logger} = req.app.locals;
const errors = validationResult(req);

View File

@@ -45,7 +45,7 @@ function retrieveCallSession(callSid, opts) {
router.post('/:callSid', async(req, res) => {
const logger = req.app.locals.logger;
const callSid = req.params.callSid;
logger.debug({body: req.body}, 'got upateCall request');
logger.debug({body: req.body}, 'got updateCall request');
try {
const cs = retrieveCallSession(callSid, req.body);
if (!cs) {

View File

@@ -46,11 +46,6 @@ const createCallSchema = checkSchema({
optional: true,
errorMessage: 'Invalid tag',
},
'tag.*': {
trim: true,
escape: true,
stripLow: true,
},
app_json: {
isString: true,
optional: true,
@@ -109,6 +104,31 @@ const createCallSchema = checkSchema({
}
}, ['body']);
module.exports = {
createCallSchema
const customSanitizeFunction = (value) => {
try {
if (Array.isArray(value)) {
value = value.map((item) => customSanitizeFunction(item));
} else if (typeof value === 'object') {
Object.keys(value).forEach((key) => {
value[key] = customSanitizeFunction(value[key]);
});
} else if (typeof value === 'string') {
/* trims characters at the beginning and at the end of a string */
value = value.trim();
/* Verify strings including 'http' via new URL */
if (value.includes('http')) {
value = new URL(value).toString();
}
}
} catch (error) {
value = `Error: ${error.message}`;
}
return value;
};
module.exports = {
createCallSchema,
customSanitizeFunction
};

View File

@@ -23,7 +23,8 @@ module.exports = function(srf, logger) {
lookupAppBySid,
lookupAppByRealm,
lookupAppByTeamsTenant,
registrar
registrar,
lookupClientByAccountAndUsername
} = srf.locals.dbHelpers;
const {
writeAlerts,
@@ -35,6 +36,7 @@ module.exports = function(srf, logger) {
const callId = req.get('Call-ID');
const uri = parseUri(req.uri);
logger.info({
uri,
callId,
callingNumber: req.callingNumber,
calledNumber: req.calledNumber
@@ -47,10 +49,23 @@ module.exports = function(srf, logger) {
const account_sid = req.get('X-Account-Sid');
req.locals = {callSid, account_sid, callId};
if (req.has('X-Authenticated-User')) req.locals.originatingUser = req.get('X-Authenticated-User');
let clientDb = null;
if (req.has('X-Authenticated-User')) {
req.locals.originatingUser = req.get('X-Authenticated-User');
let clientSettings;
const arr = /^(.*)@(.*)/.exec(req.locals.originatingUser);
if (arr) {
[clientSettings] = await lookupClientByAccountAndUsername(account_sid, arr[1]);
}
clientDb = await registrar.query(req.locals.originatingUser);
clientDb = {
...clientDb,
...clientSettings,
};
}
// check for call to application
if (uri.user.startsWith('app-') && req.locals.originatingUser) {
if (uri.user?.startsWith('app-') && req.locals.originatingUser && clientDb?.allow_direct_app_calling) {
const application_sid = uri.user.match(/app-(.*)/)[1];
logger.debug(`got application from Request URI header: ${application_sid}`);
req.locals.application_sid = application_sid;
@@ -60,13 +75,13 @@ module.exports = function(srf, logger) {
req.locals.application_sid = application_sid;
}
// check for call to queue
if (uri.user.startsWith('queue-') && req.locals.originatingUser) {
if (uri.user?.startsWith('queue-') && req.locals.originatingUser && clientDb?.allow_direct_queue_calling) {
const queue_name = uri.user.match(/queue-(.*)/)[1];
logger.debug(`got Queue from Request URI header: ${queue_name}`);
req.locals.queue_name = queue_name;
}
// check for call to registered user
if (!JAMBONES_DISABLE_DIRECT_P2P_CALL && req.locals.originatingUser) {
if (!JAMBONES_DISABLE_DIRECT_P2P_CALL && req.locals.originatingUser && clientDb?.allow_direct_user_calling) {
const arr = /^(.*)@(.*)/.exec(req.locals.originatingUser);
if (arr) {
const sipRealm = arr[2];
@@ -151,7 +166,7 @@ module.exports = function(srf, logger) {
};
logger.info({callId, metadata, sdp}, 'successfully parsed SIPREC payload');
} catch (err) {
logger.info({callId}, 'Error parsing multipart payload');
logger.info({err, callId}, 'Error parsing multipart payload');
return res.send(503);
}
}
@@ -367,7 +382,7 @@ module.exports = function(srf, logger) {
},
recognizer: {
vendor: app.speech_recognizer_vendor,
...(app.speech_synthesis_label && {label: app.speech_synthesis_label}),
...(app.speech_recognizer_label && {label: app.speech_recognizer_label}),
language: app.speech_recognizer_language,
...(app.fallback_speech_recognizer_vendor && {fallback_vendor: app.fallback_speech_recognizer_vendor}),
...(app.fallback_speech_recognizer_label && {fallback_label: app.fallback_speech_recognizer_label}),

View File

@@ -1,4 +1,6 @@
const CallSession = require('./call-session');
const {CallStatus} = require('../utils/constants');
const moment = require('moment');
/**
* @classdesc Subclass of CallSession. Represents a CallSession
@@ -19,12 +21,14 @@ class AdultingCallSession extends CallSession {
rootSpan
});
this.sd = singleDialer;
this.req = callInfo.req;
this.sd.dlg.on('destroy', () => {
this.logger.info('AdultingCallSession: called party hung up');
this._callReleased();
});
this.sd.emit('adulting');
this.on('callStatusChange', this._notifyCallStatusChange.bind(this));
}
get dlg() {
@@ -49,6 +53,18 @@ class AdultingCallSession extends CallSession {
}
_callerHungup() {
if (this.dlg.connectTime) {
const duration = moment().diff(this.dlg.connectTime, 'seconds');
this.rootSpan.setAttributes({'call.termination': 'hangup by caller'});
this.callInfo.callTerminationBy = 'caller';
this.emit('callStatusChange', {
callStatus: CallStatus.Completed,
duration
});
}
this.logger.info('InboundCallSession: caller hung up');
this._callReleased();
this.req.removeAllListeners('cancel');
}
}

View File

@@ -19,11 +19,10 @@ const HttpRequestor = require('../utils/http-requestor');
const WsRequestor = require('../utils/ws-requestor');
const {
JAMBONES_INJECT_CONTENT,
JAMBONES_EAGERLY_PRE_CACHE_AUDIO,
AWS_REGION,
JAMBONZ_RECORD_WS_BASE_URL,
JAMBONZ_RECORD_WS_USERNAME,
JAMBONZ_RECORD_WS_PASSWORD,
} = require('../config');
const BackgroundTaskManager = require('../utils/background-task-manager');
const BADPRECONDITIONS = 'preconditions not met';
const CALLER_CANCELLED_ERR_MSG = 'Response not sent due to unknown transaction';
@@ -66,6 +65,11 @@ class CallSession extends Emitter {
this.callGone = false;
this.notifiedComplete = false;
this.rootSpan = rootSpan;
this.backgroundTaskManager = new BackgroundTaskManager({
cs: this,
logger,
rootSpan
});
this._origRecognizerSettings = {
vendor: this.application?.speech_recognizer_vendor,
@@ -136,8 +140,7 @@ class CallSession extends Emitter {
}
get isBackGroundListen() {
return !(this.backgroundListenTask === null ||
this.backgroundListenTask === undefined);
return this.backgroundTaskManager.isTaskRunning('listen');
}
/**
@@ -170,6 +173,18 @@ class CallSession extends Emitter {
return this.application.notifier;
}
/**
* syntheizer
*/
get synthesizer() {
return this._synthesizer;
}
set synthesizer(synth) {
this._synthesizer = synth;
}
/**
* default vendor to use for speech synthesis if not provided in the app
*/
@@ -252,6 +267,16 @@ class CallSession extends Emitter {
set fallbackSpeechRecognizerVendor(vendor) {
this.application.fallback_speech_recognizer_vendor = vendor;
}
/**
* recognizer
*/
get recognizer() {
return this._recognizer;
}
set recognizer(rec) {
this._recognizer = rec;
}
/**
* default vendor to use for speech recognition if not provided in the app
*/
@@ -357,11 +382,11 @@ class CallSession extends Emitter {
}
get isBotModeEnabled() {
return this.backgroundGatherTask;
return this.backgroundTaskManager.isTaskRunning('bargeIn');
}
get isListenEnabled() {
return this.backgroundListenTask;
return this.backgroundTaskManager.isTaskRunning('listen');
}
get b3() {
@@ -590,117 +615,41 @@ class CallSession extends Emitter {
}
}
async startBackgroundListen(opts, bugname) {
if (this.isListenEnabled) {
this.logger.info('CallSession:startBackgroundListen - listen is already enabled, ignoring request');
return;
}
try {
this.logger.debug({opts}, 'CallSession:startBackgroundListen');
const t = normalizeJambones(this.logger, [opts]);
this.backgroundListenTask = makeTask(this.logger, t[0]);
this.backgroundListenTask.bugname = bugname;
// Remove unneeded customer data to be sent to api server.
this.backgroundListenTask.ignoreCustomerData = true;
const resources = await this._evaluatePreconditions(this.backgroundListenTask);
const {span, ctx} = this.rootSpan.startChildSpan(`background-listen:${this.backgroundListenTask.summary}`);
this.backgroundListenTask.span = span;
this.backgroundListenTask.ctx = ctx;
this.backgroundListenTask.exec(this, resources)
.then(() => {
this.logger.info('CallSession:startBackgroundListen: listen completed');
this.backgroundListenTask && this.backgroundListenTask.removeAllListeners();
this.backgroundListenTask && this.backgroundListenTask.span.end();
this.backgroundListenTask = null;
return;
})
.catch((err) => {
this.logger.info({err}, 'CallSession:startBackgroundListen: listen threw error');
this.backgroundListenTask && this.backgroundListenTask.removeAllListeners();
this.backgroundListenTask && this.backgroundListenTask.span.end();
this.backgroundListenTask = null;
});
} catch (err) {
this.logger.info({err, opts}, 'CallSession:startBackgroundListen - Error creating listen task');
}
}
async stopBackgroundListen() {
this.logger.debug('CallSession:stopBackgroundListen');
try {
if (this.backgroundListenTask) {
this.backgroundListenTask.removeAllListeners();
this.backgroundListenTask.kill().catch(() => {});
}
} catch (err) {
this.logger.info({err}, 'CallSession:stopBackgroundListen - Error stopping listen task');
}
}
async enableBotMode(gather, autoEnable) {
try {
const t = normalizeJambones(this.logger, [gather]);
const task = makeTask(this.logger, t[0]);
let task;
if (this.isBotModeEnabled) {
const currInput = this.backgroundGatherTask.input;
const newInput = task.input;
task = this.backgroundTaskManager.getTask('bargeIn');
const currInput = task.input;
const t = normalizeJambones(this.logger, [gather]);
const tmpTask = makeTask(this.logger, t[0]);
const newInput = tmpTask.input;
if (JSON.stringify(currInput) === JSON.stringify(newInput)) {
this.logger.info('CallSession:enableBotMode - bot mode currently enabled, ignoring request to start again');
return;
}
else {
} else {
this.logger.info({currInput, newInput},
'CallSession:enableBotMode - restarting background gather to apply new input type');
this.backgroundGatherTask.sticky = false;
'CallSession:enableBotMode - restarting background bargeIn to apply new input type');
task.sticky = false;
await this.disableBotMode();
}
}
this.backgroundGatherTask = task;
this._bargeInEnabled = true;
this.backgroundGatherTask
.once('dtmf', this._clearTasks.bind(this, this.backgroundGatherTask))
.once('vad', this._clearTasks.bind(this, this.backgroundGatherTask))
.once('transcription', this._clearTasks.bind(this, this.backgroundGatherTask))
.once('timeout', this._clearTasks.bind(this, this.backgroundGatherTask));
this.logger.info({gather}, 'CallSession:enableBotMode - starting background gather');
const resources = await this._evaluatePreconditions(this.backgroundGatherTask);
const {span, ctx} = this.rootSpan.startChildSpan(`background-gather:${this.backgroundGatherTask.summary}`);
this.backgroundGatherTask.span = span;
this.backgroundGatherTask.ctx = ctx;
this.backgroundGatherTask.sticky = autoEnable;
this.backgroundGatherTask.exec(this, resources)
.then(() => {
this.logger.info('CallSession:enableBotMode: gather completed');
this.backgroundGatherTask && this.backgroundGatherTask.removeAllListeners();
this.backgroundGatherTask && this.backgroundGatherTask.span.end();
const sticky = this.backgroundGatherTask?.sticky;
this.backgroundGatherTask = null;
if (sticky && !this.callGone && !this._stopping && this._bargeInEnabled) {
this.logger.info('CallSession:enableBotMode: restarting background gather');
setImmediate(() => this.enableBotMode(gather, true));
}
return;
})
.catch((err) => {
this.logger.info({err}, 'CallSession:enableBotMode: gather threw error');
this.backgroundGatherTask && this.backgroundGatherTask.removeAllListeners();
this.backgroundGatherTask && this.backgroundGatherTask.span.end();
this.backgroundGatherTask = null;
});
task = await this.backgroundTaskManager.newTask('bargeIn', gather);
task.sticky = autoEnable;
task.once('bargeIn-done', () => {
if (this.requestor instanceof WsRequestor) {
try {
this.kill(true);
} catch (err) {}
}
});
this.logger.info({gather}, 'CallSession:enableBotMode - starting background bargeIn');
} catch (err) {
this.logger.info({err, gather}, 'CallSession:enableBotMode - Error creating gather task');
this.logger.info({err, gather}, 'CallSession:enableBotMode - Error creating bargeIn task');
}
}
async disableBotMode() {
this._bargeInEnabled = false;
if (this.backgroundGatherTask) {
try {
this.backgroundGatherTask.removeAllListeners();
await this.backgroundGatherTask.kill();
} catch (err) {}
this.backgroundGatherTask = null;
}
this.backgroundTaskManager.stop('bargeIn');
}
setConferenceDetails(memberId, confName, confUuid) {
@@ -833,13 +782,19 @@ class CallSession extends Emitter {
} else if ('elevenlabs' === vendor) {
return {
api_key: credential.api_key,
model_id: credential.model_id
model_id: credential.model_id,
options: credential.options
};
} else if ('assemblyai' === vendor) {
return {
speech_credential_sid: credential.speech_credential_sid,
api_key: credential.api_key
};
} else if ('whisper' === vendor) {
return {
api_key: credential.api_key,
model_id: credential.model_id
};
} else if (vendor.startsWith('custom:')) {
return {
speech_credential_sid: credential.speech_credential_sid,
@@ -878,7 +833,7 @@ class CallSession extends Emitter {
let skip = false;
this.currentTask = task;
if (TaskName.Gather === task.name && this.isBotModeEnabled) {
if (this.backgroundGatherTask.updateTaskInProgress(task) !== false) {
if (this.backgroundTaskManager.getTask('bargeIn').updateTaskInProgress(task) !== false) {
this.logger.info(`CallSession:exec skipping #${stackNum}:${taskNum}: ${task.name}`);
skip = true;
}
@@ -913,7 +868,8 @@ class CallSession extends Emitter {
if (0 === this.tasks.length &&
this.requestor instanceof WsRequestor &&
!this.requestor.closedGracefully &&
!this.callGone
!this.callGone &&
!this.isConfirmCallSession
) {
try {
await this._awaitCommandsOrHangup();
@@ -1116,6 +1072,21 @@ class CallSession extends Emitter {
transcribeTask.updateTranscribe(opts.transcribe_status);
}
/**
* perform live call control -- update customer data
* @param {object} opts
* @param {object} opts.tag - customer data
*/
_lccTag(opts) {
const {tag} = opts;
if (typeof tag !== 'object' || Array.isArray(tag) || tag === null) {
this.logger.info('CallSession:_lccTag - invalid tag data');
return;
}
this.logger.debug({customerData: tag}, 'CallSession:_lccTag set customer data in callInfo');
this.callInfo.customerData = tag;
}
async _lccMuteStatus(callSid, mute) {
// this whole thing requires us to be in a Dial or Conference verb
const task = this.currentTask;
@@ -1166,6 +1137,38 @@ class CallSession extends Emitter {
}
}
/**
* perform live call control - send RFC 2833 DTMF
* @param {obj} opts
* @param {string} opts.dtmf.digit - DTMF digit
* @param {string} opts.dtmf.duration - Optional, Duration
*/
async _lccDtmf(opts, callSid) {
const {dtmf} = opts;
const {digit, duration = 250} = dtmf;
if (!this.hasStableDialog) {
this.logger.info('CallSession:_lccDtmf - invalid command as we do not have a stable call');
return;
}
try {
const dlg = callSid === this.callSid ? this.dlg : this.currentTask.dlg;
const res = await dlg.request({
method: 'INFO',
headers: {
'Content-Type': 'application/dtmf',
'X-Reason': 'Dtmf'
},
body: `Signal=${digit}
Duration=${duration} `
});
this.logger.debug({res}, `CallSession:_lccDtmf
got response to INFO DTMF digit=${digit} and duration=${duration}`);
return res;
} catch (err) {
this.logger.error({err}, 'CallSession:_lccDtmf - error sending INFO RFC 2833 DTMF');
}
}
/**
* perform live call control -- whisper to one party or the other on a call
* @param {array} opts - array of play or say tasks
@@ -1220,7 +1223,7 @@ class CallSession extends Emitter {
/**
* perform live call control
* @param {object} opts - update instructions
* @param {string} callSid - identifies call toupdate
* @param {string} callSid - identifies call to update
*/
async updateCall(opts, callSid) {
this.logger.debug(opts, 'CallSession:updateCall');
@@ -1249,13 +1252,18 @@ class CallSession extends Emitter {
else if (opts.sip_request) {
const res = await this._lccSipRequest(opts, callSid);
return {status: res.status, reason: res.reason};
} else if (opts.dtmf) {
await this._lccDtmf(opts, callSid);
}
else if (opts.record) {
await this.notifyRecordOptions(opts.record);
}
else if (opts.tag) {
return this._lccTag(opts);
}
// whisper may be the only thing we are asked to do, or it may that
// we are doing a whisper after having muted, paused reccording etc..
// we are doing a whisper after having muted, paused recording etc..
if (opts.whisper) {
return this._lccWhisper(opts, callSid);
}
@@ -1313,7 +1321,6 @@ class CallSession extends Emitter {
get the full transcription.
*/
delete t.bargeIn.enable;
this._bargeInEnabled = false;
this.logger.info('CallSession:kill - found bargein disabled in the stack, clearing to that point');
break;
}
@@ -1324,6 +1331,35 @@ class CallSession extends Emitter {
this.taskIdx = 0;
}
_preCacheAudio(newTasks) {
for (const task of newTasks) {
if (task.name === TaskName.Config && task.hasSynthesizer) {
/* if they change synthesizer settings don't try to precache */
break;
}
if (task.name === TaskName.Say) {
/* identify vendor language, voice, and label */
const vendor = task.synthesizer.vendor && task.synthesizer.vendor !== 'default' ?
task.synthesizer.vendor :
this.speechSynthesisVendor;
const language = task.synthesizer.language && task.synthesizer.language !== 'default' ?
task.synthesizer.language :
this.speechSynthesisLanguage ;
const voice = task.synthesizer.voice && task.synthesizer.voice !== 'default' ?
task.synthesizer.voice :
this.speechSynthesisVoice;
const label = task.synthesizer.label && task.synthesizer.label !== 'default' ?
task.synthesizer.label :
this.speechSynthesisLabel;
this.logger.info({vendor, language, voice, label},
'CallSession:_preCacheAudio - precaching audio for future prompt');
task._synthesizeWithSpecificVendor(this, this.ep, {vendor, language, voice, label, preCache: true})
.catch((err) => this.logger.error(err, 'CallSession:_preCacheAudio - error precaching audio'));
}
}
}
/**
* Append tasks to the current execution stack UNLESS there is a gather in the stack.
* in that case, insert the tasks before the gather AND if the tasks include
@@ -1381,10 +1417,12 @@ class CallSession extends Emitter {
this.replaceApplication(t);
}
else if (JAMBONES_INJECT_CONTENT) {
if (JAMBONES_EAGERLY_PRE_CACHE_AUDIO) this._preCacheAudio(t);
this._injectTasks(t);
this.logger.info({tasks: listTaskNames(this.tasks)}, 'CallSession:_onCommand - updated task list');
}
else {
if (JAMBONES_EAGERLY_PRE_CACHE_AUDIO) this._preCacheAudio(t);
this.tasks.push(...t);
this.logger.info({tasks: listTaskNames(this.tasks)}, 'CallSession:_onCommand - updated task list');
}
@@ -1428,6 +1466,13 @@ class CallSession extends Emitter {
});
break;
case 'dtmf':
this._lccDtmf(data, call_sid)
.catch((err) => {
this.logger.info({err, data}, `CallSession:_onCommand - error sending RFC 2833 DTMF ${data}`);
});
break;
default:
this.logger.info(`CallSession:_onCommand - invalid command ${command}`);
}
@@ -1612,6 +1657,8 @@ class CallSession extends Emitter {
this.notifier && this.notifier.close();
this.rootSpan && this.rootSpan.end();
// close all background tasks
this.backgroundTaskManager.stopAll();
}
/**
@@ -1868,7 +1915,7 @@ class CallSession extends Emitter {
wrapDialog(dlg) {
dlg.connectTime = moment();
const origDestroy = dlg.destroy.bind(dlg);
dlg.destroy = () => {
dlg.destroy = (opts) => {
if (dlg.connected) {
dlg.connected = false;
dlg.destroy = origDestroy;
@@ -1877,7 +1924,7 @@ class CallSession extends Emitter {
this.emit('callStatusChange', {callStatus: CallStatus.Completed, duration});
this.logger.debug('CallSession: call terminated by jambonz');
this.rootSpan.setAttributes({'call.termination': 'hangup by jambonz'});
origDestroy().catch((err) => this.logger.info({err}, 'CallSession - error destroying dialog'));
origDestroy(opts).catch((err) => this.logger.info({err}, 'CallSession - error destroying dialog'));
if (this.wakeupResolver) {
this.wakeupResolver({reason: 'session ended'});
this.wakeupResolver = null;
@@ -1928,12 +1975,14 @@ class CallSession extends Emitter {
async _notifyCallStatusChange({callStatus, sipStatus, sipReason, duration}) {
if (this.callMoved) return;
// manage record all call.
if (callStatus === CallStatus.InProgress) {
// nice, call is in progress, good time to enable record
await this.enableRecordAllCall();
} else if (callStatus == CallStatus.Completed && this.isBackGroundListen) {
this.stopBackgroundListen().catch((err) => this.logger.error(
{err}, 'CallSession:_notifyCallStatusChange - error stopping background listen'));
if (this.accountInfo.account.record_all_calls ||
this.application.record_all_calls) {
this.backgroundTaskManager.newTask('record');
}
} else if (callStatus == CallStatus.Completed) {
this.backgroundTaskManager.stop('record');
}
/* race condition: we hang up at the same time as the caller */
@@ -1970,29 +2019,6 @@ class CallSession extends Emitter {
}
}
async enableRecordAllCall() {
if (this.accountInfo.account.record_all_calls || this.application.record_all_calls) {
if (!JAMBONZ_RECORD_WS_BASE_URL || !this.accountInfo.account.bucket_credential) {
this.logger.error('Record all calls: invalid configuration');
return;
}
const listenOpts = {
url: `${JAMBONZ_RECORD_WS_BASE_URL}/record/${this.accountInfo.account.bucket_credential.vendor}`,
disableBidirectionalAudio: true,
mixType : 'stereo',
passDtmf: true
};
if (JAMBONZ_RECORD_WS_USERNAME && JAMBONZ_RECORD_WS_PASSWORD) {
listenOpts.wsAuth = {
username: JAMBONZ_RECORD_WS_USERNAME,
password: JAMBONZ_RECORD_WS_PASSWORD
};
}
this.logger.debug({listenOpts}, 'Record all calls: enabling listen');
await this.startBackgroundListen({verb: 'listen', ...listenOpts}, 'jambonz-session-record');
}
}
_configMsEndpoint() {
if (this.onHoldMusic) {
this.ep.set({hold_music: `shout://${this.onHoldMusic.replace(/^https?:\/\//, '')}`});
@@ -2035,6 +2061,18 @@ class CallSession extends Emitter {
} catch (err) {}
}
}
/**
* startBackgroundTask - Start background task
*/
async startBackgroundTask(type, opts) {
await this.backgroundTaskManager.newTask(type, opts);
}
stopBackgroundTask(type) {
this.backgroundTaskManager.stop(type);
}
}
module.exports = CallSession;

View File

@@ -9,7 +9,8 @@ class TaskConfig extends Task {
'recognizer',
'bargeIn',
'record',
'listen'
'listen',
'transcribe'
].forEach((k) => this[k] = this.data[k] || {});
if ('notifyEvents' in this.data) {
@@ -30,6 +31,13 @@ class TaskConfig extends Task {
if (this.bargeIn[k]) this.gatherOpts[k] = this.bargeIn[k];
});
}
if (this.transcribe?.enable) {
this.transcribeOpts = {
verb: 'transcribe',
...this.transcribe
};
delete this.transcribeOpts.enable;
}
if (this.data.reset) {
if (typeof this.data.reset === 'string') this.data.reset = [this.data.reset];
@@ -37,7 +45,11 @@ class TaskConfig extends Task {
else this.data.reset = [];
if (this.bargeIn.sticky) this.autoEnable = true;
this.preconditions = (this.bargeIn.enable || this.record?.action || this.listen?.url || this.data.amd) ?
this.preconditions = (this.bargeIn.enable ||
this.record?.action ||
this.listen?.url ||
this.data.amd ||
this.transcribe?.enable) ?
TaskPreconditions.Endpoint :
TaskPreconditions.None;
@@ -50,6 +62,7 @@ class TaskConfig extends Task {
get hasRecognizer() { return Object.keys(this.recognizer).length; }
get hasRecording() { return Object.keys(this.record).length; }
get hasListen() { return Object.keys(this.listen).length; }
get hasTranscribe() { return Object.keys(this.transcribe).length; }
get summary() {
const phrase = [];
@@ -72,6 +85,9 @@ class TaskConfig extends Task {
if (this.hasListen) {
phrase.push(this.listen.enable ? `listen ${this.listen.url}` : 'stop listen');
}
if (this.hasTranscribe) {
phrase.push(this.transcribe.enable ? `transcribe ${this.transcribe.transcriptionHook}` : 'stop transcribe');
}
if (this.data.amd) phrase.push('enable amd');
if (this.notifyEvents) phrase.push(`event notification ${this.notifyEvents ? 'on' : 'off'}`);
if (this.onHoldMusic) phrase.push(`onHoldMusic: ${this.onHoldMusic}`);
@@ -109,6 +125,7 @@ class TaskConfig extends Task {
});
if (this.hasSynthesizer) {
cs.synthesizer = this.synthesizer;
cs.speechSynthesisVendor = this.synthesizer.vendor !== 'default'
? this.synthesizer.vendor
: cs.speechSynthesisVendor;
@@ -138,6 +155,7 @@ class TaskConfig extends Task {
this.logger.info({synthesizer: this.synthesizer}, 'Config: updated synthesizer');
}
if (this.hasRecognizer) {
cs.recognizer = this.recognizer;
cs.speechRecognizerVendor = this.recognizer.vendor !== 'default'
? this.recognizer.vendor
: cs.speechRecognizerVendor;
@@ -210,10 +228,25 @@ class TaskConfig extends Task {
const {enable, ...opts} = this.listen;
if (enable) {
this.logger.debug({opts}, 'Config: enabling listen');
cs.startBackgroundListen({verb: 'listen', ...opts});
cs.startBackgroundTask('listen', {verb: 'listen', ...opts});
} else {
this.logger.info('Config: disabling listen');
cs.stopBackgroundListen();
cs.stopBackgroundTask('listen');
}
}
if (this.hasTranscribe) {
if (this.transcribe.enable) {
this.transcribeOpts.recognizer = this.hasRecognizer ?
this.recognizer :
{
vendor: cs.speechRecognizerVendor,
language: cs.speechRecognizerLanguage
};
this.logger.debug(this.transcribeOpts, 'Config: enabling transcribe');
cs.startBackgroundTask('transcribe', this.transcribeOpts);
} else {
this.logger.info('Config: disabling transcribe');
cs.stopBackgroundTask('transcribe');
}
}
if (this.data.sipRequestWithinDialogHook) {

View File

@@ -17,7 +17,7 @@ const dbUtils = require('../utils/db-utils');
const debug = require('debug')('jambonz:feature-server');
const {parseUri} = require('drachtio-srf');
const {ANCHOR_MEDIA_ALWAYS, JAMBONZ_DISABLE_DIAL_PAI_HEADER} = require('../config');
const { isOnhold } = require('../utils/sdp-utils');
const { isOnhold, isOpusFirst } = require('../utils/sdp-utils');
const { normalizeJambones } = require('@jambonz/verb-specifications');
function parseDtmfOptions(logger, dtmfCapture) {
@@ -323,7 +323,7 @@ class TaskDial extends Task {
const by = parseUri(req.getParsedHeader('Referred-By').uri);
this.logger.info({to}, 'refer to parsed');
const json = await cs.requestor.request('verb:hook', this.referHook, {
...callInfo,
...(callInfo.toJSON()),
refer_details: {
sip_refer_to: req.get('Refer-To'),
sip_referred_by: req.get('Referred-By'),
@@ -488,7 +488,8 @@ class TaskDial extends Task {
headers: this.headers,
proxy: `sip:${sbcAddress}`,
callingNumber: this.callerId || req.callingNumber,
...(this.callerName && {callingName: this.callerName})
...(this.callerName && {callingName: this.callerName}),
opusFirst: isOpusFirst(this.cs.ep.remote.sdp)
};
const t = this.target.find((t) => t.type === 'teams');
@@ -790,9 +791,11 @@ class TaskDial extends Task {
assert(cs.ep && sd.ep);
try {
// Wait until we got new SDP from B leg to ofter to A Leg
const aLegSdp = cs.ep.remote.sdp;
await sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp);
const bLegSdp = sd.dlg.remote.sdp;
await Promise.all[sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp), cs.releaseMediaToSBC(bLegSdp)];
await cs.releaseMediaToSBC(bLegSdp);
this.epOther = null;
this.logger.info('Dial:_releaseMedia - successfully released media from freewitch');
} catch (err) {

View File

@@ -58,13 +58,13 @@ class Dialogflow extends Task {
this.vendor = this.data.tts.vendor || 'default';
this.language = this.data.tts.language || 'default';
this.voice = this.data.tts.voice || 'default';
this.speechSynthesisLabel = this.data.tts.label || 'default';
this.speechSynthesisLabel = this.data.tts.label;
// fallback tts
this.fallbackVendor = this.data.tts.fallbackVendor || 'default';
this.fallbackLanguage = this.data.tts.fallbackLanguage || 'default';
this.fallbackVoice = this.data.tts.fallbackLanguage || 'default';
this.fallbackLabel = this.data.tts.fallbackLabel || 'default';
this.fallbackLabel = this.data.tts.fallbackLabel;
}
this.bargein = this.data.bargein;
}

View File

@@ -71,6 +71,7 @@ class TaskGather extends SttTask {
/* buffer speech for continuous asr */
this._bufferedTranscripts = [];
this.partialTranscriptsCount = 0;
this.bugname_prefix = 'gather_';
}
get name() { return TaskName.Gather; }
@@ -103,7 +104,7 @@ class TaskGather extends SttTask {
async exec(cs, {ep}) {
this.logger.debug({options: this.data}, 'Gather:exec');
await super.exec(cs);
await super.exec(cs, {ep});
const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
if (cs.hasGlobalSttHints && !this.maskGlobalSttHints) {
@@ -116,14 +117,6 @@ class TaskGather extends SttTask {
this.logger.debug({hints: this.data.recognizer.hints, hintsBoost: this.data.recognizer.hintsBoost},
'Gather:exec - applying global sttHints');
}
if (cs.hasAltLanguages) {
this.data.recognizer.altLanguages = this.data.recognizer.altLanguages.concat(cs.altLanguages);
this.logger.debug({altLanguages: this.data.recognizer?.altLanguages},
'Gather:exec - applying altLanguages');
}
if (cs.hasGlobalSttPunctuation && !this.data.recognizer.punctuation) {
this.data.recognizer.punctuation = cs.globalSttPunctuation;
}
if (!this.isContinuousAsr && cs.isContinuousAsr) {
this.isContinuousAsr = true;
this.asrTimeout = cs.asrTimeout * 1000;
@@ -141,59 +134,6 @@ class TaskGather extends SttTask {
this.interim = true;
this.logger.debug('Gather:exec - early hints match enabled');
}
this.ep = ep;
if ('default' === this.vendor || !this.vendor) {
this.vendor = cs.speechRecognizerVendor;
if (this.data.recognizer) this.data.recognizer.vendor = this.vendor;
}
if ('default' === this.language || !this.language) {
this.language = cs.speechRecognizerLanguage;
if (this.data.recognizer) this.data.recognizer.language = this.language;
}
if ('default' === this.label || !this.label) {
this.label = cs.speechRecognizerLabel;
if (this.data.recognizer) this.data.recognizer.label = this.label;
}
// Fallback options
if ('default' === this.fallbackVendor || !this.fallbackVendor) {
this.fallbackVendor = cs.fallbackSpeechRecognizerVendor;
if (this.data.recognizer) this.data.recognizer.fallbackVendor = this.fallbackVendor;
}
if ('default' === this.fallbackLanguage || !this.fallbackLanguage) {
this.fallbackLanguage = cs.fallbackSpeechRecognizerLanguage;
if (this.data.recognizer) this.data.recognizer.fallbackLanguage = this.fallbackLanguage;
}
if ('default' === this.fallbackLabel || !this.fallbackLabel) {
this.fallbackLabel = cs.fallbackSpeechRecognizerLabel;
if (this.data.recognizer) this.data.recognizer.fallbackLabel = this.fallbackLabel;
}
if (!this.data.recognizer.vendor) {
this.data.recognizer.vendor = this.vendor;
}
if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
// By default, application saves cobalt model in language
this.data.recognizer.model = cs.speechRecognizerLanguage;
}
if (this.needsStt && !this.sttCredentials) {
try {
this.sttCredentials = await this._initSpeechCredentials(cs, this.vendor, this.label);
} catch (error) {
if (this.fallbackVendor && this.isHandledByPrimaryProvider) {
await this._fallback();
} else {
throw error;
}
}
}
/* when using cobalt model is required */
if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
this.notifyError({ msg: 'ASR error', details:'Cobalt requires a model to be specified'});
throw new Error('Cobalt requires a model to be specified');
}
const startListening = async(cs, ep) => {
this._startTimer();
if (this.isContinuousAsr && 0 === this.timeout) this._startAsrTimer();
@@ -281,7 +221,7 @@ class TaskGather extends SttTask {
} catch (err) {
this.logger.error(err, 'TaskGather:exec error');
}
this.removeSpeechListeners(ep);
this.removeCustomEventListeners();
}
kill(cs) {
@@ -319,6 +259,16 @@ class TaskGather extends SttTask {
this._resolve('dtmf-terminator-key');
}
else if (this.input.includes('digits')) {
if (this.digitBuffer.length === 0 && this.needsStt) {
// DTMF is higher priority than STT.
this.removeCustomEventListeners();
ep.stopTranscription({
vendor: this.vendor,
bugname: this.bugname,
})
.catch((err) => this.logger.error({err},
` Received DTMF, Error stopping transcription for vendor ${this.vendor}`));
}
this.digitBuffer += evt.dtmf;
const len = this.digitBuffer.length;
if (len === this.numDigits || len === this.maxDigits) {
@@ -354,34 +304,38 @@ class TaskGather extends SttTask {
const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.data.recognizer);
switch (this.vendor) {
case 'google':
this.bugname = 'google_transcribe';
ep.addCustomEventListener(GoogleTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
ep.addCustomEventListener(GoogleTranscriptionEvents.EndOfUtterance, this._onEndOfUtterance.bind(this, cs, ep));
ep.addCustomEventListener(GoogleTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
this.bugname = `${this.bugname_prefix}google_transcribe`;
this.addCustomEventListener(
ep, GoogleTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
this.addCustomEventListener(
ep, GoogleTranscriptionEvents.EndOfUtterance, this._onEndOfUtterance.bind(this, cs, ep));
this.addCustomEventListener(
ep, GoogleTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
break;
case 'aws':
case 'polly':
this.bugname = 'aws_transcribe';
ep.addCustomEventListener(AwsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
ep.addCustomEventListener(AwsTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
this.bugname = `${this.bugname_prefix}aws_transcribe`;
this.addCustomEventListener(ep, AwsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
this.addCustomEventListener(ep, AwsTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
break;
case 'microsoft':
this.bugname = 'azure_transcribe';
ep.addCustomEventListener(AzureTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
ep.addCustomEventListener(AzureTranscriptionEvents.NoSpeechDetected,
this.bugname = `${this.bugname_prefix}azure_transcribe`;
this.addCustomEventListener(
ep, AzureTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
this.addCustomEventListener(ep, AzureTranscriptionEvents.NoSpeechDetected,
this._onNoSpeechDetected.bind(this, cs, ep));
ep.addCustomEventListener(AzureTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
this.addCustomEventListener(ep, AzureTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
break;
case 'nuance':
this.bugname = 'nuance_transcribe';
ep.addCustomEventListener(NuanceTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}nuance_transcribe`;
this.addCustomEventListener(ep, NuanceTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep));
ep.addCustomEventListener(NuanceTranscriptionEvents.StartOfSpeech,
this.addCustomEventListener(ep, NuanceTranscriptionEvents.StartOfSpeech,
this._onStartOfSpeech.bind(this, cs, ep));
ep.addCustomEventListener(NuanceTranscriptionEvents.TranscriptionComplete,
this.addCustomEventListener(ep, NuanceTranscriptionEvents.TranscriptionComplete,
this._onTranscriptionComplete.bind(this, cs, ep));
ep.addCustomEventListener(NuanceTranscriptionEvents.VadDetected,
this.addCustomEventListener(ep, NuanceTranscriptionEvents.VadDetected,
this._onVadDetected.bind(this, cs, ep));
/* stall timers until prompt finishes playing */
@@ -391,10 +345,11 @@ class TaskGather extends SttTask {
break;
case 'deepgram':
this.bugname = 'deepgram_transcribe';
ep.addCustomEventListener(DeepgramTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
ep.addCustomEventListener(DeepgramTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
ep.addCustomEventListener(DeepgramTranscriptionEvents.ConnectFailure,
this.bugname = `${this.bugname_prefix}deepgram_transcribe`;
this.addCustomEventListener(
ep, DeepgramTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, DeepgramTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep));
/* if app sets deepgramOptions.utteranceEndMs they essentially want continuous asr */
@@ -402,13 +357,15 @@ class TaskGather extends SttTask {
break;
case 'soniox':
this.bugname = 'soniox_transcribe';
ep.addCustomEventListener(SonioxTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
this.bugname = `${this.bugname_prefix}soniox_transcribe`;
this.addCustomEventListener(
ep, SonioxTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
break;
case 'cobalt':
this.bugname = 'cobalt_transcribe';
ep.addCustomEventListener(CobaltTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
this.bugname = `${this.bugname_prefix}cobalt_transcribe`;
this.addCustomEventListener(
ep, CobaltTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
/* cobalt doesnt have language, it has model, which is required */
if (!this.data.recognizer.model) {
@@ -437,22 +394,22 @@ class TaskGather extends SttTask {
break;
case 'ibm':
this.bugname = 'ibm_transcribe';
ep.addCustomEventListener(IbmTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
ep.addCustomEventListener(IbmTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
ep.addCustomEventListener(IbmTranscriptionEvents.ConnectFailure,
this.bugname = `${this.bugname_prefix}ibm_transcribe`;
this.addCustomEventListener(ep, IbmTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
this.addCustomEventListener(ep, IbmTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, IbmTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep));
break;
case 'nvidia':
this.bugname = 'nvidia_transcribe';
ep.addCustomEventListener(NvidiaTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}nvidia_transcribe`;
this.addCustomEventListener(ep, NvidiaTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep));
ep.addCustomEventListener(NvidiaTranscriptionEvents.StartOfSpeech,
this.addCustomEventListener(ep, NvidiaTranscriptionEvents.StartOfSpeech,
this._onStartOfSpeech.bind(this, cs, ep));
ep.addCustomEventListener(NvidiaTranscriptionEvents.TranscriptionComplete,
this.addCustomEventListener(ep, NvidiaTranscriptionEvents.TranscriptionComplete,
this._onTranscriptionComplete.bind(this, cs, ep));
ep.addCustomEventListener(NvidiaTranscriptionEvents.VadDetected,
this.addCustomEventListener(ep, NvidiaTranscriptionEvents.VadDetected,
this._onVadDetected.bind(this, cs, ep));
/* I think nvidia has this (??) - stall timers until prompt finishes playing */
@@ -462,20 +419,22 @@ class TaskGather extends SttTask {
break;
case 'assemblyai':
this.bugname = 'assemblyai_transcribe';
ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}assemblyai_transcribe`;
this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep));
ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
ep.addCustomEventListener(AssemblyAiTranscriptionEvents.ConnectFailure,
this.addCustomEventListener(
ep, AssemblyAiTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep));
break;
default:
if (this.vendor.startsWith('custom:')) {
this.bugname = `${this.vendor}_transcribe`;
ep.addCustomEventListener(JambonzTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
ep.addCustomEventListener(JambonzTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
ep.addCustomEventListener(JambonzTranscriptionEvents.ConnectFailure,
this.bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
this.addCustomEventListener(
ep, JambonzTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
this.addCustomEventListener(ep, JambonzTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, JambonzTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep));
break;
}
@@ -487,7 +446,7 @@ class TaskGather extends SttTask {
}
/* common handler for all stt engine errors */
ep.addCustomEventListener(JambonzTranscriptionEvents.Error, this._onJambonzError.bind(this, cs, ep));
this.addCustomEventListener(ep, JambonzTranscriptionEvents.Error, this._onJambonzError.bind(this, cs, ep));
await ep.set(opts)
.catch((err) => this.logger.info(err, 'Error setting channel variables'));
}
@@ -528,7 +487,11 @@ class TaskGather extends SttTask {
this._clearTimer();
this._timeoutTimer = setTimeout(() => {
if (this.isContinuousAsr) this._startAsrTimer();
else this._resolve(this.digitBuffer.length >= this.minDigits ? 'dtmf-num-digits' : 'timeout');
else if (this.interDigitTimeout <= 0 ||
this.digitBuffer.length < this.minDigits ||
this.needsStt && this.digitBuffer.length === 0) {
this._resolve(this.digitBuffer.length >= this.minDigits ? 'dtmf-num-digits' : 'timeout');
}
}, this.timeout);
}
@@ -632,7 +595,8 @@ class TaskGather extends SttTask {
return;
}
evt = this.normalizeTranscription(evt, this.vendor, 1, this.language, this.shortUtterance);
evt = this.normalizeTranscription(evt, this.vendor, 1, this.language,
this.shortUtterance, this.data.recognizer.punctuation);
if (evt.alternatives.length === 0) {
this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, continue listening');
return;
@@ -801,7 +765,10 @@ class TaskGather extends SttTask {
async _onJambonzError(cs, ep, evt) {
this.logger.info({evt}, 'TaskGather:_onJambonzError');
if (this.isHandledByPrimaryProvider && this.fallbackVendor) {
ep.stopTranscription({vendor: this.vendor})
ep.stopTranscription({
vendor: this.vendor,
bugname: this.bugname
})
.catch((err) => this.logger.error({err}, `Error stopping transcription for primary vendor ${this.vendor}`));
const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
try {
@@ -882,7 +849,10 @@ class TaskGather extends SttTask {
'stt.result': JSON.stringify(evt)
});
if (this.needsStt && this.ep && this.ep.connected) {
this.ep.stopTranscription({vendor: this.vendor})
this.ep.stopTranscription({
vendor: this.vendor,
bugname: this.bugname
})
.catch((err) => this.logger.error({err}, 'Error stopping transcription'));
}

View File

@@ -16,6 +16,7 @@ class TaskRestDial extends Task {
this.to = this.data.to;
this.call_hook = this.data.call_hook;
this.timeout = this.data.timeout || 60;
this.sipRequestWithinDialogHook = this.data.sipRequestWithinDialogHook;
this.on('connect', this._onConnect.bind(this));
this.on('callStatus', this._onCallStatus.bind(this));
@@ -64,7 +65,7 @@ class TaskRestDial extends Task {
const cs = this.callSession;
cs.setDialog(dlg);
this.logger.debug('TaskRestDial:_onConnect - call connected');
if (this.sipRequestWithinDialogHook) this._initSipRequestWithinDialogHandler(cs, dlg);
try {
const b3 = this.getTracingPropagation();
const httpHeaders = b3 && {b3};
@@ -142,6 +143,16 @@ class TaskRestDial extends Task {
this.logger.error({err}, 'Rest:dial:_onAmdEvent - error calling actionHook');
});
}
_initSipRequestWithinDialogHandler(cs, dlg) {
cs.sipRequestWithinDialogHook = this.sipRequestWithinDialogHook;
dlg.on('info', this._onRequestWithinDialog.bind(this, cs));
dlg.on('message', this._onRequestWithinDialog.bind(this, cs));
}
async _onRequestWithinDialog(cs, req, res) {
cs._onRequestWithinDialog(req, res);
}
}
module.exports = TaskRestDial;

View File

@@ -59,7 +59,7 @@ class TaskSay extends Task {
}
}
async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label}) {
async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label, preCache = false}) {
const {srf} = cs;
const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, srf);
const {writeAlerts, AlertType, stats} = srf.locals;
@@ -76,6 +76,8 @@ class TaskSay extends Task {
voice = arr[1];
model = arr[2];
}
} else if (vendor === 'deepgram') {
model = voice;
}
/* allow for microsoft custom region voice and api_key to be specified as an override */
@@ -86,9 +88,16 @@ class TaskSay extends Task {
credentials.api_key = this.options.apiKey || credentials.apiKey;
credentials.region = this.options.region || credentials.region;
voice = this.options.voice || voice;
} else if (vendor === 'elevenlabs') {
credentials = credentials || {};
credentials.model_id = this.options.model_id || credentials.model_id;
credentials.voice_settings = this.options.voice_settings || {};
credentials.optimize_streaming_latency = this.options.optimize_streaming_latency
|| credentials.optimize_streaming_latency;
voice = this.options.voice_id || voice;
}
this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
if (!preCache) this.logger.info({vendor, language, voice, model}, 'TaskSay:exec');
try {
if (!credentials) {
writeAlerts({
@@ -111,11 +120,15 @@ class TaskSay extends Task {
if (text.startsWith('silence_stream://')) return text;
/* otel: trace time for tts */
const {span} = this.startChildSpan('tts-generation', {
'tts.vendor': vendor,
'tts.language': language,
'tts.voice': voice
});
let otelSpan;
if (!preCache) {
const {span} = this.startChildSpan('tts-generation', {
'tts.vendor': vendor,
'tts.language': language,
'tts.voice': voice
});
otelSpan = span;
}
try {
const {filePath, servedFromCache, rtt} = await synthAudio(stats, {
account_sid: cs.accountSid,
@@ -127,6 +140,7 @@ class TaskSay extends Task {
model,
salt,
credentials,
options: this.options,
disableTtsCache : this.disableTtsCache
});
this.logger.debug(`file ${filePath}, served from cache ${servedFromCache}`);
@@ -136,9 +150,9 @@ class TaskSay extends Task {
updateSpeechCredentialLastUsed(credentials.speech_credential_sid)
.catch(() => {/*already logged error */});
}
span.setAttributes({'tts.cached': servedFromCache});
span.end();
if (!servedFromCache && rtt) {
if (otelSpan) otelSpan.setAttributes({'tts.cached': servedFromCache});
if (otelSpan) otelSpan.end();
if (!servedFromCache && rtt && !preCache) {
this.notifyStatus({
event: 'synthesized-audio',
vendor,
@@ -150,7 +164,7 @@ class TaskSay extends Task {
return filePath;
} catch (err) {
this.logger.info({err}, 'Error synthesizing tts');
span.end();
if (otelSpan) otelSpan.end();
writeAlerts({
account_sid: cs.accountSid,
alert_type: AlertType.TTS_FAILURE,

View File

@@ -14,17 +14,15 @@ class SttTask extends Task {
const {
setChannelVarsForStt,
normalizeTranscription,
removeSpeechListeners,
setSpeechCredentialsAtRuntime,
compileSonioxTranscripts,
consolidateTranscripts
} = require('../utils/transcription-utils')(logger);
this.setChannelVarsForStt = setChannelVarsForStt;
this.normalizeTranscription = normalizeTranscription;
this.removeSpeechListeners = removeSpeechListeners;
this.compileSonioxTranscripts = compileSonioxTranscripts;
this.consolidateTranscripts = consolidateTranscripts;
this.eventHandlers = [];
this.isHandledByPrimaryProvider = true;
if (this.data.recognizer) {
const recognizer = this.data.recognizer;
@@ -49,11 +47,108 @@ class SttTask extends Task {
/* buffer for soniox transcripts */
this._sonioxTranscripts = [];
/*bug name prefix */
this.bugname_prefix = '';
}
async exec(cs, {ep, ep2}) {
super.exec(cs);
this.ep = ep;
this.ep2 = ep2;
// copy all value from config verb to this object.
if (cs.recognizer) {
for (const k in cs.recognizer) {
if (Array.isArray(this.data.recognizer[k]) ||
Array.isArray(cs.recognizer[k])) {
this.data.recognizer[k] = [
...this.data.recognizer[k],
...cs.recognizer[k]
];
} else if (typeof this.data.recognizer[k] === 'object' ||
typeof cs.recognizer[k] === 'object'
) {
this.data.recognizer[k] = {
...this.data.recognizer[k],
...cs.recognizer[k]
};
} else {
this.data.recognizer[k] = cs.recognizer[k] || this.data.recognizer[k];
}
}
}
if ('default' === this.vendor || !this.vendor) {
this.vendor = cs.speechRecognizerVendor;
if (this.data.recognizer) this.data.recognizer.vendor = this.vendor;
}
if ('default' === this.language || !this.language) {
this.language = cs.speechRecognizerLanguage;
if (this.data.recognizer) this.data.recognizer.language = this.language;
}
if ('default' === this.label || !this.label) {
this.label = cs.speechRecognizerLabel;
if (this.data.recognizer) this.data.recognizer.label = this.label;
}
// Fallback options
if ('default' === this.fallbackVendor || !this.fallbackVendor) {
this.fallbackVendor = cs.fallbackSpeechRecognizerVendor;
if (this.data.recognizer) this.data.recognizer.fallbackVendor = this.fallbackVendor;
}
if ('default' === this.fallbackLanguage || !this.fallbackLanguage) {
this.fallbackLanguage = cs.fallbackSpeechRecognizerLanguage;
if (this.data.recognizer) this.data.recognizer.fallbackLanguage = this.fallbackLanguage;
}
if ('default' === this.fallbackLabel || !this.fallbackLabel) {
this.fallbackLabel = cs.fallbackSpeechRecognizerLabel;
if (this.data.recognizer) this.data.recognizer.fallbackLabel = this.fallbackLabel;
}
if (!this.data.recognizer.vendor) {
this.data.recognizer.vendor = this.vendor;
}
if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
// By default, application saves cobalt model in language
this.data.recognizer.model = cs.speechRecognizerLanguage;
}
if (!this.sttCredentials) {
try {
this.sttCredentials = await this._initSpeechCredentials(this.cs, this.vendor, this.label);
} catch (error) {
if (this.fallbackVendor && this.isHandledByPrimaryProvider) {
await this._fallback();
} else {
throw error;
}
}
}
/* when using cobalt model is required */
if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
this.notifyError({ msg: 'ASR error', details:'Cobalt requires a model to be specified'});
throw new Error('Cobalt requires a model to be specified');
}
if (cs.hasAltLanguages) {
this.data.recognizer.altLanguages = this.data.recognizer.altLanguages.concat(cs.altLanguages);
this.logger.debug({altLanguages: this.altLanguages},
'STT:exec - applying altLanguages');
}
if (cs.hasGlobalSttPunctuation && !this.data.recognizer.punctuation) {
this.data.recognizer.punctuation = cs.globalSttPunctuation;
}
}
addCustomEventListener(ep, event, handler) {
this.eventHandlers.push({ep, event, handler});
ep.addCustomEventListener(event, handler);
}
removeCustomEventListeners() {
this.eventHandlers.forEach((h) => h.ep.removeCustomEventListener(h.event, h.handler));
}
async _initSpeechCredentials(cs, vendor, label) {
const {getNuanceAccessToken, getIbmAccessToken} = this.cs.srf.locals.dbHelpers;
const {getNuanceAccessToken, getIbmAccessToken} = cs.srf.locals.dbHelpers;
let credentials = cs.getSpeechCredentials(vendor, 'stt', label);
if (!credentials) {

View File

@@ -12,7 +12,7 @@ class TaskTag extends Task {
async exec(cs) {
super.exec(cs);
cs.callInfo.customerData = this.data;
//this.logger.debug({callInfo: cs.callInfo.toJSON()}, 'TaskTag:exec set customer data in callInfo');
this.logger.debug({customerData: cs.callInfo.customerData}, 'TaskTag:exec set customer data in callInfo');
}
}

View File

@@ -33,19 +33,29 @@ class TaskTranscribe extends SttTask {
this.childSpan = [null, null];
// Continuos asr timeout
// Continuous asr timeout
this.asrTimeout = typeof this.data.recognizer.asrTimeout === 'number' ? this.data.recognizer.asrTimeout * 1000 : 0;
if (this.asrTimeout > 0) {
this.isContinuousAsr = true;
}
/* buffer speech for continuous asr */
this._bufferedTranscripts = [];
this.bugname_prefix = 'transcribe_';
}
get name() { return TaskName.Transcribe; }
async exec(cs, {ep, ep2}) {
super.exec(cs);
await super.exec(cs, {ep, ep2});
if (this.data.recognizer.vendor === 'nuance') {
this.data.recognizer.nuanceOptions = {
// by default, nuance STT will recognize only 1st utterance.
// enable multiple allow nuance detact all utterances
utteranceDetectionMode: 'multiple',
...this.data.recognizer.nuanceOptions
};
}
const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
if (cs.hasGlobalSttHints) {
@@ -55,67 +65,6 @@ class TaskTranscribe extends SttTask {
this.logger.debug({hints: this.data.recognizer.hints, hintsBoost: this.data.recognizer.hintsBoost},
'Transcribe:exec - applying global sttHints');
}
if (cs.hasAltLanguages) {
this.data.recognizer.altLanguages = this.data.recognizer.altLanguages.concat(cs.altLanguages);
this.logger.debug({altLanguages: this.altLanguages},
'Transcribe:exec - applying altLanguages');
}
if (cs.hasGlobalSttPunctuation && !this.data.recognizer.punctuation) {
this.data.recognizer.punctuation = cs.globalSttPunctuation;
}
this.ep = ep;
this.ep2 = ep2;
if ('default' === this.vendor || !this.vendor) {
this.vendor = cs.speechRecognizerVendor;
if (this.data.recognizer) this.data.recognizer.vendor = this.vendor;
}
if ('default' === this.language || !this.language) {
this.language = cs.speechRecognizerLanguage;
if (this.data.recognizer) this.data.recognizer.language = this.language;
}
if ('default' === this.label || !this.label) {
this.label = cs.speechRecognizerLabel;
if (this.data.recognizer) this.data.recognizer.label = this.label;
}
// Fallback options
if ('default' === this.fallbackVendor || !this.fallbackVendor) {
this.fallbackVendor = cs.fallbackSpeechRecognizerVendor;
if (this.data.recognizer) this.data.recognizer.fallbackVendor = this.fallbackVendor;
}
if ('default' === this.fallbackLanguage || !this.fallbackLanguage) {
this.fallbackLanguage = cs.fallbackSpeechRecognizerLanguage;
if (this.data.recognizer) this.data.recognizer.fallbackLanguage = this.fallbackLanguage;
}
if ('default' === this.fallbackLabel || !this.fallbackLabel) {
this.fallbackLabel = cs.fallbackSpeechRecognizerLabel;
if (this.data.recognizer) this.data.recognizer.fallbackLabel = this.fallbackLabel;
}
if (!this.data.recognizer.vendor) {
this.data.recognizer.vendor = this.vendor;
}
if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
// By default, application saves cobalt model in language
this.data.recognizer.model = cs.speechRecognizerLanguage;
}
if (!this.sttCredentials) {
try {
this.sttCredentials = await this._initSpeechCredentials(cs, this.vendor, this.label);
} catch (error) {
if (this.fallbackVendor && this.isHandledByPrimaryProvider) {
await this._fallback();
} else {
throw error;
}
}
}
/* when using cobalt model is required */
if (this.vendor === 'cobalt' && !this.data.recognizer.model) {
this.notifyError({ msg: 'ASR error', details:'Cobalt requires a model to be specified'});
throw new Error('Cobalt requires a model to be specified');
}
try {
await this._startTranscribing(cs, ep, 1);
@@ -131,14 +80,17 @@ class TaskTranscribe extends SttTask {
this.logger.info(err, 'TaskTranscribe:exec - error');
this.parentTask && this.parentTask.emit('error', err);
}
this.removeSpeechListeners(ep);
this.removeCustomEventListeners();
}
async _stopTranscription() {
let stopTranscription = false;
if (this.ep?.connected) {
stopTranscription = true;
this.ep.stopTranscription({vendor: this.vendor})
this.ep.stopTranscription({
vendor: this.vendor,
bugname: this.bugname
})
.catch((err) => this.logger.info(err, 'Error TaskTranscribe:kill'));
}
if (this.separateRecognitionPerChannel && this.ep2 && this.ep2.connected) {
@@ -189,48 +141,44 @@ class TaskTranscribe extends SttTask {
const opts = this.setChannelVarsForStt(this, this.sttCredentials, this.data.recognizer);
switch (this.vendor) {
case 'google':
this.bugname = 'google_transcribe';
ep.addCustomEventListener(GoogleTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}google_transcribe`;
this.addCustomEventListener(ep, GoogleTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
ep.addCustomEventListener(GoogleTranscriptionEvents.NoAudioDetected,
this.addCustomEventListener(ep, GoogleTranscriptionEvents.NoAudioDetected,
this._onNoAudio.bind(this, cs, ep, channel));
ep.addCustomEventListener(GoogleTranscriptionEvents.MaxDurationExceeded,
this.addCustomEventListener(ep, GoogleTranscriptionEvents.MaxDurationExceeded,
this._onMaxDurationExceeded.bind(this, cs, ep, channel));
break;
case 'aws':
case 'polly':
this.bugname = 'aws_transcribe';
ep.addCustomEventListener(AwsTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}aws_transcribe`;
this.addCustomEventListener(ep, AwsTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
ep.addCustomEventListener(AwsTranscriptionEvents.NoAudioDetected,
this.addCustomEventListener(ep, AwsTranscriptionEvents.NoAudioDetected,
this._onNoAudio.bind(this, cs, ep, channel));
ep.addCustomEventListener(AwsTranscriptionEvents.MaxDurationExceeded,
this.addCustomEventListener(ep, AwsTranscriptionEvents.MaxDurationExceeded,
this._onMaxDurationExceeded.bind(this, cs, ep, channel));
break;
case 'microsoft':
this.bugname = 'azure_transcribe';
ep.addCustomEventListener(AzureTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}azure_transcribe`;
this.addCustomEventListener(ep, AzureTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
ep.addCustomEventListener(AzureTranscriptionEvents.NoSpeechDetected,
this.addCustomEventListener(ep, AzureTranscriptionEvents.NoSpeechDetected,
this._onNoAudio.bind(this, cs, ep, channel));
break;
case 'nuance':
this.bugname = 'nuance_transcribe';
ep.addCustomEventListener(NuanceTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}nuance_transcribe`;
this.addCustomEventListener(ep, NuanceTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
ep.addCustomEventListener(NuanceTranscriptionEvents.StartOfSpeech,
this._onStartOfSpeech.bind(this, cs, ep, channel));
ep.addCustomEventListener(NuanceTranscriptionEvents.TranscriptionComplete,
this._onTranscriptionComplete.bind(this, cs, ep, channel));
break;
case 'deepgram':
this.bugname = 'deepgram_transcribe';
ep.addCustomEventListener(DeepgramTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}deepgram_transcribe`;
this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
ep.addCustomEventListener(DeepgramTranscriptionEvents.Connect,
this.addCustomEventListener(ep, DeepgramTranscriptionEvents.Connect,
this._onVendorConnect.bind(this, cs, ep));
ep.addCustomEventListener(DeepgramTranscriptionEvents.ConnectFailure,
this.addCustomEventListener(ep, DeepgramTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep, channel));
/* if app sets deepgramOptions.utteranceEndMs they essentially want continuous asr */
@@ -238,13 +186,13 @@ class TaskTranscribe extends SttTask {
break;
case 'soniox':
this.bugname = 'soniox_transcribe';
ep.addCustomEventListener(SonioxTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}soniox_transcribe`;
this.addCustomEventListener(ep, SonioxTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
break;
case 'cobalt':
this.bugname = 'cobalt_transcribe';
ep.addCustomEventListener(CobaltTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}cobalt_transcribe`;
this.addCustomEventListener(ep, CobaltTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
/* cobalt doesnt have language, it has model, which is required */
@@ -273,45 +221,40 @@ class TaskTranscribe extends SttTask {
break;
case 'ibm':
this.bugname = 'ibm_transcribe';
ep.addCustomEventListener(IbmTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}ibm_transcribe`;
this.addCustomEventListener(ep, IbmTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
ep.addCustomEventListener(IbmTranscriptionEvents.Connect,
this.addCustomEventListener(ep, IbmTranscriptionEvents.Connect,
this._onVendorConnect.bind(this, cs, ep));
ep.addCustomEventListener(IbmTranscriptionEvents.ConnectFailure,
this.addCustomEventListener(ep, IbmTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep, channel));
break;
case 'nvidia':
this.bugname = 'nvidia_transcribe';
ep.addCustomEventListener(NvidiaTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep));
ep.addCustomEventListener(NvidiaTranscriptionEvents.StartOfSpeech,
this._onStartOfSpeech.bind(this, cs, ep));
ep.addCustomEventListener(NvidiaTranscriptionEvents.TranscriptionComplete,
this._onTranscriptionComplete.bind(this, cs, ep));
ep.addCustomEventListener(NvidiaTranscriptionEvents.VadDetected,
this._onVadDetected.bind(this, cs, ep));
this.bugname = `${this.bugname_prefix}nvidia_transcribe`;
this.addCustomEventListener(ep, NvidiaTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
break;
case 'assemblyai':
this.bugname = 'assemblyai_transcribe';
ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}assemblyai_transcribe`;
this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
ep.addCustomEventListener(AssemblyAiTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
ep.addCustomEventListener(AssemblyAiTranscriptionEvents.ConnectFailure,
this.addCustomEventListener(ep,
AssemblyAiTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.Error, this._onVendorError.bind(this, cs, ep));
this.addCustomEventListener(ep, AssemblyAiTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep, channel));
break;
default:
if (this.vendor.startsWith('custom:')) {
this.bugname = `${this.vendor}_transcribe`;
ep.addCustomEventListener(JambonzTranscriptionEvents.Transcription,
this.bugname = `${this.bugname_prefix}${this.vendor}_transcribe`;
this.addCustomEventListener(ep, JambonzTranscriptionEvents.Transcription,
this._onTranscription.bind(this, cs, ep, channel));
ep.addCustomEventListener(JambonzTranscriptionEvents.Connect, this._onJambonzConnect.bind(this, cs, ep));
ep.addCustomEventListener(JambonzTranscriptionEvents.ConnectFailure,
this._onJambonzConnectFailure.bind(this, cs, ep));
this.addCustomEventListener(ep, JambonzTranscriptionEvents.Connect, this._onVendorConnect.bind(this, cs, ep));
this.addCustomEventListener(ep, JambonzTranscriptionEvents.ConnectFailure,
this._onVendorConnectFailure.bind(this, cs, ep));
break;
}
else {
@@ -322,7 +265,7 @@ class TaskTranscribe extends SttTask {
}
/* common handler for all stt engine errors */
ep.addCustomEventListener(JambonzTranscriptionEvents.Error, this._onJambonzError.bind(this, cs, ep));
this.addCustomEventListener(ep, JambonzTranscriptionEvents.Error, this._onJambonzError.bind(this, cs, ep));
await ep.set(opts)
.catch((err) => this.logger.info(err, 'Error setting channel variables'));
}
@@ -337,6 +280,8 @@ class TaskTranscribe extends SttTask {
}
async _transcribe(ep) {
this.logger.debug(
`TaskTranscribe:_transcribe - starting transcription vendor ${this.vendor} bugname ${this.bugname}`);
await ep.startTranscription({
vendor: this.vendor,
interim: this.interim ? true : false,
@@ -369,7 +314,8 @@ class TaskTranscribe extends SttTask {
}
this.logger.debug({evt}, 'TaskTranscribe:_onTranscription - before normalization');
evt = this.normalizeTranscription(evt, this.vendor, channel, this.language);
evt = this.normalizeTranscription(evt, this.vendor, channel, this.language, undefined,
this.data.recognizer.punctuation);
this.logger.debug({evt}, 'TaskTranscribe:_onTranscription');
if (evt.alternatives.length === 0) {
this.logger.info({evt}, 'TaskTranscribe:_onTranscription - got empty transcript, continue listening');
@@ -495,7 +441,10 @@ class TaskTranscribe extends SttTask {
async _onJambonzError(cs, _ep, evt) {
this.logger.info({evt}, 'TaskTranscribe:_onJambonzError');
if (this.isHandledByPrimaryProvider && this.fallbackVendor) {
_ep.stopTranscription({vendor: this.vendor})
_ep.stopTranscription({
vendor: this.vendor,
bugname: this.bugname
})
.catch((err) => this.logger.error({err}, `Error stopping transcription for primary vendor ${this.vendor}`));
const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
try {

View File

@@ -0,0 +1,197 @@
const { normalizeJambones } = require('@jambonz/verb-specifications');
const makeTask = require('../tasks/make_task');
const { JAMBONZ_RECORD_WS_BASE_URL, JAMBONZ_RECORD_WS_USERNAME, JAMBONZ_RECORD_WS_PASSWORD } = require('../config');
const Emitter = require('events');
class BackgroundTaskManager extends Emitter {
constructor({cs, logger, rootSpan}) {
super();
this.tasks = new Map();
this.cs = cs;
this.logger = logger;
this.rootSpan = rootSpan;
}
isTaskRunning(type) {
return this.tasks.has(type);
}
getTask(type) {
if (this.tasks.has(type)) {
return this.tasks.get(type);
}
}
count() {
return this.tasks.size;
}
async newTask(type, taskOpts) {
this.logger.info({taskOpts}, `initiating Background task ${type}`);
if (this.tasks.has(type)) {
this.logger.info(`Background task ${type} is running, skiped`);
return;
}
let task;
switch (type) {
case 'listen':
task = await this._initListen(taskOpts);
break;
case 'bargeIn':
task = await this._initBargeIn(taskOpts);
break;
case 'record':
task = await this._initRecord();
break;
case 'transcribe':
task = await this._initTranscribe(taskOpts);
break;
default:
break;
}
if (task) {
this.tasks.set(type, task);
}
return task;
}
stop(type) {
const task = this.getTask(type);
if (task) {
this.logger.info(`stopping background task: ${type}`);
task.removeAllListeners();
task.span.end();
task.kill();
// Remove task from managed List
this.tasks.delete(type);
} else {
this.logger.info(`stopping background task, ${type} is not running, skipped`);
}
}
stopAll() {
this.logger.info('BackgroundTaskManager:stopAll');
for (const key of this.tasks.keys()) {
this.stop(key);
}
}
// Initiate Task
// Initiate Listen
async _initListen(opts, bugname = 'jambonz-background-listen', ignoreCustomerData = false, type = 'listen') {
let task;
try {
const t = normalizeJambones(this.logger, [opts]);
task = makeTask(this.logger, t[0]);
task.bugname = bugname;
task.ignoreCustomerData = ignoreCustomerData;
const resources = await this.cs._evaluatePreconditions(task);
const {span, ctx} = this.rootSpan.startChildSpan(`background-${type}:${task.summary}`);
task.span = span;
task.ctx = ctx;
task.exec(this.cs, resources)
.then(this._taskCompleted.bind(this, type, task))
.catch(this._taskError.bind(this, type, task));
} catch (err) {
this.logger.info({err, opts}, `BackgroundTaskManager:_initListen - Error creating ${bugname} task`);
}
return task;
}
// Initiate Gather
async _initBargeIn(opts) {
let task;
try {
const t = normalizeJambones(this.logger, [opts]);
task = makeTask(this.logger, t[0]);
task
.once('dtmf', this._bargeInTaskCompleted.bind(this))
.once('vad', this._bargeInTaskCompleted.bind(this))
.once('transcription', this._bargeInTaskCompleted.bind(this))
.once('timeout', this._bargeInTaskCompleted.bind(this));
const resources = await this.cs._evaluatePreconditions(task);
const {span, ctx} = this.rootSpan.startChildSpan(`background-bargeIn:${task.summary}`);
task.span = span;
task.ctx = ctx;
task.bugname_prefix = 'background_bargeIn_';
task.exec(this.cs, resources)
.then(() => {
this._taskCompleted('bargeIn', task);
if (task.sticky && !this.cs.callGone && !this.cs._stopping) {
this.logger.info('BackgroundTaskManager:_initBargeIn: restarting background bargeIn');
this.newTask('bargeIn', opts);
}
return;
})
.catch(this._taskError.bind(this, 'bargeIn', task));
} catch (err) {
this.logger.info(err, 'BackgroundTaskManager:_initGather - Error creating bargeIn task');
}
return task;
}
// Initiate Record
async _initRecord() {
if (this.cs.accountInfo.account.record_all_calls || this.cs.application.record_all_calls) {
if (!JAMBONZ_RECORD_WS_BASE_URL || !this.cs.accountInfo.account.bucket_credential) {
this.logger.error(`_initRecord: invalid configuration,
missing JAMBONZ_RECORD_WS_BASE_URL or bucket configuration`);
return undefined;
}
const listenOpts = {
url: `${JAMBONZ_RECORD_WS_BASE_URL}/record/${this.cs.accountInfo.account.bucket_credential.vendor}`,
disableBidirectionalAudio: true,
mixType : 'stereo',
passDtmf: true
};
if (JAMBONZ_RECORD_WS_USERNAME && JAMBONZ_RECORD_WS_PASSWORD) {
listenOpts.wsAuth = {
username: JAMBONZ_RECORD_WS_USERNAME,
password: JAMBONZ_RECORD_WS_PASSWORD
};
}
this.logger.debug({listenOpts}, '_initRecord: enabling listen');
return await this._initListen({verb: 'listen', ...listenOpts}, 'jambonz-session-record', true, 'record');
}
}
// Initiate Transcribe
async _initTranscribe(opts) {
let task;
try {
const t = normalizeJambones(this.logger, [opts]);
task = makeTask(this.logger, t[0]);
const resources = await this.cs._evaluatePreconditions(task);
const {span, ctx} = this.rootSpan.startChildSpan(`background-transcribe:${task.summary}`);
task.span = span;
task.ctx = ctx;
task.bugname_prefix = 'background_transcribe_';
task.exec(this.cs, resources)
.then(this._taskCompleted.bind(this, 'transcribe', task))
.catch(this._taskError.bind(this, 'transcribe', task));
} catch (err) {
this.logger.info(err, 'BackgroundTaskManager:_initTranscribe - Error creating transcribe task');
}
return task;
}
_taskCompleted(type, task) {
this.logger.info({type, task}, 'BackgroundTaskManager:_taskCompleted: task completed');
task.removeAllListeners();
task.span.end();
this.tasks.delete(type);
}
_taskError(type, task, error) {
this.logger.info({type, task, error}, 'BackgroundTaskManager:_taskError: task Error');
task.removeAllListeners();
task.span.end();
this.tasks.delete(type);
}
_bargeInTaskCompleted(evt) {
this.logger.info({evt}, 'BackgroundTaskManager:_bargeInTaskCompleted on event from background bargeIn');
this.emit('bargeIn-done', evt);
}
}
module.exports = BackgroundTaskManager;

View File

@@ -91,9 +91,14 @@ const speechMapper = (cred) => {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
obj.model_id = o.model_id;
obj.options = o.options;
} else if ('assemblyai' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
} else if ('whisper' === obj.vendor) {
const o = JSON.parse(decrypt(credential));
obj.api_key = o.api_key;
obj.model_id = o.model_id;
} else if (obj.vendor.startsWith('custom:')) {
const o = JSON.parse(decrypt(credential));
obj.auth_token = o.auth_token;

View File

@@ -14,6 +14,7 @@ const {
HTTP_PROXY_PORT,
HTTP_PROXY_PROTOCOL,
NODE_ENV,
HTTP_USER_AGENT_HEADER,
} = require('../config');
const toBase64 = (str) => Buffer.from(str || '', 'utf8').toString('base64');
@@ -116,6 +117,10 @@ class HttpRequestor extends BaseRequestor {
const url = hook.url || hook;
const method = hook.method || 'POST';
let buf = '';
httpHeaders = {
...httpHeaders,
...(HTTP_USER_AGENT_HEADER && {'user-agent' : HTTP_USER_AGENT_HEADER})
};
assert.ok(url, 'HttpRequestor:request url was not provided');
assert.ok, (['GET', 'POST'].includes(method), `HttpRequestor:request method must be 'GET' or 'POST' not ${method}`);

View File

@@ -8,9 +8,6 @@ const {
JAMBONES_MYSQL_CONNECTION_LIMIT,
JAMBONES_MYSQL_PORT,
JAMBONES_FREESWITCH,
JAMBONES_REDIS_HOST,
JAMBONES_REDIS_PORT,
JAMBONES_REDIS_SENTINELS,
SMPP_URL,
JAMBONES_TIME_SERIES_HOST,
JAMBONES_ESL_LISTEN_ADDRESS,
@@ -140,7 +137,8 @@ function installSrfLocals(srf, logger) {
lookupTeamsByAccount,
lookupAccountBySid,
lookupAccountCapacitiesBySid,
lookupSmppGateways
lookupSmppGateways,
lookupClientByAccountAndUsername
} = require('@jambonz/db-helpers')({
host: JAMBONES_MYSQL_HOST,
user: JAMBONES_MYSQL_USER,
@@ -174,19 +172,13 @@ function installSrfLocals(srf, logger) {
retrieveByPatternSortedSet,
sortedSetLength,
sortedSetPositionByPattern
} = require('@jambonz/realtimedb-helpers')(JAMBONES_REDIS_SENTINELS || {
host: JAMBONES_REDIS_HOST,
port: JAMBONES_REDIS_PORT || 6379
}, logger, tracer);
} = require('@jambonz/realtimedb-helpers')({}, logger, tracer);
const registrar = new Registrar(logger, client);
const {
synthAudio,
getNuanceAccessToken,
getIbmAccessToken,
} = require('@jambonz/speech-utils')(JAMBONES_REDIS_SENTINELS || {
host: JAMBONES_REDIS_HOST,
port: JAMBONES_REDIS_PORT || 6379
}, logger, tracer);
} = require('@jambonz/speech-utils')({}, logger);
const {
writeAlerts,
AlertType
@@ -217,6 +209,7 @@ function installSrfLocals(srf, logger) {
lookupAccountBySid,
lookupAccountCapacitiesBySid,
lookupSmppGateways,
lookupClientByAccountAndUsername,
updateCallStatus,
retrieveCall,
listCalls,

View File

@@ -13,6 +13,9 @@ const moment = require('moment');
const stripCodecs = require('./strip-ancillary-codecs');
const RootSpan = require('./call-tracer');
const uuidv4 = require('uuid-random');
const HttpRequestor = require('./http-requestor');
const WsRequestor = require('./ws-requestor');
const {makeOpusFirst} = require('./sdp-utils');
class SingleDialer extends Emitter {
constructor({logger, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan, dialTask}) {
@@ -76,7 +79,8 @@ class SingleDialer extends Emitter {
...(this.from.host && {'X-Preferred-From-Host': this.from.host}),
'X-Jambonz-Routing': this.target.type,
'X-Call-Sid': this.callSid,
...(this.applicationSid && {'X-Application-Sid': this.applicationSid})
...(this.applicationSid && {'X-Application-Sid': this.applicationSid}),
...(this.target.proxy && {'X-SIP-Proxy': this.target.proxy})
};
if (srf.locals.fsUUID) {
opts.headers = {
@@ -148,7 +152,7 @@ class SingleDialer extends Emitter {
Object.assign(opts, {
proxy: `sip:${this.sbcAddress}`,
localSdp: this.ep.local.sdp
localSdp: opts.opusFirst ? makeOpusFirst(this.ep.local.sdp) : this.ep.local.sdp
});
if (this.target.auth) opts.auth = this.target.auth;
inviteSpan = this.startSpan('invite', {
@@ -176,6 +180,7 @@ class SingleDialer extends Emitter {
* (a) create a logger for this call
*/
req.srf = srf;
this.req = req;
this.callInfo = new CallInfo({
direction: CallDirection.Outbound,
parentCallInfo: this.parentCallInfo,
@@ -334,6 +339,7 @@ class SingleDialer extends Emitter {
// verify it contains only allowed verbs
const allowedTasks = tasks.filter((task) => {
return [
TaskPreconditions.None,
TaskPreconditions.StableCall,
TaskPreconditions.Endpoint
].includes(task.preconditions);
@@ -381,15 +387,35 @@ class SingleDialer extends Emitter {
this.dlg.linkedSpanId = this.rootSpan.traceId;
const rootSpan = new RootSpan('outbound-call', this.dlg);
const newLogger = logger.child({traceId: rootSpan.traceId});
//clone application from parent call with new requestor
//parrent application will be closed in case the parent hangup
const app = {...application};
if ('WS' === app.call_hook?.method ||
app.call_hook?.url.startsWith('ws://') || app.call_hook?.url.startsWith('wss://')) {
const requestor = new WsRequestor(logger, this.accountInfo.account.account_sid,
app.call_hook, this.accountInfo.account.webhook_secret);
app.requestor = requestor;
app.notifier = requestor;
app.call_hook.method = 'WS';
}
else {
app.requestor = new HttpRequestor(logger, this.accountInfo.account.account_sid,
app.call_hook, this.accountInfo.account.webhook_secret);
if (app.call_status_hook) app.notifier = new HttpRequestor(logger,
this.accountInfo.account.account_sid, app.call_status_hook,
this.accountInfo.account.webhook_secret);
else app.notifier = {request: () => {}, close: () => {}};
}
const cs = new AdultingCallSession({
logger: newLogger,
singleDialer: this,
application,
application: app,
callInfo: this.callInfo,
accountInfo: this.accountInfo,
tasks,
rootSpan
});
cs.req = this.req;
cs.exec().catch((err) => newLogger.error({err}, 'doAdulting: error executing session'));
return cs;
}

View File

@@ -12,6 +12,30 @@ const mergeSdpMedia = (sdp1, sdp2) => {
return sdpTransform.write(parsedSdp1);
};
const getCodecPlacement = (parsedSdp, codec) => parsedSdp?.media[0]?.rtp?.findIndex((e) => e.codec === codec);
const isOpusFirst = (sdp) => {
return getCodecPlacement(sdpTransform.parse(sdp), 'opus') === 0;
};
const makeOpusFirst = (sdp) => {
const parsedSdp = sdpTransform.parse(sdp);
// Find the index of the OPUS codec
const opusIndex = getCodecPlacement(parsedSdp, 'opus');
// Move OPUS codec to the beginning
if (opusIndex > 0) {
const opusEntry = parsedSdp.media[0].rtp.splice(opusIndex, 1)[0];
parsedSdp.media[0].rtp.unshift(opusEntry);
// Also move the corresponding payload type in the "m" line
const opusPayloadType = parsedSdp.media[0].payloads.split(' ')[opusIndex];
const otherPayloadTypes = parsedSdp.media[0].payloads.split(' ').filter((pt) => pt != opusPayloadType);
parsedSdp.media[0].payloads = [opusPayloadType, ...otherPayloadTypes].join(' ');
}
return sdpTransform.write(parsedSdp);
};
const extractSdpMedia = (sdp) => {
const parsedSdp1 = sdpTransform.parse(sdp);
if (parsedSdp1.media.length > 1) {
@@ -28,5 +52,7 @@ const extractSdpMedia = (sdp) => {
module.exports = {
isOnhold,
mergeSdpMedia,
extractSdpMedia
extractSdpMedia,
isOpusFirst,
makeOpusFirst
};

View File

@@ -97,8 +97,12 @@ const parseSiprecPayload = (req, logger) => {
obj[`${prefix}participantstreamassoc`].forEach((ps) => {
const part = participants[ps.$.participant_id];
if (part) {
part.send = ps[`${prefix}send`][0];
part.recv = ps[`${prefix}recv`][0];
if (ps.hasOwnProperty(`${prefix}send`)) {
part.send = ps[`${prefix}send`][0];
}
if (ps.hasOwnProperty(`${prefix}recv`)) {
part.recv = ps[`${prefix}recv`][0];
}
}
});
}
@@ -109,9 +113,9 @@ const parseSiprecPayload = (req, logger) => {
obj[`${prefix}stream`].forEach((s) => {
const streamId = s.$.stream_id;
let sender;
for (const [k, v] of Object.entries(participants)) {
for (const v of Object.values(participants)) {
if (v.send === streamId) {
sender = k;
sender = v;
break;
}
}
@@ -121,9 +125,15 @@ const parseSiprecPayload = (req, logger) => {
sender.label = s[`${prefix}label`][0];
if (-1 !== ['1', 'a_leg', 'inbound'].indexOf(sender.label)) {
opts.caller.aor = sender.aor ;
if (-1 !== ['1', 'a_leg', 'inbound', '10'].indexOf(sender.label)) {
opts.caller.aor = sender.aor;
if (sender.name) opts.caller.name = sender.name;
// Remap the sdp stream base on sender label
if (!opts.sdp1.includes(`a=label:${sender.label}`)) {
const tmp = opts.sdp1;
opts.sdp1 = opts.sdp2;
opts.sdp2 = tmp;
}
}
else {
opts.callee.aor = sender.aor ;

View File

@@ -1,15 +1,5 @@
const {
TaskName,
AzureTranscriptionEvents,
GoogleTranscriptionEvents,
AwsTranscriptionEvents,
NuanceTranscriptionEvents,
DeepgramTranscriptionEvents,
SonioxTranscriptionEvents,
NvidiaTranscriptionEvents,
CobaltTranscriptionEvents,
JambonzTranscriptionEvents,
AssemblyAiTranscriptionEvents
} = require('./constants.json');
const stickyVars = {
@@ -338,19 +328,20 @@ const normalizeNuance = (evt, channel, language) => {
};
};
const normalizeMicrosoft = (evt, channel, language) => {
const normalizeMicrosoft = (evt, channel, language, punctuation = true) => {
const copy = JSON.parse(JSON.stringify(evt));
const nbest = evt.NBest;
const language_code = evt.PrimaryLanguage?.Language || language;
const alternatives = nbest ? nbest.map((n) => {
return {
confidence: n.Confidence,
transcript: n.Display
// remove all puntuation if needed
transcript: punctuation ? n.Display : n.Display.replace(/\p{P}/gu, '')
};
}) :
[
{
transcript: evt.DisplayText || evt.Text
transcript: punctuation ? evt.DisplayText || evt.Text : (evt.DisplayText || evt.Text).replace(/\p{P}/gu, '')
}
];
@@ -400,14 +391,14 @@ const normalizeAssemblyAi = (evt, channel, language) => {
};
module.exports = (logger) => {
const normalizeTranscription = (evt, vendor, channel, language, shortUtterance) => {
const normalizeTranscription = (evt, vendor, channel, language, shortUtterance, punctuation) => {
//logger.debug({ evt, vendor, channel, language }, 'normalizeTranscription');
switch (vendor) {
case 'deepgram':
return normalizeDeepgram(evt, channel, language, shortUtterance);
case 'microsoft':
return normalizeMicrosoft(evt, channel, language);
return normalizeMicrosoft(evt, channel, language, punctuation);
case 'google':
return normalizeGoogle(evt, channel, language);
case 'aws':
@@ -473,7 +464,8 @@ module.exports = (logger) => {
...(rOpts.hints?.length > 0 && typeof rOpts.hints[0] === 'object' &&
{GOOGLE_SPEECH_HINTS: JSON.stringify(rOpts.hints)}),
...(typeof rOpts.hintsBoost === 'number' && {GOOGLE_SPEECH_HINTS_BOOST: rOpts.hintsBoost}),
...(rOpts.altLanguages?.length > 0 &&
// When altLanguages is emptylist, we have to send value to freeswitch to clear the previous settings
...(rOpts.altLanguages &&
{GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES: [...new Set(rOpts.altLanguages)].join(',')}),
...(rOpts.interactionType &&
{GOOGLE_SPEECH_METADATA_INTERACTION_TYPE: rOpts.interactionType}),
@@ -503,7 +495,8 @@ module.exports = (logger) => {
{AZURE_SPEECH_HINTS: rOpts.hints.map((h) => h.trim()).join(',')}),
...(rOpts.hints?.length > 0 && typeof rOpts.hints[0] === 'object' &&
{AZURE_SPEECH_HINTS: rOpts.hints.map((h) => h.phrase).join(',')}),
...(rOpts.altLanguages && rOpts.altLanguages.length > 0 &&
// When altLanguages is emptylist, we have to send value to freeswitch to clear the previous settings
...(rOpts.altLanguages &&
{AZURE_SPEECH_ALTERNATIVE_LANGUAGE_CODES: [...new Set(rOpts.altLanguages)].join(',')}),
...(rOpts.requestSnr && {AZURE_REQUEST_SNR: 1}),
...(rOpts.profanityOption && {AZURE_PROFANITY_OPTION: rOpts.profanityOption}),
@@ -612,7 +605,7 @@ module.exports = (logger) => {
...(deepgramOptions.keywords) &&
{DEEPGRAM_SPEECH_KEYWORDS: deepgramOptions.keywords.join(',')},
...('endpointing' in deepgramOptions) &&
{DEEPGRAM_SPEECH_ENDPOINTING: deepgramOptions.endpointing},
{DEEPGRAM_SPEECH_ENDPOINTING: deepgramOptions.endpointing === false ? 'false' : deepgramOptions.endpointing},
...(deepgramOptions.utteranceEndMs) &&
{DEEPGRAM_SPEECH_UTTERANCE_END_MS: deepgramOptions.utteranceEndMs},
...(deepgramOptions.vadTurnoff) &&
@@ -740,7 +733,7 @@ module.exports = (logger) => {
opts = {
...opts,
JAMBONZ_STT_API_KEY: auth_token,
...(auth_token && {JAMBONZ_STT_API_KEY: auth_token}),
JAMBONZ_STT_URL: custom_stt_url,
...(Object.keys(options).length > 0 && {JAMBONZ_STT_OPTIONS: JSON.stringify(options)}),
};
@@ -752,48 +745,6 @@ module.exports = (logger) => {
return opts;
};
const removeSpeechListeners = (ep) => {
ep.removeCustomEventListener(GoogleTranscriptionEvents.Transcription);
ep.removeCustomEventListener(GoogleTranscriptionEvents.EndOfUtterance);
ep.removeCustomEventListener(GoogleTranscriptionEvents.VadDetected);
ep.removeCustomEventListener(AwsTranscriptionEvents.Transcription);
ep.removeCustomEventListener(AwsTranscriptionEvents.VadDetected);
ep.removeCustomEventListener(AzureTranscriptionEvents.Transcription);
ep.removeCustomEventListener(AzureTranscriptionEvents.NoSpeechDetected);
ep.removeCustomEventListener(AzureTranscriptionEvents.VadDetected);
ep.removeCustomEventListener(NuanceTranscriptionEvents.Transcription);
ep.removeCustomEventListener(NuanceTranscriptionEvents.TranscriptionComplete);
ep.removeCustomEventListener(NuanceTranscriptionEvents.StartOfSpeech);
ep.removeCustomEventListener(NuanceTranscriptionEvents.VadDetected);
ep.removeCustomEventListener(DeepgramTranscriptionEvents.Transcription);
ep.removeCustomEventListener(DeepgramTranscriptionEvents.Connect);
ep.removeCustomEventListener(DeepgramTranscriptionEvents.ConnectFailure);
ep.removeCustomEventListener(SonioxTranscriptionEvents.Transcription);
ep.removeCustomEventListener(CobaltTranscriptionEvents.Transcription);
ep.removeCustomEventListener(CobaltTranscriptionEvents.CompileContext);
ep.removeCustomEventListener(NvidiaTranscriptionEvents.Transcription);
ep.removeCustomEventListener(NvidiaTranscriptionEvents.TranscriptionComplete);
ep.removeCustomEventListener(NvidiaTranscriptionEvents.StartOfSpeech);
ep.removeCustomEventListener(NvidiaTranscriptionEvents.VadDetected);
ep.removeCustomEventListener(JambonzTranscriptionEvents.Transcription);
ep.removeCustomEventListener(JambonzTranscriptionEvents.Connect);
ep.removeCustomEventListener(JambonzTranscriptionEvents.ConnectFailure);
ep.removeCustomEventListener(JambonzTranscriptionEvents.Error);
ep.removeCustomEventListener(AssemblyAiTranscriptionEvents.Transcription);
ep.removeCustomEventListener(AssemblyAiTranscriptionEvents.Connect);
ep.removeCustomEventListener(AssemblyAiTranscriptionEvents.ConnectFailure);
};
const setSpeechCredentialsAtRuntime = (recognizer) => {
if (!recognizer) return;
if (recognizer.vendor === 'nuance') {
@@ -832,7 +783,6 @@ module.exports = (logger) => {
return {
normalizeTranscription,
setChannelVarsForStt,
removeSpeechListeners,
setSpeechCredentialsAtRuntime,
compileSonioxTranscripts,
consolidateTranscripts

View File

@@ -9,7 +9,8 @@ const {
JAMBONES_WS_PING_INTERVAL_MS,
MAX_RECONNECTS,
JAMBONES_WS_HANDSHAKE_TIMEOUT_MS,
JAMBONES_WS_MAX_PAYLOAD
JAMBONES_WS_MAX_PAYLOAD,
HTTP_USER_AGENT_HEADER
} = require('../config');
class WsRequestor extends BaseRequestor {
@@ -228,6 +229,9 @@ class WsRequestor extends BaseRequestor {
maxRedirects: 2,
handshakeTimeout,
maxPayload: JAMBONES_WS_MAX_PAYLOAD ? parseInt(JAMBONES_WS_MAX_PAYLOAD) : 24 * 1024,
headers: {
...(HTTP_USER_AGENT_HEADER && {'user-agent' : HTTP_USER_AGENT_HEADER})
}
};
if (this.username && this.password) opts = {...opts, auth: `${this.username}:${this.password}`};

2835
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,6 @@
{
"name": "jambonz-feature-server",
"version": "0.8.4",
"version": "0.8.5",
"main": "app.js",
"engines": {
"node": ">= 10.16.0"
@@ -30,11 +30,11 @@
"@jambonz/db-helpers": "^0.9.1",
"@jambonz/http-health-check": "^0.0.1",
"@jambonz/mw-registrar": "^0.2.4",
"@jambonz/realtimedb-helpers": "^0.8.6",
"@jambonz/speech-utils": "^0.0.24",
"@jambonz/realtimedb-helpers": "^0.8.7",
"@jambonz/speech-utils": "^0.0.33",
"@jambonz/stats-collector": "^0.1.9",
"@jambonz/time-series": "^0.2.8",
"@jambonz/verb-specifications": "^0.0.45",
"@jambonz/verb-specifications": "^0.0.50",
"@opentelemetry/api": "^1.4.0",
"@opentelemetry/exporter-jaeger": "^1.9.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.35.0",
@@ -47,8 +47,8 @@
"bent": "^7.3.12",
"debug": "^4.3.4",
"deepcopy": "^2.1.0",
"drachtio-fsmrf": "^3.0.27",
"drachtio-srf": "^4.5.29",
"drachtio-fsmrf": "^3.0.33",
"drachtio-srf": "^4.5.31",
"express": "^4.18.2",
"express-validator": "^7.0.1",
"ip": "^1.1.8",

View File

@@ -99,6 +99,8 @@ test('test create-call call-hook basic authentication', async(t) => {
let obj = await getJSON(`http:127.0.0.1:3100/lastRequest/${from}`)
t.ok(obj.headers.Authorization = 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=',
'create-call: call-hook contains basic authentication header');
t.ok(obj.headers['user-agent'] = 'jambonz',
'create-call: call-hook contains user-agent header');
disconnect();
} catch (err) {
console.log(`error received: ${err}`);

View File

@@ -42,7 +42,7 @@ services:
ipv4_address: 172.38.0.7
drachtio:
image: drachtio/drachtio-server:0.8.22
image: drachtio/drachtio-server:0.8.24
restart: always
command: drachtio --contact "sip:*;transport=udp" --mtu 4096 --address 0.0.0.0 --port 9022
ports:
@@ -57,7 +57,7 @@ services:
condition: service_healthy
freeswitch:
image: drachtio/drachtio-freeswitch-mrf:0.4.33
image: drachtio/drachtio-freeswitch-mrf:0.6.1
restart: always
command: freeswitch --rtp-range-start 20000 --rtp-range-end 20100
environment:

53
test/hangup-test.js Normal file
View File

@@ -0,0 +1,53 @@
const test = require('tape');
const { sippUac } = require('./sipp')('test_fs');
const clearModule = require('clear-module');
const {provisionCallHook, provisionCustomHook} = require('./utils')
const bent = require('bent');
const getJSON = bent('json')
process.on('unhandledRejection', (reason, p) => {
console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
});
function connect(connectable) {
return new Promise((resolve, reject) => {
connectable.on('connect', () => {
return resolve();
});
});
}
test('\'hangup\' custom headers', async(t) => {
clearModule.all();
const {srf, disconnect} = require('../app');
try {
await connect(srf);
// GIVEN
const verbs = [
{
verb: 'play',
url: 'https://example.com/example.mp3'
},
{
"verb": "hangup",
"headers": {
"X-Reason" : "maximum call duration exceeded"
}
}
];
const from = 'hangup_custom_headers';
await provisionCallHook(from, verbs)
// THEN
await sippUac('uac-success-received-bye.xml', '172.38.0.10', from);
t.pass('play: succeeds when using single link');
disconnect();
} catch (err) {
console.log(`error received: ${err}`);
disconnect();
t.error(err);
}
});

View File

@@ -16,6 +16,8 @@ require('./listen-tests');
require('./config-test');
require('./queue-test');
require('./in-dialog-test');
require('./hangup-test');
require('./sdp-utils-test');
require('./http-proxy-test');
require('./remove-test-db');
require('./docker_stop');

View File

@@ -188,7 +188,7 @@ test('\'play\' tests with seekOffset and actionHook', async(t) => {
const seconds = parseInt(obj.body.playback_seconds);
const milliseconds = parseInt(obj.body.playback_milliseconds);
const lastOffsetPos = parseInt(obj.body.playback_last_offset_pos);
//console.log({obj}, 'lastRequest');
console.log({obj}, 'lastRequest');
t.ok(obj.body.reason === "playCompleted", "play: actionHook success received");
t.ok(seconds === 2, "playback_seconds: actionHook success received");
t.ok(milliseconds === 2048, "playback_milliseconds: actionHook success received");

26
test/sdp-utils-test.js Normal file
View File

@@ -0,0 +1,26 @@
const test = require('tape');
const {makeOpusFirst, isOpusFirst} = require('../lib/utils/sdp-utils');
const sdpTransform = require('sdp-transform');
test('test opus first', (t) => {
const sdp = 'v=0\r\no=- 3348584794228993675 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0\r\na=extmap-allow-mixed\r\na=msid-semantic: WMS caca8b77-5ae5-4e73-a4d5-de1fce930335\r\nm=audio 57088 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126\r\nc=IN IP4 14.238.89.50\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:1401281302 1 udp 2122260223 10.231.36.146 57088 typ host generation 0 network-id 1 network-cost 10\r\na=candidate:2173263513 1 udp 1686052607 14.238.89.50 57088 typ srflx raddr 10.231.36.146 rport 57088 generation 0 network-id 1 network-cost 10\r\na=ice-ufrag:k5nc\r\na=ice-pwd:J0qwMs6HrIcFNZbDG5m8Kqpk\r\na=ice-options:trickle\r\na=fingerprint:sha-256 66:DE:9A:76:CE:11:2D:65:C4:08:C7:87:B4:90:7E:F1:8D:07:B9:F4:FF:E3:81:D7:E7:7D:C6:56:47:01:6E:55\r\na=setup:actpass\r\na=mid:0\r\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=sendrecv\r\na=msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\na=rtcp-mux\r\na=rtcp-fb:111 transport-cc\r\na=fmtp:111 minptime=10;useinbandfec=1\r\na=rtpmap:63 red/48000/2\r\na=fmtp:63 111/111\r\na=rtpmap:9 G722/8000\r\na=rtpmap:0 PCMU/8000\r\na=rtpmap:8 PCMA/8000\r\na=rtpmap:13 CN/8000\r\na=rtpmap:111 opus/48000/2\r\na=rtpmap:110 telephone-event/48000\r\na=rtpmap:126 telephone-event/8000\r\na=ssrc:3207459321 cname:4nyPJ6KXvseBUIhu\r\na=ssrc:3207459321 msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\n';
const opusSdp = makeOpusFirst(sdp);
const parsedSdp = sdpTransform.parse(opusSdp);
const opusIndex = parsedSdp.media[0].rtp.findIndex((entry) => entry.codec === 'opus');
t.ok(opusIndex === 0, 'succesffuly move opus to be first offer')
t.end();
});
test('test is opus first', (t) => {
const sdp = 'v=0\r\no=- 3348584794228993675 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0\r\na=extmap-allow-mixed\r\na=msid-semantic: WMS caca8b77-5ae5-4e73-a4d5-de1fce930335\r\nm=audio 57088 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126\r\nc=IN IP4 14.238.89.50\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:1401281302 1 udp 2122260223 10.231.36.146 57088 typ host generation 0 network-id 1 network-cost 10\r\na=candidate:2173263513 1 udp 1686052607 14.238.89.50 57088 typ srflx raddr 10.231.36.146 rport 57088 generation 0 network-id 1 network-cost 10\r\na=ice-ufrag:k5nc\r\na=ice-pwd:J0qwMs6HrIcFNZbDG5m8Kqpk\r\na=ice-options:trickle\r\na=fingerprint:sha-256 66:DE:9A:76:CE:11:2D:65:C4:08:C7:87:B4:90:7E:F1:8D:07:B9:F4:FF:E3:81:D7:E7:7D:C6:56:47:01:6E:55\r\na=setup:actpass\r\na=mid:0\r\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=sendrecv\r\na=msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\na=rtcp-mux\r\na=rtpmap:111 opus/48000/2\r\na=rtcp-fb:111 transport-cc\r\na=fmtp:111 minptime=10;useinbandfec=1\r\na=rtpmap:63 red/48000/2\r\na=fmtp:63 111/111\r\na=rtpmap:9 G722/8000\r\na=rtpmap:0 PCMU/8000\r\na=rtpmap:8 PCMA/8000\r\na=rtpmap:13 CN/8000\r\na=rtpmap:110 telephone-event/48000\r\na=rtpmap:126 telephone-event/8000\r\na=ssrc:3207459321 cname:4nyPJ6KXvseBUIhu\r\na=ssrc:3207459321 msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\n';
t.ok(isOpusFirst(sdp), "opus is first offer");
const sdp2 = 'v=0\r\no=- 3348584794228993675 2 IN IP4 127.0.0.1\r\ns=-\r\nt=0 0\r\na=group:BUNDLE 0\r\na=extmap-allow-mixed\r\na=msid-semantic: WMS caca8b77-5ae5-4e73-a4d5-de1fce930335\r\nm=audio 57088 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126\r\nc=IN IP4 14.238.89.50\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=candidate:1401281302 1 udp 2122260223 10.231.36.146 57088 typ host generation 0 network-id 1 network-cost 10\r\na=candidate:2173263513 1 udp 1686052607 14.238.89.50 57088 typ srflx raddr 10.231.36.146 rport 57088 generation 0 network-id 1 network-cost 10\r\na=ice-ufrag:k5nc\r\na=ice-pwd:J0qwMs6HrIcFNZbDG5m8Kqpk\r\na=ice-options:trickle\r\na=fingerprint:sha-256 66:DE:9A:76:CE:11:2D:65:C4:08:C7:87:B4:90:7E:F1:8D:07:B9:F4:FF:E3:81:D7:E7:7D:C6:56:47:01:6E:55\r\na=setup:actpass\r\na=mid:0\r\na=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\na=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\na=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\na=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=sendrecv\r\na=msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\na=rtcp-mux\r\na=rtcp-fb:111 transport-cc\r\na=fmtp:111 minptime=10;useinbandfec=1\r\na=rtpmap:63 red/48000/2\r\na=fmtp:63 111/111\r\na=rtpmap:9 G722/8000\r\na=rtpmap:0 PCMU/8000\r\na=rtpmap:8 PCMA/8000\r\na=rtpmap:13 CN/8000\r\na=rtpmap:111 opus/48000/2\r\na=rtpmap:110 telephone-event/48000\r\na=rtpmap:126 telephone-event/8000\r\na=ssrc:3207459321 cname:4nyPJ6KXvseBUIhu\r\na=ssrc:3207459321 msid:caca8b77-5ae5-4e73-a4d5-de1fce930335 52ad01f1-b1df-4b8e-a208-9201e98b6f7b\r\n';
t.ok(!isOpusFirst(sdp2), "opus is not first offer")
const sdp3 = 'v=0\r\no=xhoaluu2 1314 1504 IN IP4 192.168.1.4\r\ns=Talk\r\nc=IN IP4 192.168.1.4\r\nt=0 0\r\na=ice-pwd:397d063ea23fdc05164e3ee4\r\na=ice-ufrag:16c449a3\r\na=rtcp-xr:rcvr-rtt=all:10000 stat-summary=loss,dup,jitt,TTL voip-metrics\r\na=group:BUNDLE as\r\na=record:off\r\nm=audio 56542 RTP/AVPF 0 8\r\nc=IN IP4 14.226.233.151\r\na=rtcp-mux\r\na=mid:as\r\na=extmap:1 urn:ietf:params:rtp-hdrext:sdes:mid\r\na=rtcp:63076 IN IP4 192.168.1.4\r\na=candidate:1 1 UDP 2130706303 192.168.1.4 56542 typ host\r\na=candidate:1 2 UDP 2130706302 192.168.1.4 63076 typ host\r\na=candidate:2 1 UDP 2130706431 2001:ee0:d744:dcf0:c1d3:d73f:7a93:dc9f 56542 typ host\r\na=candidate:2 2 UDP 2130706430 2001:ee0:d744:dcf0:c1d3:d73f:7a93:dc9f 63076 typ host\r\na=candidate:3 1 UDP 2130706431 2001:ee0:d744:dcf0:15:6be3:8e6b:b736 56542 typ host\r\na=candidate:3 2 UDP 2130706430 2001:ee0:d744:dcf0:15:6be3:8e6b:b736 63076 typ host\r\na=candidate:4 1 UDP 1694498687 14.226.233.151 56542 typ srflx raddr 192.168.1.4 rport 56542\r\na=rtcp-fb:* trr-int 1000\r\na=rtcp-fb:* ccm tmmbr';
t.ok(!isOpusFirst(sdp2), "opus is not first offer")
t.end();
});

View File

@@ -52,6 +52,7 @@ test('\'transcribe\' test - google', async(t) => {
// THEN
await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
//console.log(JSON.stringify(obj));
t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().startsWith('i\'d like to speak to customer support'),
'transcribe: succeeds when using google credentials');
@@ -89,6 +90,7 @@ test('\'transcribe\' test - microsoft', async(t) => {
// THEN
await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
//console.log(JSON.stringify(obj));
t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().startsWith('i\'d like to speak to customer support'),
'transcribe: succeeds when using microsoft credentials');
@@ -126,6 +128,7 @@ test('\'transcribe\' test - aws', async(t) => {
// THEN
await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
//console.log(JSON.stringify(obj));
t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().startsWith('i\'d like to speak to customer support'),
'transcribe: succeeds when using aws credentials');
@@ -137,6 +140,71 @@ test('\'transcribe\' test - aws', async(t) => {
}
});
test('\'transcribe\' test - deepgram config options', async(t) => {
if (!DEEPGRAM_API_KEY ) {
t.pass('skipping deepgram tests');
return t.end();
}
clearModule.all();
const {srf, disconnect} = require('../app');
try {
await connect(srf);
// GIVEN
let verbs = [
{
"verb": "config",
"recognizer": {
"vendor": "deepgram",
"language": "en-US",
"altLanguages": [
"en-US"
],
"deepgramOptions": {
"model": "2-ea",
"tier": "nova",
"numerals": true,
"ner": true,
"vadTurnoff": 10,
"keywords": [
"CPT"
]
}
}
},
{
"verb": "transcribe",
"transcriptionHook": "/transcriptionHook",
"recognizer": {
"vendor": "deepgram",
"altLanguages": [
"en-AU"
],
"hints": ["customer support", "sales", "human resources", "HR"],
"deepgramOptions": {
"apiKey": DEEPGRAM_API_KEY,
}
}
}
];
let from = "gather_success";
await provisionCallHook(from, verbs);
// THEN
await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
//console.log(JSON.stringify(obj));
t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().includes('like to speak to customer support'),
'transcribe: succeeds when using deepgram credentials');
disconnect();
} catch (err) {
console.log(`error received: ${err}`);
disconnect();
t.error(err);
}
});
test('\'transcribe\' test - deepgram', async(t) => {
if (!DEEPGRAM_API_KEY ) {
t.pass('skipping deepgram tests');
@@ -166,6 +234,7 @@ test('\'transcribe\' test - deepgram', async(t) => {
// THEN
await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
//console.log(JSON.stringify(obj));
t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().includes('like to speak to customer support'),
'transcribe: succeeds when using deepgram credentials');
@@ -245,9 +314,133 @@ test('\'transcribe\' test - google with asrTimeout', async(t) => {
// THEN
await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
//console.log(JSON.stringify(obj));
t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().startsWith('i\'d like to speak to customer support'),
'transcribe: succeeds when using google credentials');
disconnect();
} catch (err) {
console.log(`error received: ${err}`);
disconnect();
t.error(err);
}
});
test('\'transcribe\' test - deepgram config options altLanguages', async(t) => {
if (!DEEPGRAM_API_KEY ) {
t.pass('skipping deepgram tests');
return t.end();
}
clearModule.all();
const {srf, disconnect} = require('../app');
try {
await connect(srf);
// GIVEN
let verbs = [
{
"verb": "config",
"recognizer": {
"vendor": "deepgram",
"language": "en-US",
"altLanguages": [
"en-US"
],
"deepgramOptions": {
"model": "2-ea",
"tier": "nova",
"numerals": true,
"ner": true,
"vadTurnoff": 10,
"keywords": [
"CPT"
]
}
}
},
{
"verb": "transcribe",
"transcriptionHook": "/transcriptionHook",
"recognizer": {
"vendor": "deepgram",
"hints": ["customer support", "sales", "human resources", "HR"],
"deepgramOptions": {
"apiKey": DEEPGRAM_API_KEY,
}
}
}
];
let from = "gather_success_no_altLanguages";
await provisionCallHook(from, verbs);
// THEN
await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
//console.log(JSON.stringify(obj));
t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().includes('like to speak to customer support'),
'transcribe: succeeds when using deepgram credentials');
disconnect();
} catch (err) {
console.log(`error received: ${err}`);
disconnect();
t.error(err);
}
});
test('\'transcribe\' test - deepgram config options altLanguages', async(t) => {
if (!DEEPGRAM_API_KEY ) {
t.pass('skipping deepgram tests');
return t.end();
}
clearModule.all();
const {srf, disconnect} = require('../app');
try {
await connect(srf);
// GIVEN
let verbs = [
{
"verb": "config",
"recognizer": {
"vendor": "deepgram",
"language": "en-US",
"altLanguages": [
"en-US"
],
"deepgramOptions": {
"model": "2-ea",
"tier": "nova",
"numerals": true,
"ner": true,
"vadTurnoff": 10,
"keywords": [
"CPT"
]
}
}
},
{
"verb": "transcribe",
"transcriptionHook": "/transcriptionHook",
"recognizer": {
"vendor": "deepgram",
"hints": ["customer support", "sales", "human resources", "HR"],
"altLanguages": [],
"deepgramOptions": {
"apiKey": DEEPGRAM_API_KEY,
}
}
}
];
let from = "gather_success_has_altLanguages";
await provisionCallHook(from, verbs);
// THEN
await sippUac('uac-gather-account-creds-success.xml', '172.38.0.10', from);
let obj = await getJSON(`http://127.0.0.1:3100/lastRequest/${from}_actionHook`);
//console.log(JSON.stringify(obj));
t.ok(obj.body.speech.alternatives[0].transcript.toLowerCase().includes('like to speak to customer support'),
'transcribe: succeeds when using deepgram credentials');
disconnect();
} catch (err) {
console.log(`error received: ${err}`);