Compare commits

...

9 Commits

Author SHA1 Message Date
snyk-bot
875bd307dc fix: package.json & package-lock.json to reduce vulnerabilities
The following vulnerabilities are fixed with an upgrade:
- https://snyk.io/vuln/SNYK-JS-MOMENT-2944238
2022-07-08 01:01:40 +00:00
Paulo Telles
92fa3c249c improve dockerfile to fix snyk security issues (#126)
Co-authored-by: p.souza <p.souza@cognigy.com>
2022-07-07 15:20:26 +02:00
Dave Horton
7f808c6107 listen: when passDtmf is true, send dtmf events down websocket connection as json test frames (#129) 2022-07-07 11:55:29 +02:00
Dave Horton
f95524863d update parse-url, improve Dockerfile 2022-07-06 19:16:31 +02:00
Dave Horton
aceaa5b7da bugfix: continuous asr - if ended by dtmf allow collection of final transcript 2022-06-28 10:10:31 -04:00
Dave Horton
7d57c85153 bugfix #121: Dial verb not ending when call no answer timeout exceeded 2022-06-24 10:50:29 -04:00
Dave Horton
9aa0df256d initial changes to support siprec recording (#120)
* initial changes to support siprec recording

* include additional params on SIP INFO to start recording

* add support for maniupulating recording via REST API

* fixes from testing pause/resume recording
2022-06-23 16:21:35 -04:00
Dave Horton
627c38899f Feature/continuous asr (#119)
* bugfix: background gather for speech-only should still kill audio on dtmf entry when dtmfBargein is true

* initial changes for continuous asr

* move properties under recognizer

* update drachtio-srf@4.5.1

* catch exception on destroy
2022-06-21 10:35:27 -04:00
Dave Horton
bdb40b3aa0 update to drachtio-fsmrf@3.0.1 2022-06-18 15:55:23 -04:00
13 changed files with 6641 additions and 113 deletions

View File

@@ -1,10 +1,23 @@
FROM node:lts-slim
FROM --platform=linux/amd64 node:16.15.1-alpine as base
RUN apk --update --no-cache add --virtual .builds-deps build-base python3
WORKDIR /opt/app/
FROM base as build
COPY package.json package-lock.json ./
RUN npm ci
RUN npm prune
COPY . /opt/app
COPY . .
FROM base
COPY --from=build /opt/app /opt/app/
ARG NODE_ENV
ENV NODE_ENV $NODE_ENV
CMD [ "npm", "start" ]
CMD [ "node", "app.js" ]

View File

@@ -1,6 +1,13 @@
const Emitter = require('events');
const fs = require('fs');
const {CallDirection, TaskPreconditions, CallStatus, TaskName, KillReason} = require('../utils/constants');
const {
CallDirection,
TaskPreconditions,
CallStatus,
TaskName,
KillReason,
RecordState
} = require('../utils/constants');
const moment = require('moment');
const assert = require('assert');
const sessionTracker = require('./session-tracker');
@@ -54,6 +61,8 @@ class CallSession extends Emitter {
assert(rootSpan);
this._recordState = RecordState.RecordingOff;
this.tmpFiles = new Set();
if (!this.isSmsCallSession) {
@@ -85,6 +94,10 @@ class CallSession extends Emitter {
return this.callInfo.direction;
}
get applicationSid() {
return this.callInfo.applicationSid;
}
/**
* SIP call-id for the call
*/
@@ -234,6 +247,153 @@ class CallSession extends Emitter {
return this.rootSpan?.getTracingPropagation();
}
get recordState() { return this._recordState; }
async notifyRecordOptions(opts) {
const {action} = opts;
this.logger.debug({opts}, 'CallSession:notifyRecordOptions');
/* if we have not answered yet, just save the details for later */
if (!this.dlg) {
if (action === 'startCallRecording') {
this.recordOptions = opts;
return true;
}
return false;
}
/* check validity of request */
if (action == 'startCallRecording' && this.recordState !== RecordState.RecordingOff) {
this.logger.info({recordState: this.recordState},
'CallSession:notifyRecordOptions: recording is already started, ignoring request');
return false;
}
if (action == 'stopCallRecording' && this.recordState === RecordState.RecordingOff) {
this.logger.info({recordState: this.recordState},
'CallSession:notifyRecordOptions: recording is already stopped, ignoring request');
return false;
}
if (action == 'pauseCallRecording' && this.recordState !== RecordState.RecordingOn) {
this.logger.info({recordState: this.recordState},
'CallSession:notifyRecordOptions: cannot pause recording, ignoring request ');
return false;
}
if (action == 'resumeCallRecording' && this.recordState !== RecordState.RecordingPaused) {
this.logger.info({recordState: this.recordState},
'CallSession:notifyRecordOptions: cannot resume recording, ignoring request ');
return false;
}
this.recordOptions = opts;
switch (action) {
case 'startCallRecording':
return await this.startRecording();
case 'stopCallRecording':
return await this.stopRecording();
case 'pauseCallRecording':
return await this.pauseRecording();
case 'resumeCallRecording':
return await this.resumeRecording();
default:
throw new Error(`invalid record action ${action}`);
}
}
async startRecording() {
const {recordingID, siprecServerURL} = this.recordOptions;
assert(this.dlg);
this.logger.debug(`CallSession:startRecording - sending to ${siprecServerURL}`);
try {
const res = await this.dlg.request({
method: 'INFO',
headers: {
'X-Reason': 'startCallRecording',
'X-Srs-Url': siprecServerURL,
'X-Srs-Recording-ID': recordingID,
'X-Call-Sid': this.callSid,
'X-Account-Sid': this.accountSid,
'X-Application-Sid': this.applicationSid,
}
});
if (res.status === 200) {
this._recordState = RecordState.RecordingOn;
return true;
}
this.logger.info(`CallSession:startRecording - ${res.status} failure sending to ${siprecServerURL}`);
return false;
} catch (err) {
this.logger.info({err}, `CallSession:startRecording - failure sending to ${siprecServerURL}`);
return false;
}
}
async stopRecording() {
assert(this.dlg);
this.logger.debug('CallSession:stopRecording');
try {
const res = await this.dlg.request({
method: 'INFO',
headers: {
'X-Reason': 'stopCallRecording',
}
});
if (res.status === 200) {
this._recordState = RecordState.RecordingOff;
return true;
}
this.logger.info(`CallSession:stopRecording - ${res.status} failure`);
return false;
} catch (err) {
this.logger.info({err}, 'CallSession:startRecording - failure sending');
return false;
}
}
async pauseRecording() {
assert(this.dlg);
this.logger.debug('CallSession:pauseRecording');
try {
const res = await this.dlg.request({
method: 'INFO',
headers: {
'X-Reason': 'pauseCallRecording',
}
});
if (res.status === 200) {
this._recordState = RecordState.RecordingPaused;
return true;
}
this.logger.info(`CallSession:pauseRecording - ${res.status} failure`);
return false;
} catch (err) {
this.logger.info({err}, 'CallSession:pauseRecording - failure sending');
return false;
}
}
async resumeRecording() {
assert(this.dlg);
this.logger.debug('CallSession:resumeRecording');
try {
const res = await this.dlg.request({
method: 'INFO',
headers: {
'X-Reason': 'resumeCallRecording',
}
});
if (res.status === 200) {
this._recordState = RecordState.RecordingOn;
return true;
}
this.logger.info(`CallSession:resumeRecording - ${res.status} failure`);
return false;
} catch (err) {
this.logger.info({err}, 'CallSession:resumeRecording - failure sending');
return false;
}
}
async enableBotMode(gather, autoEnable) {
try {
const t = normalizeJambones(this.logger, [gather]);
@@ -724,6 +884,9 @@ class CallSession extends Emitter {
const res = await this._lccSipRequest(opts, callSid);
return {status: res.status, reason: res.reason};
}
else if (opts.record) {
await this.notifyRecordOptions(opts.record);
}
// whisper may be the only thing we are asked to do, or it may that
// we are doing a whisper after having muted, paused reccording etc..
@@ -1076,6 +1239,9 @@ class CallSession extends Emitter {
this.dlg.callSid = this.callSid;
this.emit('callStatusChange', {sipStatus: 200, sipReason: 'OK', callStatus: CallStatus.InProgress});
if (this.recordOptions && this.recordState === RecordState.RecordingOff) {
this.startRecording();
}
this.dlg.on('modify', this._onReinvite.bind(this));
this.dlg.on('refer', this._onRefer.bind(this));
@@ -1258,7 +1424,7 @@ class CallSession extends Emitter {
this.emit('callStatusChange', {callStatus: CallStatus.Completed, duration});
this.logger.debug('CallSession: call terminated by jambones');
this.rootSpan.setAttributes({'call.termination': 'hangup by jambonz'});
origDestroy();
origDestroy().catch((err) => this.logger.info({err}, 'CallSession - error destroying dialog'));
if (this.wakeupResolver) {
this.wakeupResolver({reason: 'session ended'});
this.wakeupResolver = null;

View File

@@ -9,7 +9,8 @@ class TaskConfig extends Task {
[
'synthesizer',
'recognizer',
'bargeIn'
'bargeIn',
'record'
].forEach((k) => this[k] = this.data[k] || {});
if (this.bargeIn.enable) {
@@ -74,7 +75,15 @@ class TaskConfig extends Task {
cs.speechRecognizerLanguage = this.recognizer.language !== 'default'
? this.recognizer.language
: cs.speechRecognizerLanguage;
this.logger.info({recognizer: this.recognizer}, 'Config: updated recognizer');
cs.isContinuousAsr = typeof this.recognizer.asrTimeout === 'number' ? true : false;
if (cs.isContinuousAsr) {
cs.asrTimeout = this.recognizer.asrTimeout;
cs.asrDtmfTerminationDigit = this.recognizer.asrDtmfTerminationDigit;
}
this.logger.info({
recognizer: this.recognizer,
isContinuousAsr: cs.isContinuousAsr
}, 'Config: updated recognizer');
}
if ('enable' in this.bargeIn) {
if (this.gatherOpts) {
@@ -92,6 +101,7 @@ class TaskConfig extends Task {
cs.disableBotMode();
}
}
if (this.record.action) cs.notifyRecordOptions(this.record);
}
async kill(cs) {

View File

@@ -404,6 +404,11 @@ class TaskDial extends Task {
this.logger.info(`Dial:_attemptCall: ring no answer timer ${this.timeout}s exceeded`);
this.timerRing = null;
this._killOutdials();
this.result = {
dialCallStatus: CallStatus.NoAnswer,
dialSipStatus: 487
};
this.kill(cs);
}, this.timeout * 1000);
this.span.setAttributes({'dial.target': JSON.stringify(this.target)});

View File

@@ -23,7 +23,7 @@ class TaskGather extends Task {
].forEach((k) => this[k] = this.data[k]);
/* when collecting dtmf, bargein on dtmf is true unless explicitly set to false */
if (this.dtmfBargein !== false && this.input.includes('digits')) this.dtmfBargein = true;
if (this.dtmfBargein !== false && this.input.includes('digits')) this.dtmfBargein = true;
/* timeout of zero means no timeout */
this.timeout = this.timeout === 0 ? 0 : (this.timeout || 15) * 1000;
@@ -49,6 +49,11 @@ class TaskGather extends Task {
this.naicsCode = recognizer.naicsCode || 0;
this.altLanguages = recognizer.altLanguages || [];
/* continuous ASR (i.e. compile transcripts until a special timeout or dtmf key) */
this.asrTimeout = typeof recognizer.asrTimeout === 'number' ? recognizer.asrTimeout * 1000 : 0;
if (this.asrTimeout > 0) this.asrDtmfTerminationDigit = recognizer.asrDtmfTerminationDigit;
this.isContinuousAsr = this.asrTimeout > 0;
/* vad: if provided, we dont connect to recognizer until voice activity is detected */
const {enable, voiceMs = 0, mode = -1} = recognizer.vad || {};
this.vad = {enable, voiceMs, mode};
@@ -65,6 +70,10 @@ class TaskGather extends Task {
this.initialSpeechTimeoutMs = recognizer.initialSpeechTimeoutMs || 0;
this.azureServiceEndpoint = recognizer.azureServiceEndpoint;
}
else {
this.hints = [];
this.altLanguages = [];
}
this.digitBuffer = '';
this._earlyMedia = this.data.earlyMedia === true;
@@ -77,6 +86,9 @@ class TaskGather extends Task {
}
if (!this.sayTask && !this.playTask) this.listenDuringPrompt = false;
/* buffer speech for continueous asr */
this._bufferedTranscripts = [];
this.parentTask = parentTask;
}
@@ -109,6 +121,15 @@ class TaskGather extends Task {
await super.exec(cs);
const {updateSpeechCredentialLastUsed} = require('../utils/db-utils')(this.logger, cs.srf);
if (!this.isContinuousAsr && cs.isContinuousAsr) {
this.isContinuousAsr = true;
this.asrTimeout = cs.asrTimeout * 1000;
this.asrDtmfTerminationDigit = cs.asrDtmfTerminationDigit;
this.logger.debug({
asrTimeout: this.asrTimeout,
asrDtmfTerminationDigit: this.asrDtmfTerminationDigit
}, 'Gather:exec - enabling continuous ASR since it is turned on for the session');
}
this.ep = ep;
if ('default' === this.vendor || !this.vendor) this.vendor = cs.speechRecognizerVendor;
if ('default' === this.language || !this.language) this.language = cs.speechRecognizerLanguage;
@@ -127,6 +148,7 @@ class TaskGather extends Task {
const startListening = (cs, ep) => {
this._startTimer();
if (this.isContinuousAsr && 0 === this.timeout) this._startAsrTimer();
if (this.input.includes('speech') && !this.listenDuringPrompt) {
this._initSpeech(cs, ep)
.then(() => {
@@ -171,7 +193,7 @@ class TaskGather extends Task {
.catch(() => {/*already logged error */});
}
if (this.input.includes('digits') || this.dtmfBargein) {
if (this.input.includes('digits') || this.dtmfBargein || this.asrDtmfTerminationDigit) {
ep.on('dtmf', this._onDtmf.bind(this, cs, ep));
}
@@ -209,12 +231,15 @@ class TaskGather extends Task {
this.logger.debug(evt, 'TaskGather:_onDtmf');
clearTimeout(this.interDigitTimer);
let resolved = false;
if (this.dtmfBargein) this._killAudio(cs);
if (this.dtmfBargein) {
this._killAudio(cs);
this.emit('dtmf', evt);
}
if (evt.dtmf === this.finishOnKey && this.input.includes('digits')) {
resolved = true;
this._resolve('dtmf-terminator-key');
}
else {
else if (this.input.includes('digits')) {
this.digitBuffer += evt.dtmf;
const len = this.digitBuffer.length;
if (len === this.numDigits || len === this.maxDigits) {
@@ -222,6 +247,13 @@ class TaskGather extends Task {
this._resolve('dtmf-num-digits');
}
}
else if (this.isContinuousAsr && evt.dtmf === this.asrDtmfTerminationDigit) {
this.logger.info(`continuousAsr triggered with dtmf ${this.asrDtmfTerminationDigit}`);
this._clearAsrTimer();
this._clearTimer();
this._startFinalAsrTimer();
return;
}
if (!resolved && this.interDigitTimeout > 0 && this.digitBuffer.length >= this.minDigits) {
/* start interDigitTimer */
const ms = this.interDigitTimeout * 1000;
@@ -343,14 +375,10 @@ class TaskGather extends Task {
_startTimer() {
if (0 === this.timeout) return;
if (this._timeoutTimer) {
clearTimeout(this._timeoutTimer);
this._timeoutTimer = null;
}
assert(!this._timeoutTimer);
this.logger.debug(`Gather:_startTimer: timeout ${this.timeout}`);
this._clearTimer();
this._timeoutTimer = setTimeout(() => {
this._resolve(this.digitBuffer.length >= this.minDigits ? 'dtmf-num-digits' : 'timeout');
if (this.isContinuousAsr) this._startAsrTimer();
else this._resolve(this.digitBuffer.length >= this.minDigits ? 'dtmf-num-digits' : 'timeout');
}, this.timeout);
}
@@ -361,6 +389,35 @@ class TaskGather extends Task {
}
}
_startAsrTimer() {
assert(this.isContinuousAsr);
this._clearAsrTimer();
this._asrTimer = setTimeout(() => {
this.logger.debug('_startAsrTimer - asr timer went off');
this._resolve(this._bufferedTranscripts.length > 0 ? 'speech' : 'timeout');
}, this.asrTimeout);
this.logger.debug(`_startAsrTimer: set for ${this.asrTimeout}ms`);
}
_clearAsrTimer() {
if (this._asrTimer) clearTimeout(this._asrTimer);
this._asrTimer = null;
}
_startFinalAsrTimer() {
this._clearFinalAsrTimer();
this._finalAsrTimer = setTimeout(() => {
this.logger.debug('_startFinalAsrTimer - final asr timer went off');
this._resolve(this._bufferedTranscripts.length > 0 ? 'speech' : 'timeout');
}, 1000);
this.logger.debug('_startFinalAsrTimer: set for 1 second');
}
_clearFinalAsrTimer() {
if (this._finalAsrTimer) clearTimeout(this._finalAsrTimer);
this._finalAsrTimer = null;
}
_killAudio(cs) {
if (!this.sayTask && !this.playTask && this.bargein) {
if (this.ep?.connected && !this.playComplete) {
@@ -419,7 +476,19 @@ class TaskGather extends Task {
this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, listen again');
return this._startTranscribing(ep);
}
this._resolve('speech', evt);
if (this.isContinuousAsr) {
/* append the transcript and start listening again for asrTimeout */
this.logger.info({evt}, 'TaskGather:_onTranscription - got transcript during continous asr');
this._bufferedTranscripts.push(evt);
this._clearTimer();
if (this._finalAsrTimer) {
this._clearFinalAsrTimer();
return this._resolve(this._bufferedTranscripts.length > 0 ? 'speech' : 'timeout');
}
this._startAsrTimer();
return this._startTranscribing(ep);
}
else this._resolve('speech', evt);
}
else {
/* google has a measure of stability:
@@ -476,6 +545,15 @@ class TaskGather extends Task {
this.resolved = true;
clearTimeout(this.interDigitTimer);
this._clearTimer();
if (this.isContinuousAsr && reason.startsWith('speech')) {
evt = {
is_final: true,
transcripts: this._bufferedTranscripts
};
this.logger.debug({evt}, 'TaskGather:resolve continuous asr');
}
this.span.setAttributes({'stt.resolve': reason, 'stt.result': JSON.stringify(evt)});
if (this.ep && this.ep.connected) {
@@ -483,8 +561,6 @@ class TaskGather extends Task {
.catch((err) => this.logger.error({err}, 'Error stopping transcription'));
}
this._clearTimer();
if (this.callSession && this.callSession.callGone) {
this.logger.debug('TaskGather:_resolve - call is gone, not invoking web callback');
this.notifyTaskDone();

View File

@@ -22,8 +22,6 @@ class TaskListen extends Task {
this.results = {};
if (this.transcribe) this.transcribeTask = makeTask(logger, {'transcribe': opts.transcribe}, this);
this._dtmfHandler = this._onDtmf.bind(this);
}
get name() { return TaskName.Listen; }
@@ -31,6 +29,7 @@ class TaskListen extends Task {
async exec(cs, ep) {
await super.exec(cs);
this.ep = ep;
this._dtmfHandler = this._onDtmf.bind(this, ep);
try {
this.hook = this.normalizeUrl(this.url, 'GET', this.wsAuth);
@@ -148,7 +147,13 @@ class TaskListen extends Task {
}
_onDtmf(evt) {
_onDtmf(ep, evt) {
this.logger.debug({evt}, `TaskListen:_onDtmf received dtmf ${evt.dtmf}`);
if (this.passDtmf && this.ep?.connected) {
const obj = {event: 'dtmf', dtmf: evt.dtmf, duration: evt.duration};
this.ep.forkAudioSendText(obj)
.catch((err) => this.logger.info({err}, 'TaskListen:_onDtmf error sending dtmf'));
}
if (evt.dtmf === this.finishOnKey) {
this.logger.info(`TaskListen:_onDtmf terminating task due to dtmf ${evt.dtmf}`);
this.results.digits = evt.dtmf;

View File

@@ -12,6 +12,7 @@ class TaskSipRequest extends Task {
this.method = this.data.method.toUpperCase();
this.headers = this.data.headers || {};
this.body = this.data.body;
if (this.body) this.body = `${this.body}\n`;
}
get name() { return TaskName.SipRequest; }

View File

@@ -36,7 +36,8 @@
"properties": {
"synthesizer": "#synthesizer",
"recognizer": "#recognizer",
"bargeIn": "#bargeIn"
"bargeIn": "#bargeIn",
"record": "#recordOptions"
},
"required": []
},
@@ -307,6 +308,19 @@
"path"
]
},
"recordOptions": {
"properties": {
"action": {
"type": "string",
"enum": ["startCallRecording", "stopCallRecording", "pauseCallRecording", "resumeCallRecording"]
},
"recordingID": "string",
"siprecServerURL": "string"
},
"required": [
"action"
]
},
"redirect": {
"properties": {
"actionHook": "object|string"
@@ -477,7 +491,9 @@
},
"requestSnr": "boolean",
"initialSpeechTimeoutMs": "number",
"azureServiceEndpoint": "string"
"azureServiceEndpoint": "string",
"asrDtmfTerminationDigit": "string",
"asrTimeout": "number"
},
"required": [
"vendor"

View File

@@ -145,7 +145,7 @@ class Task extends Emitter {
async performAction(results, expectResponse = true) {
if (this.actionHook) {
const params = results ? Object.assign(results, this.cs.callInfo.toJSON()) : this.cs.callInfo.toJSON();
const params = results ? Object.assign(this.cs.callInfo.toJSON(), results) : this.cs.callInfo.toJSON();
const span = this.startSpan('verb:hook', {'hook.url': this.actionHook});
const b3 = this.getTracingPropagation('b3', span);
const httpHeaders = b3 && {b3};

View File

@@ -120,6 +120,11 @@
"verb:hook",
"jambonz:error"
],
"RecordState": {
"RecordingOn": "recording_on",
"RecordingOff": "recording_off",
"RecordingPaused": "recording_paused"
},
"MAX_SIMRINGS": 10,
"BONG_TONE": "tone_stream://v=-7;%(100,0,941.0,1477.0);v=-7;>=2;+=.1;%(1400,0,350,440)",
"FS_UUID_SET_NAME": "fsUUIDs"

View File

@@ -23,9 +23,8 @@ class HttpRequestor extends BaseRequestor {
this.authHeader = basicAuth(hook.username, hook.password);
const u = parseUrl(this.url);
const myPort = u.port ? `:${u.port}` : '';
const baseUrl = this._baseUrl = `${u.protocol}://${u.resource}${myPort}`;
//const myPort = u.port ? `:${u.port}` : '';
const baseUrl = this._baseUrl = `${u.protocol}://${u.resource}`;
this.get = bent(baseUrl, 'GET', 'buffer', 200, 201);
this.post = bent(baseUrl, 'POST', 'buffer', 200, 201);

6386
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -44,19 +44,19 @@
"bent": "^7.3.12",
"debug": "^4.3.4",
"deepcopy": "^2.1.0",
"drachtio-fsmrf": "^3.0.0",
"drachtio-srf": "^4.5.0",
"drachtio-fsmrf": "^3.0.1",
"drachtio-srf": "^4.5.1",
"express": "^4.18.1",
"helmet": "^5.1.0",
"ip": "^1.1.8",
"moment": "^2.29.3",
"parse-url": "^5.0.8",
"moment": "^2.29.4",
"parse-url": "^7.0.2",
"pino": "^6.14.0",
"sdp-transform": "^2.14.1",
"short-uuid": "^4.2.0",
"to-snake-case": "^1.0.0",
"uuid": "^8.3.2",
"verify-aws-sns-signature": "^0.0.6",
"verify-aws-sns-signature": "^0.0.7",
"ws": "^8.8.0",
"xml2js": "^0.4.23"
},