From 24a66fed64a8f2bbaa89abb84f0a912fdff88520 Mon Sep 17 00:00:00 2001 From: Dave Horton Date: Tue, 19 Nov 2024 09:37:00 -0500 Subject: [PATCH] wip (#979) --- lib/session/call-session.js | 42 ++++++++++++++--- lib/tasks/dial.js | 94 ++++++++++++++++++++++++++++++------- lib/utils/constants.json | 5 ++ lib/utils/place-outdial.js | 24 +++++++--- package-lock.json | 14 +++--- package.json | 2 +- 6 files changed, 142 insertions(+), 39 deletions(-) diff --git a/lib/session/call-session.js b/lib/session/call-session.js index 6b754e0e..87b73bda 100644 --- a/lib/session/call-session.js +++ b/lib/session/call-session.js @@ -2,6 +2,7 @@ const Emitter = require('events'); const fs = require('fs'); const { CallDirection, + MediaPath, TaskPreconditions, CallStatus, TaskName, @@ -1589,6 +1590,15 @@ Duration=${duration} ` this.logger.info({response}, '_lccBoostAudioSignal: response from freeswitch'); } + async _lccMediaPath(desiredPath) { + const task = this.currentTask; + if (!task || task.name !== TaskName.Dial) { + return this.logger.info('CallSession:_lccMediaPath - invalid command since we are not in a dial verb'); + } + task.updateMediaPath(desiredPath) + .catch((err) => this.logger.error(err, 'CallSession:_lccMediaPath')); + } + _lccToolOutput(tool_call_id, opts, callSid) { // only valid if we are in an LLM verb const task = this.currentTask; @@ -1672,6 +1682,9 @@ Duration=${duration} ` else if (opts.boostAudioSignal) { return this._lccBoostAudioSignal(opts, callSid); } + else if (opts.media_path) { + return this._lccMediaPath(opts.media_path, callSid); + } else if (opts.llm_tool_output) { return this._lccToolOutput(opts.tool_call_id, opts.llm_tool_output, callSid); } @@ -1975,6 +1988,13 @@ Duration=${duration} ` }); break; + case 'media:path': + this._lccMediaPath(data, call_sid) + .catch((err) => { + this.logger.info({err, data}, 'CallSession:_onCommand - error setting media path'); + }); + break; + case 'llm:tool-output': this._lccToolOutput(tool_call_id, data, call_sid); break; @@ -2505,27 +2525,35 @@ Duration=${duration} ` }; } - async releaseMediaToSBC(remoteSdp) { + async releaseMediaToSBC(remoteSdp, releaseMediaEntirely) { assert(this.dlg && this.dlg.connected && this.ep && typeof remoteSdp === 'string'); await this.dlg.modify(remoteSdp, { headers: { - 'X-Reason': 'release-media' + 'X-Reason': releaseMediaEntirely ? 'release-media-entirely' : 'release-media' } }); - this.ep.destroy() - .then(() => this.ep = null) - .catch((err) => this.logger.error({err}, 'CallSession:releaseMediaToSBC: Error destroying endpoint')); + try { + await this.ep.destroy(); + } catch (err) { + this.logger.error({err}, 'CallSession:releaseMediaToSBC: Error destroying endpoint'); + } + this.ep = null; } - async reAnchorMedia() { + async reAnchorMedia(currentMediaRoute = MediaPath.PartialMedia) { assert(this.dlg && this.dlg.connected && !this.ep); + this.ep = await this.ms.createEndpoint({remoteSdp: this.dlg.remote.sdp}); + this._configMsEndpoint(); await this.dlg.modify(this.ep.local.sdp, { headers: { 'X-Reason': 'anchor-media' } }); - this._configMsEndpoint(); + + if (currentMediaRoute === MediaPath.NoMedia) { + await this.ep.modify(this.dlg.remote.sdp); + } } async handleReinviteAfterMediaReleased(req, res) { diff --git a/lib/tasks/dial.js b/lib/tasks/dial.js index 88f5673c..05e7cb32 100644 --- a/lib/tasks/dial.js +++ b/lib/tasks/dial.js @@ -6,6 +6,7 @@ const { TaskName, TaskPreconditions, MAX_SIMRINGS, + MediaPath, KillReason } = require('../utils/constants'); const assert = require('assert'); @@ -108,6 +109,7 @@ class TaskDial extends Task { this.proxy = this.data.proxy; this.tag = this.data.tag; this.boostAudioSignal = this.data.boostAudioSignal; + this._mediaPath = MediaPath.FullMedia; if (this.dtmfHook) { const {parentDtmfCollector, childDtmfCollector} = parseDtmfOptions(logger, this.data.dtmfCapture || {}); @@ -155,17 +157,21 @@ class TaskDial extends Task { get canReleaseMedia() { const keepAnchor = this.data.anchorMedia || - this.cs.isBackGroundListen || - this.cs.onHoldMusic || - ANCHOR_MEDIA_ALWAYS || - this.listenTask || - this.dubTasks || - this.transcribeTask || - this.startAmd; + this.cs.isBackGroundListen || + this.cs.onHoldMusic || + ANCHOR_MEDIA_ALWAYS || + this.listenTask || + this.dubTasks || + this.transcribeTask || + this.startAmd; return !keepAnchor; } + get shouldExitMediaPathEntirely() { + return this.data.exitMediaPath; + } + get summary() { if (this.target.length === 1) { const target = this.target[0]; @@ -186,6 +192,16 @@ class TaskDial extends Task { async exec(cs) { await super.exec(cs); + + if (this.data.anchorMedia && this.data.exitMediaPath) { + this.logger.info('Dial:exec - incompatible anchorMedia and exitMediaPath are both set, will obey anchorMedia'); + delete this.data.exitMediaPath; + } + if (!this.canReleaseMedia && this.data.exitMediaPath) { + this.logger.info( + 'Dial:exec - exitMediaPath is set so features such as transcribe and record will not work on this call'); + } + try { if (this.listenTask) { const {span, ctx} = this.startChildSpan(`nested:${this.listenTask.summary}`); @@ -303,7 +319,7 @@ class TaskDial extends Task { if (!cs.callGone && this.epOther) { /* if we can release the media back to the SBC, do so now */ - if (this.canReleaseMedia) this._releaseMedia(cs, this.sd); + if (this.canReleaseMedia) this._releaseMedia(cs, this.sd, this.shouldExitMediaPathEntirely); else this.epOther.bridge(this.ep); } } catch (err) { @@ -752,7 +768,7 @@ class TaskDial extends Task { // Offhold, time to release media const newSdp = await this.ep.modify(req.body); await res.send(200, {body: newSdp}); - await this._releaseMedia(this.cs, this.sd); + await this._releaseMedia(this.cs, this.sd, this.shouldExitMediaPathEntirely); this.isOutgoingLegHold = false; } else { this.logger.debug('Dial: _onReinvite receive unhold Request, update media server'); @@ -861,7 +877,9 @@ class TaskDial extends Task { } /* if we can release the media back to the SBC, do so now */ - if (this.canReleaseMedia) setTimeout(this._releaseMedia.bind(this, cs, sd), 200); + if (this.canReleaseMedia || this.shouldExitMediaPathEntirely) { + setTimeout(this._releaseMedia.bind(this, cs, sd, this.shouldExitMediaPathEntirely), 200); + } } _bridgeEarlyMedia(sd) { @@ -873,22 +891,57 @@ class TaskDial extends Task { } } + /* public api */ + async updateMediaPath(desiredPath) { + this.logger.info(`Dial:updateMediaPath - ${this._mediaPath} => ${desiredPath}`); + switch (desiredPath) { + case MediaPath.NoMedia: + assert(this._mediaPath !== MediaPath.NoMedia, 'updateMediaPath: already no-media'); + await this._releaseMedia(this.cs, this.sd, true); + break; + + case MediaPath.PartialMedia: + assert(this._mediaPath !== MediaPath.PartialMedia, 'updateMediaPath: already partial-media'); + if (this._mediaPath === MediaPath.FullMedia) { + await this._releaseMedia(this.cs, this.sd, false); + } + else { + // to go from no-media to partial-media we need to go through full-media first + await this.reAnchorMedia(this.cs, this.sd); + await this._releaseMedia(this.cs, this.sd, false); + } + assert(!this.epOther, 'updateMediaPath: epOther should be null'); + assert(!this.ep, 'updateMediaPath: ep should be null'); + + break; + case MediaPath.FullMedia: + assert(this._mediaPath !== MediaPath.FullMedia, 'updateMediaPath: already full-media'); + await this.reAnchorMedia(this.cs, this.sd); + break; + + default: + assert(false, `updateMediaPath: invalid path request ${desiredPath}`); + } + } + /** * Release the media from freeswitch * @param {*} cs * @param {*} sd */ - async _releaseMedia(cs, sd) { + async _releaseMedia(cs, sd, releaseEntirely = false) { assert(cs.ep && sd.ep); try { // Wait until we got new SDP from B leg to ofter to A Leg const aLegSdp = cs.ep.remote.sdp; - await sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp); + await sd.releaseMediaToSBC(aLegSdp, cs.ep.local.sdp, releaseEntirely); const bLegSdp = sd.dlg.remote.sdp; - await cs.releaseMediaToSBC(bLegSdp); + await cs.releaseMediaToSBC(bLegSdp, releaseEntirely); this.epOther = null; - this.logger.info('Dial:_releaseMedia - successfully released media from freewitch'); + this._mediaPath = releaseEntirely ? MediaPath.NoMedia : MediaPath.PartialMedia; + this.logger.info( + `Dial:_releaseMedia - successfully released media from freewitch, media path is now ${this._mediaPath}`); } catch (err) { this.logger.info({err}, 'Dial:_releaseMedia error'); } @@ -898,8 +951,14 @@ class TaskDial extends Task { if (cs.ep && sd.ep) return; this.logger.info('Dial:reAnchorMedia - re-anchoring media to freewitch'); - await Promise.all([sd.reAnchorMedia(), cs.reAnchorMedia()]); + await Promise.all([sd.reAnchorMedia(this._mediaPath), cs.reAnchorMedia(this._mediaPath)]); this.epOther = cs.ep; + + this.epOther.bridge(this.ep); + + this._mediaPath = MediaPath.FullMedia; + this.logger.info( + `Dial:_releaseMedia - successfully re-anchored media to freewitch, media path is now ${this._mediaPath}`); } // Handle RE-INVITE hold from caller leg. @@ -918,11 +977,12 @@ class TaskDial extends Task { } this._onHoldHook(req); } else if (!isOnhold(req.body)) { - if (this.epOther && this.ep && this.isIncomingLegHold && this.canReleaseMedia) { + if (this.epOther && this.ep && this.isIncomingLegHold && + (this.canReleaseMedia || this.shouldExitMediaPathEntirely)) { // Offhold, time to release media const newSdp = await this.epOther.modify(req.body); await res.send(200, {body: newSdp}); - await this._releaseMedia(this.cs, this.sd); + await this._releaseMedia(this.cs, this.sd, this.shouldExitMediaPathEntirely); isHandled = true; } this.isIncomingLegHold = false; diff --git a/lib/utils/constants.json b/lib/utils/constants.json index 996e6819..090a7285 100644 --- a/lib/utils/constants.json +++ b/lib/utils/constants.json @@ -221,6 +221,11 @@ "ToneTimeout": "amd_tone_timeout", "Stopped": "amd_stopped" }, + "MediaPath": { + "NoMedia": "no-media", + "PartialMedia": "partial-media", + "FullMedia": "full-media" + }, "MAX_SIMRINGS": 10, "BONG_TONE": "tone_stream://v=-7;%(100,0,941.0,1477.0);v=-7;>=2;+=.1;%(1400,0,350,440)", "FS_UUID_SET_NAME": "fsUUIDs" diff --git a/lib/utils/place-outdial.js b/lib/utils/place-outdial.js index 70111b6c..bb105e4f 100644 --- a/lib/utils/place-outdial.js +++ b/lib/utils/place-outdial.js @@ -1,5 +1,5 @@ const Emitter = require('events'); -const {CallStatus} = require('./constants'); +const {CallStatus, MediaPath} = require('./constants'); const SipError = require('drachtio-srf').SipError; const {TaskPreconditions, CallDirection} = require('../utils/constants'); const CallInfo = require('../session/call-info'); @@ -455,21 +455,26 @@ class SingleDialer extends Emitter { return cs; } - async releaseMediaToSBC(remoteSdp, localSdp) { + async releaseMediaToSBC(remoteSdp, localSdp, releaseMediaEntirely) { assert(this.dlg && this.dlg.connected && this.ep && typeof remoteSdp === 'string'); const sdp = stripCodecs(this.logger, remoteSdp, localSdp) || remoteSdp; await this.dlg.modify(sdp, { headers: { - 'X-Reason': 'release-media' + 'X-Reason': releaseMediaEntirely ? 'release-media-entirely' : 'release-media' } }); - this.ep.destroy() - .then(() => this.ep = null) - .catch((err) => this.logger.error({err}, 'SingleDialer:releaseMediaToSBC: Error destroying endpoint')); + try { + await this.ep.destroy(); + } catch (err) { + this.logger.error({err}, 'SingleDialer:releaseMediaToSBC: Error destroying endpoint'); + } + this.ep = null; } - async reAnchorMedia() { + async reAnchorMedia(currentMediaRoute = MediaPath.PartialMedia) { assert(this.dlg && this.dlg.connected && !this.ep); + + this.logger.debug('SingleDialer:reAnchorMedia: re-anchoring media after partial media'); this.ep = await this.ms.createEndpoint({remoteSdp: this.dlg.remote.sdp}); this._configMsEndpoint(); await this.dlg.modify(this.ep.local.sdp, { @@ -477,6 +482,11 @@ class SingleDialer extends Emitter { 'X-Reason': 'anchor-media' } }); + + if (currentMediaRoute === MediaPath.NoMedia) { + this.logger.debug('SingleDialer:reAnchorMedia: repoint endpoint after no media'); + await this.ep.modify(this.dlg.remote.sdp); + } } _notifyCallStatusChange({callStatus, sipStatus, sipReason, duration}) { diff --git a/package-lock.json b/package-lock.json index f76392cc..f770585f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "@jambonz/speech-utils": "^0.1.22", "@jambonz/stats-collector": "^0.1.10", "@jambonz/time-series": "^0.2.9", - "@jambonz/verb-specifications": "^0.0.85", + "@jambonz/verb-specifications": "^0.0.86", "@opentelemetry/api": "^1.8.0", "@opentelemetry/exporter-jaeger": "^1.23.0", "@opentelemetry/exporter-trace-otlp-http": "^0.50.0", @@ -1581,9 +1581,9 @@ } }, "node_modules/@jambonz/verb-specifications": { - "version": "0.0.85", - "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.85.tgz", - "integrity": "sha512-NQRXcfamrs3lRGy/1pAaAIuZcZPj4zjwUTYf7Sv+It0fuzB4KWbiqL9yeSq61qtCX2AutwRsV4WA8OZQYfLdgw==", + "version": "0.0.86", + "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.86.tgz", + "integrity": "sha512-nTMMeFJtkSIVD3icQajOKv+zFBiAaUNuky2htVgYKipE2AIq/H/SsdurYdBij3KQq1o58/FuUuXcqCwjnZnoUg==", "dependencies": { "debug": "^4.3.4", "pino": "^8.8.0" @@ -10751,9 +10751,9 @@ } }, "@jambonz/verb-specifications": { - "version": "0.0.85", - "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.85.tgz", - "integrity": "sha512-NQRXcfamrs3lRGy/1pAaAIuZcZPj4zjwUTYf7Sv+It0fuzB4KWbiqL9yeSq61qtCX2AutwRsV4WA8OZQYfLdgw==", + "version": "0.0.86", + "resolved": "https://registry.npmjs.org/@jambonz/verb-specifications/-/verb-specifications-0.0.86.tgz", + "integrity": "sha512-nTMMeFJtkSIVD3icQajOKv+zFBiAaUNuky2htVgYKipE2AIq/H/SsdurYdBij3KQq1o58/FuUuXcqCwjnZnoUg==", "requires": { "debug": "^4.3.4", "pino": "^8.8.0" diff --git a/package.json b/package.json index 2ca0f393..b69b4ce5 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@jambonz/speech-utils": "^0.1.22", "@jambonz/stats-collector": "^0.1.10", "@jambonz/time-series": "^0.2.9", - "@jambonz/verb-specifications": "^0.0.85", + "@jambonz/verb-specifications": "^0.0.86", "@opentelemetry/api": "^1.8.0", "@opentelemetry/exporter-jaeger": "^1.23.0", "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",