Compare commits

...

7 Commits

Author SHA1 Message Date
Dave Horton
a4726cbf1d add startup logging 2025-08-21 13:15:36 -04:00
Dave Horton
2800c4cc4c turn down some logging 2025-08-21 13:00:23 -04:00
Dave Horton
b4b096d07c wip 2025-08-20 18:16:36 -04:00
Dave Horton
10b5ceeef1 wip 2025-08-20 15:15:56 -04:00
Dave Horton
ea954eca0b bump speech utils 2025-08-20 11:02:03 -04:00
Dave Horton
d93da88bff modify tts and say task to track current playback id and match against start and stop events 2025-08-20 10:29:11 -04:00
Dave Horton
10fb6b1e67 update to speech-utils that generates playback id 2025-08-20 10:13:47 -04:00
8 changed files with 51 additions and 19 deletions

6
app.js
View File

@@ -29,6 +29,12 @@ const {LifeCycleEvents, FS_UUID_SET_NAME, SystemState, FEATURE_SERVER} = require
const installSrfLocals = require('./lib/utils/install-srf-locals'); const installSrfLocals = require('./lib/utils/install-srf-locals');
const createHttpListener = require('./lib/utils/http-listener'); const createHttpListener = require('./lib/utils/http-listener');
const healthCheck = require('@jambonz/http-health-check'); const healthCheck = require('@jambonz/http-health-check');
const ProcessMonitor = require('./lib/utils/process-monitor');
const monitor = new ProcessMonitor(logger);
// Log startup
monitor.logStartup();
monitor.setupSignalHandlers();
logger.on('level-change', (lvl, _val, prevLvl, _prevVal, instance) => { logger.on('level-change', (lvl, _val, prevLvl, _prevVal, instance) => {
if (logger !== instance) { if (logger !== instance) {

View File

@@ -218,7 +218,7 @@ class TaskLlmUltravox_S2S extends Task {
async _onServerEvent(_ep, evt) { async _onServerEvent(_ep, evt) {
let endConversation = false; let endConversation = false;
const type = evt.type; const type = evt.type;
this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent'); //this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent');
/* server errors of some sort */ /* server errors of some sort */
if (type === 'error') { if (type === 'error') {

View File

@@ -264,14 +264,11 @@ class TaskSay extends TtsTask {
await this.playToConfMember(ep, memberId, confName, confUuid, filepath[segment]); await this.playToConfMember(ep, memberId, confName, confUuid, filepath[segment]);
} }
else { else {
let playbackId;
const isStreaming = filepath[segment].startsWith('say:{'); const isStreaming = filepath[segment].startsWith('say:{');
if (isStreaming) { if (isStreaming) {
const arr = /^say:\{.*\}\s*(.*)$/.exec(filepath[segment]); const arr = /^say:\{.*\}\s*(.*)$/.exec(filepath[segment]);
if (arr) this.logger.debug(`Say:exec sending streaming tts request: ${arr[1].substring(0, 64)}..`); if (arr) this.logger.debug(`Say:exec sending streaming tts request ${arr[1].substring(0, 64)}..`);
} else this.logger.debug(`Say:exec sending ${filepath[segment].substring(0, 64)}`);
else {
this.logger.debug(`Say:exec sending ${filepath[segment].substring(0, 64)}`);
} }
const onPlaybackStop = (evt) => { const onPlaybackStop = (evt) => {
@@ -283,10 +280,11 @@ class TaskSay extends TtsTask {
* If we got a playback id on both the start and stop events, and they don't match, * If we got a playback id on both the start and stop events, and they don't match,
* then we must have received a playback-stop event for an earlier play request. * then we must have received a playback-stop event for an earlier play request.
*/ */
const unmatchedResponse = (!!playbackId && !!evt.variable_tts_playback_id) && const playbackId = this.getPlaybackId(segment);
evt.variable_tts_playback_id !== playbackId; // eslint-disable-next-line max-len
const unmatchedResponse = (!!playbackId && !!evt.variable_tts_playback_id) && evt.variable_tts_playback_id !== playbackId;
if (unmatchedResponse) { if (unmatchedResponse) {
this.logger.info({currentPlaybackId: playbackId, stopPPlaybackId: evt.variable_tts_playback_id}, this.logger.info({currentPlaybackId: playbackId, stopPlaybackId: evt.variable_tts_playback_id},
'Say:exec discarding playback-stop for earlier play'); 'Say:exec discarding playback-stop for earlier play');
ep.once('playback-stop', this._boundOnPlaybackStop); ep.once('playback-stop', this._boundOnPlaybackStop);
@@ -358,9 +356,17 @@ class TaskSay extends TtsTask {
}; };
this._boundOnPlaybackStop = onPlaybackStop.bind(this); this._boundOnPlaybackStop = onPlaybackStop.bind(this);
ep.once('playback-start', (evt) => { const onPlaybackStart = (evt) => {
try { try {
playbackId = evt.variable_tts_playback_id; const playbackId = this.getPlaybackId(segment);
// eslint-disable-next-line max-len
const unmatchedResponse = (!!playbackId && !!evt.variable_tts_playback_id) && evt.variable_tts_playback_id !== playbackId;
if (unmatchedResponse) {
this.logger.info({currentPlaybackId: playbackId, stopPlaybackId: evt.variable_tts_playback_id},
'Say:exec playback-start - unmatched playback_id');
ep.once('playback-start', this._boundOnPlaybackStart);
return;
}
this.logger.debug({evt}, this.logger.debug({evt},
`Say got playback-start ${evt.variable_tts_playback_id ? evt.variable_tts_playback_id : ''}`); `Say got playback-start ${evt.variable_tts_playback_id ? evt.variable_tts_playback_id : ''}`);
if (this.otelSpan) { if (this.otelSpan) {
@@ -374,8 +380,11 @@ class TaskSay extends TtsTask {
} catch (err) { } catch (err) {
this.logger.info({err}, 'Error handling playback-start event'); this.logger.info({err}, 'Error handling playback-start event');
} }
}); };
this._boundOnPlaybackStart = onPlaybackStart.bind(this);
ep.once('playback-stop', this._boundOnPlaybackStop); ep.once('playback-stop', this._boundOnPlaybackStop);
ep.once('playback-start', this._boundOnPlaybackStart);
// wait for playback-stop event received to confirm if the playback is successful // wait for playback-stop event received to confirm if the playback is successful
this._playPromise = new Promise((resolve, reject) => { this._playPromise = new Promise((resolve, reject) => {

View File

@@ -3,6 +3,16 @@ const { TaskPreconditions } = require('../utils/constants');
const { SpeechCredentialError } = require('../utils/error'); const { SpeechCredentialError } = require('../utils/error');
const dbUtils = require('../utils/db-utils'); const dbUtils = require('../utils/db-utils');
const extractPlaybackId = (str) => {
// Match say:{...} and capture the content inside braces
const match = str.match(/say:\{([^}]*)\}/);
if (!match) return null;
// Look for playback_id=value within the captured content
const playbackMatch = match[1].match(/playback_id=([^,]*)/);
return playbackMatch ? playbackMatch[1] : null;
};
class TtsTask extends Task { class TtsTask extends Task {
constructor(logger, data, parentTask) { constructor(logger, data, parentTask) {
@@ -22,6 +32,11 @@ class TtsTask extends Task {
this.disableTtsCache = this.data.disableTtsCache; this.disableTtsCache = this.data.disableTtsCache;
this.options = this.synthesizer.options || {}; this.options = this.synthesizer.options || {};
this.instructions = this.data.instructions; this.instructions = this.data.instructions;
this.playbackIds = [];
}
getPlaybackId(offset) {
return this.playbackIds[offset];
} }
async exec(cs) { async exec(cs) {
@@ -280,6 +295,7 @@ class TtsTask extends Task {
renderForCaching: preCache renderForCaching: preCache
}); });
if (!filePath.startsWith('say:')) { if (!filePath.startsWith('say:')) {
this.playbackIds.push(null);
this.logger.debug(`Say: file ${filePath}, served from cache ${servedFromCache}`); this.logger.debug(`Say: file ${filePath}, served from cache ${servedFromCache}`);
if (filePath) cs.trackTmpFile(filePath); if (filePath) cs.trackTmpFile(filePath);
if (this.otelSpan) { if (this.otelSpan) {
@@ -309,7 +325,8 @@ class TtsTask extends Task {
} }
} }
else { else {
this.logger.debug('Say: a streaming tts api will be used'); this.playbackIds.push(extractPlaybackId(filePath));
this.logger.debug({playbackIds: this.playbackIds}, 'Say: a streaming tts api will be used');
const modifiedPath = filePath.replace('say:{', `say:{session-uuid=${ep.uuid},`); const modifiedPath = filePath.replace('say:{', `say:{session-uuid=${ep.uuid},`);
return modifiedPath; return modifiedPath;
} }

View File

View File

@@ -293,7 +293,7 @@ class WsRequestor extends BaseRequestor {
/* send the message */ /* send the message */
this.ws.send(JSON.stringify(obj), async() => { this.ws.send(JSON.stringify(obj), async() => {
this.logger.debug({obj}, `WsRequestor:request websocket: sent (${url})`); if (obj.type !== 'llm:event') this.logger.debug({obj}, `WsRequestor:request websocket: sent (${url})`);
// If session:reconnect is waiting for ack, hold here until ack to send queuedMsgs // If session:reconnect is waiting for ack, hold here until ack to send queuedMsgs
if (this._reconnectPromise) { if (this._reconnectPromise) {
try { try {

8
package-lock.json generated
View File

@@ -15,7 +15,7 @@
"@jambonz/http-health-check": "^0.0.1", "@jambonz/http-health-check": "^0.0.1",
"@jambonz/mw-registrar": "^0.2.7", "@jambonz/mw-registrar": "^0.2.7",
"@jambonz/realtimedb-helpers": "^0.8.15", "@jambonz/realtimedb-helpers": "^0.8.15",
"@jambonz/speech-utils": "^0.2.19", "@jambonz/speech-utils": "^0.2.22",
"@jambonz/stats-collector": "^0.1.10", "@jambonz/stats-collector": "^0.1.10",
"@jambonz/time-series": "^0.2.14", "@jambonz/time-series": "^0.2.14",
"@jambonz/verb-specifications": "^0.0.113", "@jambonz/verb-specifications": "^0.0.113",
@@ -1376,9 +1376,9 @@
} }
}, },
"node_modules/@jambonz/speech-utils": { "node_modules/@jambonz/speech-utils": {
"version": "0.2.19", "version": "0.2.22",
"resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.2.19.tgz", "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.2.22.tgz",
"integrity": "sha512-7Sw2pgmsMg/3y3PRhRts/oQrtMlowNS1dn6DgduiHviKSclJNx8oY8S7X8wsBQCe3xdFZYEDxfn9vpcGm4lqZw==", "integrity": "sha512-heSKhoIEAbIjmzwo4CKLkpClGBYrLEo7tud5V0kj2Su3MmgBjCNkPh2WVrP0Qj4Ix8ROKXLASzApkrL60zwNYg==",
"dependencies": { "dependencies": {
"23": "^0.0.0", "23": "^0.0.0",
"@aws-sdk/client-polly": "^3.496.0", "@aws-sdk/client-polly": "^3.496.0",

View File

@@ -31,7 +31,7 @@
"@jambonz/http-health-check": "^0.0.1", "@jambonz/http-health-check": "^0.0.1",
"@jambonz/mw-registrar": "^0.2.7", "@jambonz/mw-registrar": "^0.2.7",
"@jambonz/realtimedb-helpers": "^0.8.15", "@jambonz/realtimedb-helpers": "^0.8.15",
"@jambonz/speech-utils": "^0.2.19", "@jambonz/speech-utils": "^0.2.22",
"@jambonz/stats-collector": "^0.1.10", "@jambonz/stats-collector": "^0.1.10",
"@jambonz/time-series": "^0.2.14", "@jambonz/time-series": "^0.2.14",
"@jambonz/verb-specifications": "^0.0.113", "@jambonz/verb-specifications": "^0.0.113",