Feat/tts streaming (#994)

* wip

* add TtsStreamingBuffer class to abstract handling of streaming tokens

* wip

* add throttling support

* support background ttsStream (#995)

* wip

* add TtsStreamingBuffer class to abstract handling of streaming tokens

* wip

* support background ttsStream

* wip

---------

Co-authored-by: Dave Horton <daveh@beachdognet.com>

* wip

* dont send if we have nothing to send

* initial testing with cartesia

* wip

---------

Co-authored-by: Hoan Luu Huu <110280845+xquanluu@users.noreply.github.com>

lib/session/call-session.js

@@ -9,7 +9,8 @@ const {
   KillReason,
   RecordState,
   AllowedSipRecVerbs,
-  AllowedConfirmSessionVerbs
+  AllowedConfirmSessionVerbs,
+  TtsStreamingEvents
 } = require('../utils/constants');
 const moment = require('moment');
 const assert = require('assert');
@@ -21,6 +22,7 @@ const listTaskNames = require('../utils/summarize-tasks');
 const HttpRequestor = require('../utils/http-requestor');
 const WsRequestor = require('../utils/ws-requestor');
 const ActionHookDelayProcessor = require('../utils/action-hook-delay');
+const TtsStreamingBuffer = require('../utils/tts-streaming-buffer');
 const {parseUri} = require('drachtio-srf');
 const {
   JAMBONES_INJECT_CONTENT,
@@ -413,27 +415,24 @@ class CallSession extends Emitter {
   get isAdultingCallSession() {
     return this.constructor.name === 'AdultingCallSession';
   }
-
-  /**
-   * returns true if this session is a ConfirmCallSession
-   */
   get isConfirmCallSession() {
     return this.constructor.name === 'ConfirmCallSession';
   }
-
-  /**
-   * returns true if this session is a SipRecCallSession
-   */
   get isSipRecCallSession() {
     return this.constructor.name === 'SipRecCallSession';
   }
-
-  /**
-   * returns true if this session is a SmsCallSession
-   */
   get isSmsCallSession() {
     return this.constructor.name === 'SmsCallSession';
   }
+  get isRestCallSession() {
+    return this.constructor.name === 'RestCallSession';
+  }
+  get InboundCallSession() {
+    return this.constructor.name === 'InboundCallSession';
+  }
+  get isNormalCallSession() {
+    return this.constructor.name === 'InboundCallSession' || this.constructor.name === 'RestCallSession';
+  }
 
   get is3pccInvite() {
     return this.isInboundCallSession && this.req?.body?.length === 0;
@@ -451,6 +450,10 @@ class CallSession extends Emitter {
     return this.backgroundTaskManager.isTaskRunning('bargeIn');
   }
 
+  get isTtsStreamEnabled() {
+    return this.backgroundTaskManager.isTaskRunning('ttsStream');
+  }
+
   get isListenEnabled() {
     return this.backgroundTaskManager.isTaskRunning('listen');
   }
@@ -513,6 +516,10 @@ class CallSession extends Emitter {
     this._sipRequestWithinDialogHook = url;
   }
 
+  get isTtsStreamOpen() {
+    return this.currentTask?.isStreamingTts ||
+      this.backgroundTaskManager.getTask('ttsStream')?.isStreamingTts;
+  }
   // Bot Delay (actionHook delayed)
   get actionHookDelayEnabled() {
     return this._actionHookDelayEnabled;
@@ -587,6 +594,25 @@ class CallSession extends Emitter {
     }
   }
 
+  getTsStreamingVendor() {
+    let v;
+    if (this.currentTask?.isStreamingTts) {
+      const {vendor} = this.currentTask.getTtsVendorData(this);
+      v = vendor;
+    }
+    else if (this.backgroundTaskManager.getTask('ttsStream')?.isStreamingTts) {
+      const {vendor} = this.backgroundTaskManager.getTask('ttsStream').getTtsVendorData(this);
+      v = vendor;
+    }
+    return v;
+  }
+
+  get appIsUsingWebsockets() {
+    return this.requestor instanceof WsRequestor;
+  }
+
+  /* end of getters and setters */
+
   async clearOrRestoreActionHookDelayProcessor() {
     if (this._actionHookDelayProcessor) {
       await this._actionHookDelayProcessor.stop();
@@ -804,6 +830,36 @@ class CallSession extends Emitter {
     }
   }
 
+  async enableBackgroundTtsStream(say) {
+    try {
+      if (this.isTtsStreamEnabled) {
+        this.logger.debug('CallSession:enableBackgroundTtsStream - ttsStream currently enabled, ignoring request');
+      } else if (this.appIsUsingWebsockets && this.isNormalCallSession) {
+        await this.backgroundTaskManager.newTask('ttsStream', say);
+        this.logger.debug('CallSession:enableBackgroundTtsStream - ttsStream enabled');
+      } else {
+        this.logger.debug(
+          'CallSession:enableBackgroundTtsStream - ignoring request as call does not have required conditions');
+      }
+    }  catch (err) {
+      this.logger.info({err, say}, 'CallSession:enableBackgroundTtsStream - Error creating background tts stream task');
+    }
+  }
+
+  disableTtsStream() {
+    if (this.isTtsStreamEnabled) {
+      this.backgroundTaskManager.stop('ttsStream');
+      this.logger.debug('CallSession:disableTtsStream - ttsStream disabled');
+    }
+  }
+  clearTtsStream() {
+    this.ttsStreamingBuffer?.clear();
+  }
+
+  startTtsStream() {
+    this.ttsStreamingBuffer?.start();
+  }
+
   async enableBotMode(gather, autoEnable) {
     try {
       let task;
@@ -1063,6 +1119,17 @@ class CallSession extends Emitter {
       this.inbandDtmfEnabled = voipCarrier?.dtmf_type === 'tones';
     }
 
+    if (this.isNormalCallSession) {
+      this.ttsStreamingBuffer = new TtsStreamingBuffer(this);
+      this.ttsStreamingBuffer.on(TtsStreamingEvents.Empty, this._onTtsStreamingEmpty.bind(this));
+      this.ttsStreamingBuffer.on(TtsStreamingEvents.Pause, this._onTtsStreamingPause.bind(this));
+      this.ttsStreamingBuffer.on(TtsStreamingEvents.Resume, this._onTtsStreamingResume.bind(this));
+      this.ttsStreamingBuffer.on(TtsStreamingEvents.ConnectFailure, this._onTtsStreamingConnectFailure.bind(this));
+    }
+    else {
+      this.logger.info(`CallSession:exec - not a normal call session: ${this.constructor.name}`);
+    }
+
     while (this.tasks.length && !this.callGone) {
       const taskNum = ++this.taskIdx;
       const stackNum = this.stackIdx;
@@ -1646,6 +1713,39 @@ Duration=${duration} `
       .catch((err) => this.logger.error(err, 'CallSession:_lccLlmUpdate'));
   }
 
+  async _lccTtsTokens(opts) {
+    const {id, tokens} = opts;
+
+    if (id === undefined) {
+      this.logger.info({opts}, 'CallSession:_lccTtsTokens - invalid command since id is missing');
+      return;
+    }
+    else if (tokens === undefined) {
+      this.logger.info({opts}, 'CallSession:_lccTtsTokens - invalid command since id is missing');
+      return this.requestor.request('tts:tokens-result', '/tokens-result', {
+        id,
+        status: 'failed',
+        reason: 'missing tokens'
+      }).catch((err) => this.logger.debug({err}, 'CallSession:_notifyTaskStatus - Error sending'));
+    }
+    let res;
+    try {
+      res = await this.ttsStreamingBuffer?.bufferTokens(tokens);
+      this.logger.info({id, res}, 'CallSession:_lccTtsTokens - tts:tokens-result');
+    } catch (err) {
+      this.logger.info(err, 'CallSession:_lccTtsTokens');
+    }
+    this.requestor.request('tts:tokens-result', '/tokens-result', {id, ...res})
+      .catch((err) => this.logger.debug({err}, 'CallSession:_notifyTaskStatus - Error sending'));
+  }
+
+  _lccTtsFlush(opts) {
+    this.ttsStreamingBuffer?.flush(opts);
+  }
+
+  _lccTtsClear(opts) {
+    this.ttsStreamingBuffer?.clear(opts);
+  }
 
   /**
    * perform call hangup by jambonz
@@ -2027,6 +2127,18 @@ Duration=${duration} `
         this._lccLlmUpdate(data, call_sid);
         break;
 
+      case 'tts:tokens':
+        this._lccTtsTokens(data);
+        break;
+
+      case 'tts:flush':
+        this._lccTtsFlush(data);
+        break;
+
+      case 'tts:clear':
+        this._lccTtsClear(data);
+        break;
+
       default:
         this.logger.info(`CallSession:_onCommand - invalid command ${command}`);
     }
@@ -2221,6 +2333,8 @@ Duration=${duration} `
     // close all background tasks
     this.backgroundTaskManager.stopAll();
     this.clearOrRestoreActionHookDelayProcessor().catch((err) => {});
+
+    this.ttsStreamingBuffer?.stop();
   }
 
   /**
@@ -2776,6 +2890,37 @@ Duration=${duration} `
       this.verbHookSpan = null;
     }
   }
+
+  _onTtsStreamingEmpty() {
+    const task = this.currentTask;
+    if (task && TaskName.Say === task.name) {
+      task.notifyTtsStreamIsEmpty();
+    }
+  }
+
+  _onTtsStreamingPause() {
+    this.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_paused'})
+      .catch((err) => this.logger.info({err}, 'CallSession:_onTtsStreamingPause - Error sending'));
+  }
+
+  _onTtsStreamingResume() {
+    this.requestor?.request('tts:streaming-event', 'streaming-event', {event_type: 'stream_resumed'})
+      .catch((err) => this.logger.info({err}, 'CallSession:_onTtsStreamingResume - Error sending'));
+  }
+
+  async _onTtsStreamingConnectFailure(vendor) {
+    const {writeAlerts, AlertType} = this.srf.locals;
+    try {
+      await writeAlerts({
+        alert_type: AlertType.TTS_STREAMING_CONNECTION_FAILURE,
+        account_sid: this.accountSid,
+        vendor
+      });
+    } catch (error) {
+      this.logger.error({error}, 'Error writing WEBHOOK_CONNECTION_FAILURE alert');
+    }
+    this.logger.info({vendor}, 'CallSession:_onTtsStreamingConnectFailure - tts streaming connect failure');
+  }
 }
 
 module.exports = CallSession;

lib/tasks/config.js

@@ -16,7 +16,8 @@ class TaskConfig extends Task {
       'fillerNoise',
       'actionHookDelayAction',
       'boostAudioSignal',
-      'vad'
+      'vad',
+      'ttsStream'
     ].forEach((k) => this[k] = this.data[k] || {});
 
     if ('notifyEvents' in this.data) {
@@ -45,6 +46,12 @@ class TaskConfig extends Task {
       };
       delete this.transcribeOpts.enable;
     }
+    if (this.ttsStream.enable) {
+      this.sayOpts = {
+        verb: 'say',
+        stream: true
+      };
+    }
 
     if (this.data.reset) {
       if (typeof this.data.reset === 'string') this.data.reset = [this.data.reset];
@@ -75,6 +82,7 @@ class TaskConfig extends Task {
   get hasVad() { return Object.keys(this.vad).length; }
   get hasFillerNoise() { return Object.keys(this.fillerNoise).length; }
   get hasReferHook() { return Object.keys(this.data).includes('referHook'); }
+  get hasTtsStream() { return Object.keys(this.ttsStream).length; }
 
   get summary() {
     const phrase = [];
@@ -106,6 +114,9 @@ class TaskConfig extends Task {
     if (this.onHoldMusic) phrase.push(`onHoldMusic: ${this.onHoldMusic}`);
     if ('boostAudioSignal' in this.data) phrase.push(`setGain ${this.data.boostAudioSignal}`);
     if (this.hasReferHook) phrase.push('set referHook');
+    if (this.hasTtsStream) {
+      phrase.push(`${this.ttsStream.enable ? 'enable' : 'disable'} ttsStream`);
+    }
     return `${this.name}{${phrase.join(',')}}`;
   }
 
@@ -305,6 +316,22 @@ class TaskConfig extends Task {
     if (this.hasReferHook) {
       cs.referHook = this.data.referHook;
     }
+
+    if (this.ttsStream.enable && this.sayOpts) {
+      this.sayOpts.synthesizer = this.hasSynthesizer ? this.synthesizer : {
+        vendor: cs.speechSynthesisVendor,
+        language: cs.speechSynthesisLanguage,
+        voice: cs.speechSynthesisVoice,
+        ...(cs.speechSynthesisLabel && {
+          label: cs.speechSynthesisLabel
+        })
+      };
+      this.logger.info({opts: this.gatherOpts}, 'Config: enabling ttsStream');
+      cs.enableBackgroundTtsStream(this.sayOpts);
+    } else if (!this.ttsStream.enable) {
+      this.logger.info('Config: disabling ttsStream');
+      cs.disableTtsStream();
+    }
   }
 
   async kill(cs) {

lib/tasks/gather.js

@@ -723,6 +723,7 @@ class TaskGather extends SttTask {
         this._fillerNoiseOn = false;  // in a race, if we just started audio it may sneak through here
         this.ep.api('uuid_break', this.ep.uuid)
           .catch((err) => this.logger.info(err, 'Error killing audio'));
+        cs.clearTtsStream();
       }
       return;
     }
@@ -1170,7 +1171,6 @@ class TaskGather extends SttTask {
     } catch (err) {  /*already logged error*/ }
 
     // Gather got response from hook, cancel actionHookDelay processing
-    this.logger.debug('TaskGather:_resolve - checking ahd');
     if (this.cs.actionHookDelayProcessor) {
       if (returnedVerbs) {
         this.logger.debug('TaskGather:_resolve - got response from action hook, cancelling actionHookDelay');

lib/tasks/say.js

@@ -1,3 +1,4 @@
+const assert = require('assert');
 const TtsTask = require('./tts-task');
 const {TaskName, TaskPreconditions} = require('../utils/constants');
 const pollySSMLSplit = require('polly-ssml-split');
@@ -35,24 +36,40 @@ class TaskSay extends TtsTask {
     super(logger, opts, parentTask);
     this.preconditions = TaskPreconditions.Endpoint;
 
-    this.text = (Array.isArray(this.data.text) ? this.data.text : [this.data.text])
+    assert.ok((typeof this.data.text === 'string' || Array.isArray(this.data.text)) || this.data.stream === true,
-      .map((t) => breakLengthyTextIfNeeded(this.logger, t))
+      'Say: either text or stream:true is required');
-      .flat();
 
-    this.loop = this.data.loop || 1;
+
-    this.isHandledByPrimaryProvider = true;
+    if (this.data.stream === true) {
+      this._isStreamingTts = true;
+      this.closeOnStreamEmpty = this.data.closeOnStreamEmpty !== false;
+    }
+    else {
+      this._isStreamingTts = false;
+      this.text = (Array.isArray(this.data.text) ? this.data.text : [this.data.text])
+        .map((t) => breakLengthyTextIfNeeded(this.logger, t))
+        .flat();
+
+      this.loop = this.data.loop || 1;
+      this.isHandledByPrimaryProvider = true;
+    }
   }
 
   get name() { return TaskName.Say; }
 
   get summary() {
-    for (let i = 0; i < this.text.length; i++) {
+    if (this.isStreamingTts) return `${this.name} streaming`;
-      if (this.text[i].startsWith('silence_stream')) continue;
+    else {
-      return `${this.name}{text=${this.text[i].slice(0, 15)}${this.text[i].length > 15 ? '...' : ''}}`;
+      for (let i = 0; i < this.text.length; i++) {
+        if (this.text[i].startsWith('silence_stream')) continue;
+        return `${this.name}{text=${this.text[i].slice(0, 15)}${this.text[i].length > 15 ? '...' : ''}}`;
+      }
+      return `${this.name}{${this.text[0]}}`;
     }
-    return `${this.name}{${this.text[0]}}`;
   }
 
+  get isStreamingTts() { return this._isStreamingTts; }
+
   _validateURL(urlString) {
     try {
       new URL(urlString);
@@ -63,14 +80,19 @@ class TaskSay extends TtsTask {
   }
 
   async exec(cs, obj) {
+    if (this.isStreamingTts && !cs.appIsUsingWebsockets) {
+      throw new Error('Say: streaming say verb requires applications to use the websocket API');
+    }
+
     try {
-      await this.handling(cs, obj);
+      if (this.isStreamingTts) await this.handlingStreaming(cs, obj);
+      else await this.handling(cs, obj);
       this.emit('playDone');
     } catch (error) {
       if (error instanceof SpeechCredentialError) {
         // if say failed due to speech credentials, alarm is writtern and error notification is sent
         // finished this say to move to next task.
-        this.logger.info('Say failed due to SpeechCredentialError, finished!');
+        this.logger.info({error}, 'Say failed due to SpeechCredentialError, finished!');
         this.emit('playDone');
         return;
       }
@@ -78,6 +100,35 @@ class TaskSay extends TtsTask {
     }
   }
 
+  async handlingStreaming(cs, {ep}) {
+    const {vendor, language, voice, label} = this.getTtsVendorData(cs);
+    const  credentials = cs.getSpeechCredentials(vendor, 'tts', label);
+    if (!credentials) {
+      throw new SpeechCredentialError(
+        `No text-to-speech service credentials for ${vendor} with labels: ${label} have been configured`);
+    }
+
+    try {
+
+      await this.setTtsStreamingChannelVars(vendor, language, voice, credentials, ep);
+
+      await cs.startTtsStream();
+
+      cs.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_open'})
+        .catch((err) => this.logger.info({err}, 'TaskSay:handlingStreaming - Error sending'));
+    } catch (err) {
+      this.logger.info({err}, 'TaskSay:handlingStreaming - Error setting channel vars');
+      cs.requestor?.request('tts:streaming-event', '/streaming-event', {event_type: 'stream_closed'})
+        .catch((err) => this.logger.info({err}, 'TaskSay:handlingStreaming - Error sending'));
+
+      //TODO: send tts:streaming-event with error?
+      this.notifyTaskDone();
+    }
+
+    await this.awaitTaskDone();
+    this.logger.info('TaskSay:handlingStreaming - done');
+  }
+
   async handling(cs, {ep}) {
     const {srf, accountSid:account_sid, callSid:target_sid} = cs;
     const {writeAlerts, AlertType} = srf.locals;
@@ -96,7 +147,7 @@ class TaskSay extends TtsTask {
     let voice =  this.synthesizer.voice && this.synthesizer.voice !== 'default' ?
       this.synthesizer.voice :
       cs.speechSynthesisVoice;
-    let label = this.taskInlcudeSynthesizer ? this.synthesizer.label : cs.speechSynthesisLabel;
+    let label = this.taskIncludeSynthesizer ? this.synthesizer.label : cs.speechSynthesisLabel;
 
     const fallbackVendor = this.synthesizer.fallbackVendor && this.synthesizer.fallbackVendor !== 'default' ?
       this.synthesizer.fallbackVendor :
@@ -107,7 +158,7 @@ class TaskSay extends TtsTask {
     const fallbackVoice =  this.synthesizer.fallbackVoice && this.synthesizer.fallbackVoice !== 'default' ?
       this.synthesizer.fallbackVoice :
       cs.fallbackSpeechSynthesisVoice;
-    const fallbackLabel = this.taskInlcudeSynthesizer ?
+    const fallbackLabel = this.taskIncludeSynthesizer ?
       this.synthesizer.fallbackLabel : cs.fallbackSpeechSynthesisLabel;
 
     if (cs.hasFallbackTts) {
@@ -253,6 +304,7 @@ class TaskSay extends TtsTask {
         this._playResolve = null;
       }
     }
+    this.notifyTaskDone();
   }
 
   _addStreamingTtsAttributes(span, evt) {
@@ -273,6 +325,13 @@ class TaskSay extends TtsTask {
     delete attrs['cache_filename']; //no value in adding this to the span
     span.setAttributes(attrs);
   }
+
+  notifyTtsStreamIsEmpty() {
+    if (this.isStreamingTts && this.closeOnStreamEmpty) {
+      this.logger.info('TaskSay:notifyTtsStreamIsEmpty - stream is empty, killing task');
+      this.notifyTaskDone();
+    }
+  }
 }
 
 const spanMapping = {

lib/tasks/tts-task.js

@@ -13,11 +13,11 @@ class TtsTask extends Task {
 
     this.earlyMedia = this.data.earlyMedia === true || (parentTask && parentTask.earlyMedia);
     /**
-     * Task use taskInlcudeSynthesizer to identify
+     * Task use taskIncludeSynthesizer to identify
-     * if taskInlcudeSynthesizer === true, use label from verb.synthesizer, even it's empty
+     * if taskIncludeSynthesizer === true, use label from verb.synthesizer, even it's empty
-     * if taskInlcudeSynthesizer === false, use label from application.synthesizer
+     * if taskIncludeSynthesizer === false, use label from application.synthesizer
      */
-    this.taskInlcudeSynthesizer = !!this.data.synthesizer;
+    this.taskIncludeSynthesizer = !!this.data.synthesizer;
     this.synthesizer = this.data.synthesizer || {};
     this.disableTtsCache = this.data.disableTtsCache;
     this.options = this.synthesizer.options || {};
@@ -44,6 +44,47 @@ class TtsTask extends Task {
     }
   }
 
+  getTtsVendorData(cs) {
+    const vendor = this.synthesizer.vendor && this.synthesizer.vendor !== 'default' ?
+      this.synthesizer.vendor :
+      cs.speechSynthesisVendor;
+    const language = this.synthesizer.language && this.synthesizer.language !== 'default' ?
+      this.synthesizer.language :
+      cs.speechSynthesisLanguage ;
+    const voice =  this.synthesizer.voice && this.synthesizer.voice !== 'default' ?
+      this.synthesizer.voice :
+      cs.speechSynthesisVoice;
+    const label = this.taskIncludeSynthesizer ? this.synthesizer.label : cs.speechSynthesisLabel;
+    return {vendor, language, voice, label};
+  }
+
+  async setTtsStreamingChannelVars(vendor, language, voice, credentials, ep) {
+    const {api_key, cartesia_model_id, cartesia_voice_id} = credentials;
+    let obj;
+
+    switch (vendor) {
+      case 'deepgram':
+        obj = {
+          DEEPGRAM_API_KEY: api_key,
+          DEEPGRAM_TTS_STREAMING_MODEL: voice
+        };
+        break;
+      case 'cartesia':
+        obj = {
+          CARTESIA_API_KEY: api_key,
+          CARTESIA_TTS_STREAMING_MODEL_ID: cartesia_model_id,
+          CARTESIA_TTS_STREAMING_VOICE_ID: cartesia_voice_id,
+          CARTESIA_TTS_STREAMING_LANGUAGE: language || 'en'
+        };
+        break;
+      default:
+        throw new Error(`vendor ${vendor} is not supported for tts streaming yet`);
+    }
+    this.logger.info({vendor, credentials, obj}, 'setTtsStreamingChannelVars');
+
+    await ep.set(obj);
+  }
+
   async _synthesizeWithSpecificVendor(cs, ep, {vendor, language, voice, label, preCache = false}) {
     const {srf, accountSid:account_sid} = cs;
     const {writeAlerts, AlertType, stats} = srf.locals;

lib/utils/background-task-manager.js

@@ -46,6 +46,9 @@ class BackgroundTaskManager extends Emitter {
       case 'transcribe':
         task = await this._initTranscribe(opts);
         break;
+      case 'ttsStream':
+        task = await this._initTtsStream(opts);
+        break;
       default:
         break;
     }
@@ -173,6 +176,25 @@ class BackgroundTaskManager extends Emitter {
     return task;
   }
 
+  // Initiate Tts Stream
+  async _initTtsStream(opts) {
+    let task;
+    try {
+      const t = normalizeJambones(this.logger, [opts]);
+      task = makeTask(this.logger, t[0]);
+      const resources = await this.cs._evaluatePreconditions(task);
+      const {span, ctx} = this.rootSpan.startChildSpan(`background-ttsStream:${task.summary}`);
+      task.span = span;
+      task.ctx = ctx;
+      task.exec(this.cs, resources)
+        .then(this._taskCompleted.bind(this, 'ttsStream', task))
+        .catch(this._taskError.bind(this, 'ttsStream', task));
+    } catch (err) {
+      this.logger.info(err, 'BackgroundTaskManager:_initTtsStream - Error creating ttsStream task');
+    }
+    return task;
+  }
+
   _taskCompleted(type, task) {
     this.logger.debug({type, task}, `BackgroundTaskManager:_taskCompleted: task completed, sticky: ${task.sticky}`);
     task.removeAllListeners();

lib/utils/constants.json

@@ -210,6 +210,8 @@
     "verb:status",
     "llm:event",
     "llm:tool-call",
+    "tts:tokens-result",
+    "tts:streaming-event",
     "jambonz:error"
   ],
   "RecordState": {
@@ -233,6 +235,33 @@
     "PartialMedia": "partial-media",
     "FullMedia": "full-media"
   },
+  "DeepgramTtsStreamingEvents": {
+    "Empty": "deepgram_tts_streaming::empty",
+    "ConnectFailure": "deepgram_tts_streaming::connect_failed",
+    "Connect": "deepgram_tts_streaming::connect"
+  },
+  "CartesiaTtsStreamingEvents": {
+    "Empty": "cartesia_tts_streaming::empty",
+    "ConnectFailure": "cartesia_tts_streaming::connect_failed",
+    "Connect": "cartesia_tts_streaming::connect"
+  },
+  "ElevenlabsTtsStreamingEvents": {
+    "Empty": "elevenlabs_tts_streaming::empty",
+    "ConnectFailure": "elevenlabs_tts_streaming::connect_failed",
+    "Connect": "elevenlabs_tts_streaming::connect"
+  },
+  "TtsStreamingEvents": {
+    "Empty": "tts_streaming::empty",
+    "Pause": "tts_streaming::pause",
+    "Resume": "tts_streaming::resume",
+    "ConnectFailure": "tts_streaming::connect_failed"
+  },
+  "TtsStreamingConnectionStatus": {
+    "NotConnected": "not_connected",
+    "Connected": "connected",
+    "Connecting": "connecting",
+    "Failed": "failed"
+  },
   "MAX_SIMRINGS": 10,
   "BONG_TONE": "tone_stream://v=-7;%(100,0,941.0,1477.0);v=-7;>=2;+=.1;%(1400,0,350,440)",
   "FS_UUID_SET_NAME": "fsUUIDs",

lib/utils/tts-streaming-buffer.js

@@ -0,0 +1,327 @@
+const Emitter = require('events');
+const assert = require('assert');
+const {
+  TtsStreamingEvents,
+  TtsStreamingConnectionStatus
+} = require('../utils/constants');
+const FEED_INTERVAL = 2000;
+const MAX_CHUNK_SIZE = 1800;
+const HIGH_WATER_BUFFER_SIZE = 5000;
+const LOW_WATER_BUFFER_SIZE = 1000;
+const MIN_INITIAL_WORDS = 4;
+
+const findSentenceBoundary = (text, limit) => {
+  const sentenceEndRegex = /[.!?](?=\s|$)/g;
+  let lastSentenceBoundary = -1;
+  let match;
+
+  while ((match = sentenceEndRegex.exec(text)) && match.index < limit) {
+    /* Ensure it's not a decimal point (e.g., "3.14") */
+    if (match.index === 0 || !/\d$/.test(text[match.index - 1])) {
+      lastSentenceBoundary = match.index + 1; // Include the punctuation
+    }
+  }
+  return lastSentenceBoundary;
+};
+
+const findWordBoundary = (text, limit) => {
+  const wordBoundaryRegex = /\s+/g;
+  let lastWordBoundary = -1;
+  let match;
+
+  while ((match = wordBoundaryRegex.exec(text)) && match.index < limit) {
+    lastWordBoundary = match.index;
+  }
+  return lastWordBoundary;
+};
+
+class TtsStreamingBuffer extends Emitter {
+  constructor(cs) {
+    super();
+    this.cs = cs;
+    this.logger = cs.logger;
+
+    this.tokens = '';
+    this.eventHandlers = [];
+    this._isFull = false;
+    this._connectionStatus = TtsStreamingConnectionStatus.NotConnected;
+    this._flushPending = false;
+    this._countSendsInThisTurn = 0;
+  }
+
+  get isEmpty() {
+    return this.tokens.length === 0;
+  }
+
+  get isFull() {
+    return this._isFull;
+  }
+
+  get size() {
+    return this.tokens.length;
+  }
+
+  get ep() {
+    return this.cs?.ep;
+  }
+
+  async start() {
+    assert.ok(
+      this._connectionStatus === TtsStreamingConnectionStatus.NotConnected,
+      'TtsStreamingBuffer:start already started, or has failed');
+
+    this.vendor = this.cs.getTsStreamingVendor();
+    if (!this.vendor) {
+      this.logger.info('TtsStreamingBuffer:start No TTS streaming vendor configured');
+      throw new Error('No TTS streaming vendor configured');
+    }
+
+    this.logger.info(`TtsStreamingBuffer:start Connecting to TTS streaming with vendor ${this.vendor}`);
+
+    this._connectionStatus = TtsStreamingConnectionStatus.Connecting;
+    try {
+      if (this.eventHandlers.length === 0) this._initHandlers(this.ep);
+      await  this._api(this.ep, [this.ep.uuid, 'connect']);
+    } catch (err) {
+      this.logger.info({err}, 'TtsStreamingBuffer:start Error connecting to TTS streaming');
+      this._connectionStatus = TtsStreamingConnectionStatus.Failed;
+    }
+  }
+
+  stop() {
+    clearTimeout(this.timer);
+    this.removeCustomEventListeners();
+    if (this.ep) {
+      this._api(this.ep, [this.ep.uuid, 'close'])
+        .catch((err) => this.logger.info({err}, 'TtsStreamingBuffer:kill Error closing TTS streaming'));
+    }
+    this.timer = null;
+    this.tokens = '';
+    this._connectionStatus = TtsStreamingConnectionStatus.NotConnected;
+  }
+
+  /**
+   * Add tokens to the buffer and start feeding them to the endpoint if necessary.
+   */
+  async bufferTokens(tokens) {
+
+    if (this._connectionStatus === TtsStreamingConnectionStatus.Failed) {
+      this.logger.info('TtsStreamingBuffer:bufferTokens TTS streaming connection failed, rejecting request');
+      return {status: 'failed', reason: `connection to ${this.vendor} failed`};
+    }
+
+    const starting = this.tokens === '';
+    const displayedTokens = tokens.length <= 40 ? tokens : tokens.substring(0, 40);
+    const totalLength = tokens.length;
+
+    /* if we crossed the high water mark, reject the request */
+    if (this.tokens.length + totalLength > HIGH_WATER_BUFFER_SIZE) {
+      this.logger.info(
+        `TtsStreamingBuffer:bufferTokensTTS buffer is full, rejecting request to buffer ${totalLength} tokens`);
+
+      if (!this._isFull) {
+        this._isFull = true;
+        this.emit(TtsStreamingEvents.Pause);
+      }
+      return {status: 'failed', reason: 'full'};
+    }
+
+    this.logger.debug(
+      `TtsStreamingBuffer:bufferTokens "${displayedTokens}" (length: ${totalLength}), starting? ${starting}`
+    );
+    this.tokens += (tokens || '');
+
+    const  leftoverTokens = await this._feedTokens();
+
+    /* do we need to start a timer to periodically feed tokens to the endpoint? */
+    if (starting && leftoverTokens > 0) {
+      assert(!this.timer);
+      this.timer = setInterval(async() => {
+        const remaining = await this._feedTokens();
+        if (remaining === 0) {
+          clearInterval(this.timer);
+          this.timer = null;
+        }
+      }, FEED_INTERVAL);
+    }
+
+    return {status: 'ok'};
+  }
+
+  flush() {
+    this.logger.debug('TtsStreamingBuffer:flush');
+    if (this._connectionStatus === TtsStreamingConnectionStatus.Connecting) {
+      this.logger.debug('TtsStreamingBuffer:flush TTS stream is not quite ready - wait for connect');
+      this._flushPending = true;
+      return;
+    }
+    else if (this._connectionStatus === TtsStreamingConnectionStatus.Connected) {
+      this._countSendsInThisTurn = 0;
+      this._api(this.ep, [this.ep.uuid, 'flush'])
+        .catch((err) => this.logger.info({err},
+          `TtsStreamingBuffer:flush Error flushing TTS streaming: ${JSON.stringify(err)}`));
+    }
+  }
+
+  clear() {
+    this.logger.debug('TtsStreamingBuffer:clear');
+
+    if (this._connectionStatus !== TtsStreamingConnectionStatus.Connected) return;
+    clearTimeout(this.timer);
+    this._api(this.ep, [this.ep.uuid, 'clear'])
+      .catch((err) => this.logger.info({err}, 'TtsStreamingBuffer:clear Error clearing TTS streaming'));
+    this.tokens = '';
+    this.timer = null;
+    this._isFull = false;
+  }
+
+  /**
+   * Send the next chunk of tokens to the endpoint (max 2000 chars)
+   * Return the number of tokens left in the buffer.
+   */
+  async _feedTokens() {
+    this.logger.debug({tokens: this.tokens}, '_feedTokens');
+
+    try {
+      if (!this.cs.isTtsStreamOpen || !this.ep || !this.tokens) {
+        this.logger.debug('TTS stream is not open or no tokens to send');
+        return this.tokens?.length || 0;
+      }
+
+      if (this._connectionStatus === TtsStreamingConnectionStatus.NotConnected ||
+        this._connectionStatus === TtsStreamingConnectionStatus.Failed) {
+        this.logger.debug('TtsStreamingBuffer:_feedTokens TTS stream is not connected');
+        return this.tokens.length;
+      }
+
+      if (this._connectionStatus === TtsStreamingConnectionStatus.Connecting) {
+        this.logger.debug('TtsStreamingBuffer:_feedTokens TTS stream is not ready, waiting for connect');
+        return this.tokens.length;
+      }
+
+      /**
+       * Rules:
+       * 1. If this is our first send, we must have at least N words
+       * 2. Otherwise, must EITHER have N words OR be the ending of a sentence
+       *
+       * When sending, send the max size possible, capped at a limit to avoid overwhelming the server.
+       */
+
+      /* must have at least N words, or be the ending of a sentence */
+      const words = this.tokens.split(' ').length;
+      if (words < MIN_INITIAL_WORDS) {
+        const endsWithPunctuation = /[.!?]$/.test(this.tokens);
+        if (!endsWithPunctuation || this._countSendsInThisTurn === 0) {
+          this.logger.debug(`TtsStreamingBuffer:_feedTokens: only ${words} words to send, waiting for more`);
+          return this.tokens.length;
+        }
+      }
+
+      const limit = Math.min(MAX_CHUNK_SIZE, this.tokens.length);
+      let chunkEnd = findSentenceBoundary(this.tokens, limit);
+
+      if (chunkEnd === -1) {
+        this.logger.debug('TtsStreamingBuffer:_feedTokens: no sentence boundary found, look for word boundary');
+        chunkEnd = findWordBoundary(this.tokens, limit);
+      }
+
+      if (chunkEnd === -1) {
+        chunkEnd = limit;
+      }
+
+      const chunk = this.tokens.slice(0, chunkEnd);
+      this.tokens = this.tokens.slice(chunkEnd); // Remove sent chunk
+
+      /* freeswitch looks for sequence of 2 newlines to determine end of message, so insert a space */
+      const modifiedChunk = chunk.replace(/\n\n/g, '\n \n');
+
+      if (modifiedChunk.length > 0) {
+        try {
+          this._countSendsInThisTurn++;
+          this.logger.debug({tokens: modifiedChunk},
+            `TtsStreamingBuffer:_feedTokens: sending tokens, in send#${this._countSendsInThisTurn}`);
+          await this._api(this.ep, [this.ep.uuid, 'send', modifiedChunk]);
+        } catch (err) {
+          this.logger.info({err}, 'TtsStreamingBuffer:_feedTokens Error sending TTS chunk');
+        }
+
+        this.logger.debug(`TtsStreamingBuffer:_feedTokens: sent ${chunk.length}, remaining: ${this.tokens.length}`);
+
+        if (this.isFull && this.tokens.length <= LOW_WATER_BUFFER_SIZE) {
+          this.logger.info('TtsStreamingBuffer:_feedTokens TTS streaming buffer is no longer full');
+          this._isFull = false;
+          this.emit(TtsStreamingEvents.Resume);
+        }
+      }
+    } catch (err) {
+      this.logger.info({err}, 'TtsStreamingBuffer:_feedTokens Error sending TTS chunk');
+      this.tokens = '';
+    }
+
+    if (0 === this.tokens.length && this.timer) {
+      clearTimeout(this.timer);
+      this.timer = null;
+    }
+
+    return this.tokens.length;
+  }
+
+  async _api(ep, args) {
+    const apiCmd = `uuid_${this.vendor}_tts_streaming`;
+    const res = await ep.api(apiCmd, `^^|${args.join('|')}`);
+    if (!res.body?.startsWith('+OK')) {
+      throw new Error({args}, `Error calling ${apiCmd}: ${res.body}`);
+    }
+  }
+
+  _onConnectFailure(vendor) {
+    this.logger.info(`streaming tts connection failed to ${vendor}`);
+    this._connectionStatus = TtsStreamingConnectionStatus.Failed;
+    this.tokens = '';
+    this.emit(TtsStreamingEvents.ConnectFailure, {vendor});
+  }
+
+  async _onConnect(vendor) {
+    this.logger.info(`streaming tts connection made to ${vendor}`);
+    this._connectionStatus = TtsStreamingConnectionStatus.Connected;
+    if (this.tokens.length > 0) {
+      await this._feedTokens();
+    }
+    if (this._flushPending) {
+      this.flush();
+      this._flushPending = false;
+    }
+  }
+
+  _onTtsEmpty(vendor) {
+    this.emit(TtsStreamingEvents.Empty, {vendor});
+  }
+
+  addCustomEventListener(ep, event, handler) {
+    this.eventHandlers.push({ep, event, handler});
+    ep.addCustomEventListener(event, handler);
+  }
+
+  removeCustomEventListeners() {
+    this.eventHandlers.forEach((h) => h.ep.removeCustomEventListener(h.event, h.handler));
+  }
+
+  _initHandlers(ep) {
+    [
+      // DH: add other vendors here as modules are added
+      'deepgram',
+      'cartesia',
+      'elevenlabs'
+    ].forEach((vendor) => {
+      const eventClassName = `${vendor.charAt(0).toUpperCase() + vendor.slice(1)}TtsStreamingEvents`;
+      const eventClass = require('../utils/constants')[eventClassName];
+      if (!eventClass) throw new Error(`Event class for vendor ${vendor} not found`);
+
+      this.addCustomEventListener(ep, eventClass.Connect, this._onConnect.bind(this, vendor));
+      this.addCustomEventListener(ep, eventClass.ConnectFailure, this._onConnectFailure.bind(this, vendor));
+      this.addCustomEventListener(ep, eventClass.Empty, this._onTtsEmpty.bind(this, vendor));
+    });
+  }
+}
+
+module.exports = TtsStreamingBuffer;

lib/utils/ws-requestor.js

@@ -12,6 +12,20 @@ const {
   JAMBONES_WS_MAX_PAYLOAD,
   HTTP_USER_AGENT_HEADER
 } = require('../config');
+const MTYPE_WANTS_ACK = [
+  'call:status',
+  'verb:status',
+  'jambonz:error',
+  'llm:event',
+  'llm:tool-call',
+  'tts:streaming-event',
+  'tts:tokens-result',
+];
+const MTYPE_NO_DATA = [
+  'llm:tool-output',
+  'tts:flush',
+  'tts:clear'
+];
 
 class WsRequestor extends BaseRequestor {
   constructor(logger, account_sid, hook, secret) {
@@ -44,7 +58,7 @@ class WsRequestor extends BaseRequestor {
   async request(type, hook, params, httpHeaders = {}) {
     assert(HookMsgTypes.includes(type));
     const url = hook.url || hook;
-    const wantsAck = !['call:status', 'verb:status', 'jambonz:error', 'llm:event', 'llm:tool-call'].includes(type);
+    const wantsAck = !MTYPE_WANTS_ACK.includes(type);
 
     if (this.maliciousClient) {
       this.logger.info({url: this.url}, 'WsRequestor:request - discarding msg to malicious client');
@@ -408,7 +422,7 @@ class WsRequestor extends BaseRequestor {
 
         case 'command':
           assert.ok(command, 'command property not supplied');
-          assert.ok(data || command === 'llm:tool-output', 'data property not supplied');
+          assert.ok(data || MTYPE_NO_DATA.includes(command), 'data property not supplied');
           this._recvCommand(msgid, command, call_sid, queueCommand, tool_call_id, data);
           break;
 

package-lock.json

@@ -17,7 +17,7 @@
         "@jambonz/realtimedb-helpers": "^0.8.8",
         "@jambonz/speech-utils": "^0.1.22",
         "@jambonz/stats-collector": "^0.1.10",
-        "@jambonz/time-series": "^0.2.12",
+        "@jambonz/time-series": "^0.2.13",
         "@jambonz/verb-specifications": "^0.0.90",
         "@opentelemetry/api": "^1.8.0",
         "@opentelemetry/exporter-jaeger": "^1.23.0",
@@ -1572,10 +1572,9 @@
       }
     },
     "node_modules/@jambonz/time-series": {
-      "version": "0.2.12",
+      "version": "0.2.13",
-      "resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.12.tgz",
+      "resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.13.tgz",
-      "integrity": "sha512-EYw5f1QasblWrP2K/NabpJYkQm8XOCP1fem8luO8c7Jm8YTBwI+Ge3zB7rpU8ruoVdbrTot/pcihhTqzq4IYqA==",
+      "integrity": "sha512-Kj+l+YUnI27zZA4qoPRzjN7L82W7GuMXYq9ttDjXQ0ZBIdOLAzJjB6R3jJ3b+mvoNEQ6qG5MUtfoc6CpTFH5lw==",
-      "license": "MIT",
       "dependencies": {
         "debug": "^4.3.1",
         "influx": "^5.9.3"
@@ -3558,10 +3557,11 @@
       "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ=="
     },
     "node_modules/cross-spawn": {
-      "version": "7.0.3",
+      "version": "7.0.6",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
-      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "path-key": "^3.1.0",
         "shebang-command": "^2.0.0",
@@ -4545,9 +4545,9 @@
       }
     },
     "node_modules/express": {
-      "version": "4.21.1",
+      "version": "4.21.2",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
-      "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==",
+      "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
       "dependencies": {
         "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
@@ -4568,7 +4568,7 @@
         "methods": "~1.1.2",
         "on-finished": "2.4.1",
         "parseurl": "~1.3.3",
-        "path-to-regexp": "0.1.10",
+        "path-to-regexp": "0.1.12",
         "proxy-addr": "~2.0.7",
         "qs": "6.13.0",
         "range-parser": "~1.2.1",
@@ -4583,6 +4583,10 @@
       },
       "engines": {
         "node": ">= 0.10.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
       }
     },
     "node_modules/express-validator": {
@@ -7451,10 +7455,9 @@
       "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="
     },
     "node_modules/path-to-regexp": {
-      "version": "0.1.10",
+      "version": "0.1.12",
-      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz",
+      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz",
-      "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==",
+      "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ=="
-      "license": "MIT"
     },
     "node_modules/peek-readable": {
       "version": "4.1.0",
@@ -10745,9 +10748,9 @@
       }
     },
     "@jambonz/time-series": {
-      "version": "0.2.12",
+      "version": "0.2.13",
-      "resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.12.tgz",
+      "resolved": "https://registry.npmjs.org/@jambonz/time-series/-/time-series-0.2.13.tgz",
-      "integrity": "sha512-EYw5f1QasblWrP2K/NabpJYkQm8XOCP1fem8luO8c7Jm8YTBwI+Ge3zB7rpU8ruoVdbrTot/pcihhTqzq4IYqA==",
+      "integrity": "sha512-Kj+l+YUnI27zZA4qoPRzjN7L82W7GuMXYq9ttDjXQ0ZBIdOLAzJjB6R3jJ3b+mvoNEQ6qG5MUtfoc6CpTFH5lw==",
       "requires": {
         "debug": "^4.3.1",
         "influx": "^5.9.3"
@@ -12243,9 +12246,9 @@
       "integrity": "sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ=="
     },
     "cross-spawn": {
-      "version": "7.0.3",
+      "version": "7.0.6",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
-      "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+      "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
       "dev": true,
       "requires": {
         "path-key": "^3.1.0",
@@ -13005,9 +13008,9 @@
       }
     },
     "express": {
-      "version": "4.21.1",
+      "version": "4.21.2",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
-      "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==",
+      "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
       "requires": {
         "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
@@ -13028,7 +13031,7 @@
         "methods": "~1.1.2",
         "on-finished": "2.4.1",
         "parseurl": "~1.3.3",
-        "path-to-regexp": "0.1.10",
+        "path-to-regexp": "0.1.12",
         "proxy-addr": "~2.0.7",
         "qs": "6.13.0",
         "range-parser": "~1.2.1",
@@ -15151,9 +15154,9 @@
       "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="
     },
     "path-to-regexp": {
-      "version": "0.1.10",
+      "version": "0.1.12",
-      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz",
+      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz",
-      "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w=="
+      "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ=="
     },
     "peek-readable": {
       "version": "4.1.0",

package.json

@@ -33,8 +33,8 @@
     "@jambonz/realtimedb-helpers": "^0.8.8",
     "@jambonz/speech-utils": "^0.1.22",
     "@jambonz/stats-collector": "^0.1.10",
-    "@jambonz/time-series": "^0.2.12",
     "@jambonz/verb-specifications": "^0.0.90",
+    "@jambonz/time-series": "^0.2.13",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/exporter-jaeger": "^1.23.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.50.0",