fix uncaught exception with llm streaming

* turn down some logging

* add startup logging

* wip

* lint

app.js

@@ -29,6 +29,12 @@ const {LifeCycleEvents, FS_UUID_SET_NAME, SystemState, FEATURE_SERVER} = require
 const installSrfLocals = require('./lib/utils/install-srf-locals');
 const createHttpListener = require('./lib/utils/http-listener');
 const healthCheck = require('@jambonz/http-health-check');
+const ProcessMonitor = require('./lib/utils/process-monitor');
+const monitor = new ProcessMonitor(logger);
+
+// Log startup
+monitor.logStartup();
+monitor.setupSignalHandlers();
 
 logger.on('level-change', (lvl, _val, prevLvl, _prevVal, instance) => {
   if (logger !== instance) {

lib/session/call-session.js

@@ -3082,7 +3082,7 @@ Duration=${duration} `
       task.notifyTtsStreamIsEmpty();
     } else if (
       // If Gather nested say task is streaming
-      TaskName.Gather === task.name && task.sayTask && task.sayTask.isStreamingTts) {
+      task && TaskName.Gather === task.name && task.sayTask && task.sayTask.isStreamingTts) {
       const sayTask = task.sayTask;
       sayTask.notifyTtsStreamIsEmpty();
     }

lib/tasks/gather.js

@@ -368,6 +368,9 @@ class TaskGather extends SttTask {
 
   _onDtmf(cs, ep, evt) {
     this.logger.debug(evt, 'TaskGather:_onDtmf');
+    if (!this._timeoutTimer && this.timeout > 0) {
+      this._startTimer();
+    }
     clearTimeout(this.interDigitTimer);
     let resolved = false;
     if (this.dtmfBargein) {
@@ -696,6 +699,7 @@ class TaskGather extends SttTask {
 
   _startTimer() {
     if (0 === this.timeout) return;
+    this.logger.debug(`Starting timoutTimer of ${this.timeout}ms`);
     this._clearTimer();
     this._timeoutTimer = setTimeout(() => {
       if (this.isContinuousAsr) this._startAsrTimer();

lib/tasks/llm/llms/ultravox_s2s.js

@@ -218,7 +218,7 @@ class TaskLlmUltravox_S2S extends Task {
   async _onServerEvent(_ep, evt) {
     let endConversation = false;
     const type = evt.type;
-    this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent');
+    //this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent');
 
     /* server errors of some sort */
     if (type === 'error') {

lib/tasks/tts-task.js

@@ -293,7 +293,18 @@ class TtsTask extends Task {
                 vendor,
                 language,
                 characters: text.length,
-                elapsedTime: rtt
+                elapsedTime: rtt,
+                servedFromCache,
+                'id': this.id
+              });
+            }
+            if (servedFromCache) {
+              this.notifyStatus({
+                event: 'synthesized-audio',
+                vendor,
+                language,
+                servedFromCache,
+                'id': this.id
               });
             }
           }

lib/utils/process-monitor.js

@@ -0,0 +1,91 @@
+// lib/utils/process-monitor.js
+const fs = require('fs');
+const path = require('path');
+
+class ProcessMonitor {
+  constructor(logger) {
+    this.logger = logger;
+    this.packageInfo = this.getPackageInfo();
+    this.processName = this.packageInfo.name || 'unknown-app';
+  }
+
+  getPackageInfo() {
+    try {
+      const packagePath = path.join(process.cwd(), 'package.json');
+      return JSON.parse(fs.readFileSync(packagePath, 'utf8'));
+    } catch (e) {
+      return { name: 'unknown', version: 'unknown' };
+    }
+  }
+
+  logStartup(additionalInfo = {}) {
+    const startupInfo = {
+      msg: `${this.processName} started`,
+      app_name: this.processName,
+      app_version: this.packageInfo.version,
+      pid: process.pid,
+      ppid: process.ppid,
+      pm2_instance_id: process.env.NODE_APP_INSTANCE || 'not_pm2',
+      pm2_id: process.env.pm_id,
+      is_pm2: !!process.env.PM2,
+      node_version: process.version,
+      uptime: process.uptime(),
+      timestamp: new Date().toISOString(),
+      ...additionalInfo
+    };
+
+    this.logger.info(startupInfo);
+    return startupInfo;
+  }
+
+  setupSignalHandlers() {
+    // Log when we receive signals that would cause restart
+    process.on('SIGINT', () => {
+      this.logger.info({
+        msg: 'SIGINT received',
+        app_name: this.processName,
+        pid: process.pid,
+        ppid: process.ppid,
+        uptime: process.uptime(),
+        timestamp: new Date().toISOString()
+      });
+      process.exit(0);
+    });
+
+    process.on('SIGTERM', () => {
+      this.logger.info({
+        msg: 'SIGTERM received',
+        app_name: this.processName,
+        pid: process.pid,
+        ppid: process.ppid,
+        uptime: process.uptime(),
+        timestamp: new Date().toISOString()
+      });
+      process.exit(0);
+    });
+
+    process.on('uncaughtException', (error) => {
+      this.logger.error({
+        msg: 'Uncaught exception - process will restart',
+        app_name: this.processName,
+        error: error.message,
+        stack: error.stack,
+        pid: process.pid,
+        timestamp: new Date().toISOString()
+      });
+      process.exit(1);
+    });
+
+    process.on('unhandledRejection', (reason, promise) => {
+      this.logger.error({
+        msg: 'Unhandled rejection',
+        app_name: this.processName,
+        reason,
+        pid: process.pid,
+        timestamp: new Date().toISOString()
+      });
+    });
+  }
+}
+
+module.exports = ProcessMonitor;

lib/utils/ws-requestor.js

@@ -293,7 +293,7 @@ class WsRequestor extends BaseRequestor {
 
       /* send the message */
       this.ws.send(JSON.stringify(obj), async() => {
-        this.logger.debug({obj}, `WsRequestor:request websocket: sent (${url})`);
+        if (obj.type !== 'llm:event') this.logger.debug({obj}, `WsRequestor:request websocket: sent (${url})`);
         // If session:reconnect is waiting for ack, hold here until ack to send queuedMsgs
         if (this._reconnectPromise) {
           try {

package-lock.json

@@ -15,7 +15,7 @@
         "@jambonz/http-health-check": "^0.0.1",
         "@jambonz/mw-registrar": "^0.2.7",
         "@jambonz/realtimedb-helpers": "^0.8.15",
-        "@jambonz/speech-utils": "^0.2.17",
+        "@jambonz/speech-utils": "^0.2.19",
         "@jambonz/stats-collector": "^0.1.10",
         "@jambonz/time-series": "^0.2.14",
         "@jambonz/verb-specifications": "^0.0.113",
@@ -1376,10 +1376,9 @@
       }
     },
     "node_modules/@jambonz/speech-utils": {
-      "version": "0.2.17",
-      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.2.17.tgz",
-      "integrity": "sha512-YcBq8bOvo5QfQJQXP4e2N0sCPqkoW0jNM4eHhspJotJc7V9DB3T5VVA6SGUsDXDZUwyT0JPI3Tx+xof02eWQ2Q==",
-      "license": "MIT",
+      "version": "0.2.19",
+      "resolved": "https://registry.npmjs.org/@jambonz/speech-utils/-/speech-utils-0.2.19.tgz",
+      "integrity": "sha512-7Sw2pgmsMg/3y3PRhRts/oQrtMlowNS1dn6DgduiHviKSclJNx8oY8S7X8wsBQCe3xdFZYEDxfn9vpcGm4lqZw==",
       "dependencies": {
         "23": "^0.0.0",
         "@aws-sdk/client-polly": "^3.496.0",

package.json

@@ -31,7 +31,7 @@
     "@jambonz/http-health-check": "^0.0.1",
     "@jambonz/mw-registrar": "^0.2.7",
     "@jambonz/realtimedb-helpers": "^0.8.15",
-    "@jambonz/speech-utils": "^0.2.17",
+    "@jambonz/speech-utils": "^0.2.19",
     "@jambonz/stats-collector": "^0.1.10",
     "@jambonz/time-series": "^0.2.14",
     "@jambonz/verb-specifications": "^0.0.113",