Feat/ambient sounds (#678)

* initial support for coaching mode in conference * wip * wip * add support for answer verb * wip * wip * wip * wip * wip * updates to rename option to dub * wip * wip * wip * update verb-specs * wip * wip * wip * wip * wip * wip * wip * wip * add option to boost audio signal in main channel * wip * wip * wip * wip * wip * wip * for now, bypass use of streaming apis when generating tts audio for dub tracks * add nested dub to dial * wip * add support for filler noise * kill filler noise when gather killed * wip * wip * while using sayOnTrack, we have to enclose the say command in double quotes * disableTtsStreaming = false * allow transcribe of b leg only on dial verb * dub.say can either be text or object like say verb with text and synthesizer * remove loop for sayOnTrack * update speech-utils * fixes for testing transcribe verb and support for dub and boostAudioSignal in lcc commands * add dial.boostAudioSignal * fix bug where session-level recognizer settings incorrectly overwrite verb-level settings * update verb specs * update dial to support array of dub verbs * fix bug setting gain * lint * wip * update speech-utils * use new endpoint methods for mod_dub --------- Co-authored-by: Dave Horton <daveh@beachdognet.com>
2025-12-20 16:50:39 +00:00 · 2024-03-24 03:23:57 +07:00
parent ec58232b61
commit 5b1d8a8ff3
18 changed files with 915 additions and 278 deletions
--- a/lib/tasks/dub.js
+++ b/lib/tasks/dub.js
@@ -0,0 +1,142 @@
+const {TaskName} = require('../utils/constants');
+const TtsTask = require('./tts-task');
+const assert = require('assert');
+const parseDecibels = require('../utils/parse-decibels');
+
+/**
+ * Dub task: add or remove additional audio tracks into the call
+ */
+class TaskDub extends TtsTask {
+  constructor(logger, opts, parentTask) {
+    super(logger, opts, parentTask);
+
+    this.logger.debug({opts: this.data}, 'TaskDub constructor');
+    ['action', 'track', 'play', 'say', 'loop'].forEach((prop) => {
+      this[prop] = this.data[prop];
+    });
+    this.gain = parseDecibels(this.data.gain);
+
+    assert.ok(this.action, 'TaskDub: action is required');
+    assert.ok(this.track, 'TaskDub: track is required');
+  }
+
+  get name() { return TaskName.Dub; }
+
+  async exec(cs, {ep}) {
+    super.exec(cs);
+
+    try {
+      switch (this.action) {
+        case 'addTrack':
+          await this._addTrack(cs, ep);
+          break;
+        case 'removeTrack':
+          await this._removeTrack(cs, ep);
+          break;
+        case 'silenceTrack':
+          await this._silenceTrack(cs, ep);
+          break;
+        case 'playOnTrack':
+          await this._playOnTrack(cs, ep);
+          break;
+        case 'sayOnTrack':
+          await this._sayOnTrack(cs, ep);
+          break;
+        default:
+          throw new Error(`TaskDub: unsupported action ${this.action}`);
+      }
+    } catch (err) {
+      this.logger.error(err, 'Error executing dub task');
+    }
+  }
+
+  async _addTrack(cs, ep) {
+    this.logger.info(`adding track: ${this.track}`);
+    await ep.dub({
+      action: 'addTrack',
+      track: this.track
+    });
+
+    if (this.play) await this._playOnTrack(cs, ep);
+    else if (this.say) await this._sayOnTrack(cs, ep);
+  }
+
+  async _removeTrack(_cs, ep) {
+    this.logger.info(`removing track: ${this.track}`);
+    await ep.dub({
+      action: 'removeTrack',
+      track: this.track
+    });
+  }
+
+  async _silenceTrack(_cs, ep) {
+    this.logger.info(`silencing track: ${this.track}`);
+    await ep.dub({
+      action: 'silenceTrack',
+      track: this.track
+    });
+  }
+
+  async _playOnTrack(_cs, ep) {
+    this.logger.info(`playing on track: ${this.track}`);
+    await ep.dub({
+      action: 'playOnTrack',
+      track: this.track,
+      play: this.play,
+      loop: this.loop ? 'loop' : 'once',
+      gain: this.gain
+    });
+  }
+
+  async _sayOnTrack(cs, ep) {
+    const text = this.say.text || this.say;
+    this.synthesizer = this.say.synthesizer || {};
+
+    if (Object.keys(this.synthesizer).length) {
+      this.logger.info({synthesizer: this.synthesizer},
+        `saying on track ${this.track}: ${text} with synthesizer options`);
+    }
+    else {
+      this.logger.info(`saying on track ${this.track}: ${text}`);
+    }
+    this.synthesizer = this.synthesizer || {};
+
+    this.text = [text];
+
+    const vendor = this.synthesizer.vendor && this.synthesizer.vendor !== 'default' ?
+      this.synthesizer.vendor :
+      cs.speechSynthesisVendor;
+    const language = this.synthesizer.language && this.synthesizer.language !== 'default' ?
+      this.synthesizer.language :
+      cs.speechSynthesisLanguage ;
+    const voice =  this.synthesizer.voice && this.synthesizer.voice !== 'default' ?
+      this.synthesizer.voice :
+      cs.speechSynthesisVoice;
+    const label = this.synthesizer.label && this.synthesizer.label !== 'default' ?
+      this.synthesizer.label :
+      cs.speechSynthesisLabel;
+
+    const disableTtsStreaming = false;
+    const filepath = await this._synthesizeWithSpecificVendor(cs, ep, {
+      vendor, language, voice, label, disableTtsStreaming
+    });
+    assert.ok(filepath.length === 1, 'TaskDub: no filepath returned from synthesizer');
+
+    const path = filepath[0];
+    if (!path.startsWith('say:{')) {
+      /* we have a local file of mp3 or r8 of synthesized speech audio to play */
+      this.play = path;
+      await this._playOnTrack(cs, ep);
+    }
+    else {
+      await ep.dub({
+        action: 'sayOnTrack',
+        track: this.track,
+        say: path,
+        gain: this.gain
+      });
+    }
+  }
+}
+
+module.exports = TaskDub;