Feat/mod dub v2 (#22)

* support tts elevenlabs to mod_dub Signed-off-by: Quan HL <quan.luuhoang8@gmail.com> * wip Signed-off-by: Quan HL <quan.luuhoang8@gmail.com> * wip * wip * fix aws race condition when 2 start transcribes are sent at the same instant * wip * wip * wip * allow queue play on track * wip Signed-off-by: Quan HL <quan.luuhoang8@gmail.com> * revert change for aws transcribe Signed-off-by: Quan HL <quan.luuhoang8@gmail.com> * fix type * wip * wip * rename parameters * rename paramters * wip * wip * wip * wip * wip * bug: there exists scenarios where callback is not defined * wip * wip * revert unintended changes to mod_google_transcribe * fix bugs w/ streaming tts simplified arg parsing to use freeswitch conventions --------- Signed-off-by: Quan HL <quan.luuhoang8@gmail.com> Co-authored-by: Quan HL <quan.luuhoang8@gmail.com>
2026-01-25 02:08:27 +00:00 · 2024-03-23 15:37:27 -04:00
parent 4e57f73c7e
commit 9cdc5fdfca
23 changed files with 1907 additions and 1612 deletions
@@ -1,8 +1,9 @@
 #include "mod_dub.h"
-#include "audio_downloader.h"
-#include "file_loader.h"
-
+#include "tts_vendor_parser.h"
+#include "track.h"
+#include "vector_math.h"
 #include <string>
+#include <queue>

 #include <switch.h>

@@ -15,124 +16,111 @@ typedef boost::circular_buffer<int16_t> CircularBuffer_t;

 extern "C" {

-  void init_dub_track(dub_track_t *track, char* trackName, int sampleRate) {
-    track->state = DUB_TRACK_STATE_READY;
-    track->trackName = strdup(trackName);
-    track->sampleRate = sampleRate;
-    track->circularBuffer = new CircularBuffer_t(INIT_BUFFER_SIZE);
+  Track* find_track_by_name(void** tracks, const std::string& trackName) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "find_track_by_name: searching for %s\n", trackName.c_str());
+
+    for (int i = 0; i < MAX_DUB_TRACKS; i++) {
+      Track* track = static_cast<Track*>(tracks[i]);
+      std::string name = track ? track->getTrackName() : "null";
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "find_track_by_name: offset %d: %s\n", i, name.c_str());
+      if (track && 0 == track->getTrackName().compare(trackName)) {
+        return track;
+      }
+    }
+    return nullptr;
  }

-  switch_status_t silence_dub_track(dub_track_t *track) {
-    assert(track);
-    switch (track->generator) {
-      case DUB_GENERATOR_TYPE_HTTP:
-        stop_audio_download(track->generatorId);
-        break;
-      case DUB_GENERATOR_TYPE_FILE:
-        stop_file_load(track->generatorId);
-        break;
-      case DUB_GENERATOR_TYPE_TTS:
-        //TODO
-        break;
-    }
-    CircularBuffer_t* buffer = reinterpret_cast<CircularBuffer_t*>(track->circularBuffer);
-    buffer->clear();
-    track->state = DUB_TRACK_STATE_READY;
-    track->generator = DUB_GENERATOR_TYPE_UNKNOWN;
-    track->generatorId = 0;
-
-    return SWITCH_STATUS_SUCCESS;
-  }
-
-  switch_status_t remove_dub_track(dub_track_t *track) {
-    assert(track);
-    switch (track->generator) {
-      case DUB_GENERATOR_TYPE_HTTP:
-        stop_audio_download(track->generatorId);
-        break;
-      case DUB_GENERATOR_TYPE_FILE:
-        stop_file_load(track->generatorId);
-        break;
-      case DUB_GENERATOR_TYPE_TTS:
-        //TODO
-        break;
-    }
-    CircularBuffer_t* buffer = reinterpret_cast<CircularBuffer_t*>(track->circularBuffer);
-    if (buffer) {
-      delete buffer;
-    }
-    if (track->trackName) {
-      free(track->trackName);
-    }
-    memset(track, 0, sizeof(dub_track_t));
-
-    return SWITCH_STATUS_SUCCESS;
-  }
-
-  switch_status_t play_dub_track(dub_track_t *track, switch_mutex_t *mutex, char* url, int loop, int gain) {
-    bool isHttp = strncmp(url, "http", 4) == 0;
-    if (track->state != DUB_TRACK_STATE_READY) {
-      silence_dub_track(track);
-    }
-    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "play_dub_track: starting %s download: %s\n", (isHttp ? "HTTP" : "file"), url);
-    int id = isHttp ?
-      start_audio_download(url, track->sampleRate, loop, gain, mutex, (CircularBuffer_t*) track->circularBuffer) :
-      start_file_load(url, track->sampleRate, loop, gain, mutex, (CircularBuffer_t*) track->circularBuffer);
-
-    if (id == INVALID_DOWNLOAD_ID) {
-      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "play_dub_track: failed to start audio download\n");
+  switch_status_t add_track(struct cap_cb* cb, char* trackName, int sampleRate) {
+    Track* existingTrack = find_track_by_name(cb->tracks, trackName);
+    if (existingTrack) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "add_track: track %s already exists\n", trackName);
      return SWITCH_STATUS_FALSE;
    }
-    track->state = DUB_TRACK_STATE_ACTIVE;
-    track->generatorId = id;
-    track->generator = isHttp ? DUB_GENERATOR_TYPE_HTTP : DUB_GENERATOR_TYPE_FILE;
-    track->gain = gain;
+
+    for (int i = 0; i < MAX_DUB_TRACKS; i++) {
+      if (!cb->tracks[i]) {
+        cb->tracks[i] = new Track(trackName, sampleRate);
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_track: added track %s at offset %d\n", trackName, i);
+        return SWITCH_STATUS_SUCCESS;
+      }
+    }
+
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "add_track: no room for track %s\n", trackName);
+    return SWITCH_STATUS_FALSE;
+  }
+
+  switch_status_t silence_dub_track(struct cap_cb* cb, char* trackName) {
+    Track* track = find_track_by_name(cb->tracks, trackName);
+    if (track) {
+      track->removeAllAudio();
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "silence_dub_track: silenced track %s\n", trackName);
+      return SWITCH_STATUS_SUCCESS;
+    }
+
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "silence_dub_track: track %s not found\n", trackName);
+    return SWITCH_STATUS_FALSE;
+  }
+
+  switch_status_t remove_dub_track(struct cap_cb* cb, char* trackName) {
+    for (int i = 0; i < MAX_DUB_TRACKS; i++) {
+      Track* track = static_cast<Track*>(cb->tracks[i]);
+      if (track && track->getTrackName() == trackName) {
+        track->removeAllAudio();
+        delete track;
+        cb->tracks[i] = nullptr;
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "remove_dub_track: removed track %s\n", trackName);
+        return SWITCH_STATUS_SUCCESS;
+      }
+    }
+
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "remove_dub_track: track %s not found\n", trackName);
+    return SWITCH_STATUS_FALSE;
+  }
+
+  switch_status_t play_dub_track(struct cap_cb* cb, char* trackName, char* url, int loop, int gain) {
+    bool isHttp = strncmp(url, "http", 4) == 0;
+    Track* track = find_track_by_name(cb->tracks, trackName);
+
+    if (!track) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "play_dub_track: track %s not found\n", trackName);
+      return SWITCH_STATUS_FALSE;
+    }
+
+    if (isHttp) {
+      track->queueHttpGetAudio(url, gain, loop);
+    }
+    else {
+      track->queueFileAudio(url, gain, loop);
+    }

    return SWITCH_STATUS_SUCCESS;
  }

-  switch_status_t say_dub_track(dub_track_t *track, switch_mutex_t *mutex, char* text, int gain) {
-    if (track->state != DUB_TRACK_STATE_READY) {
-      silence_dub_track(track); // wait...shouldnt we queue says?
-    }
-    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "say_dub_track: starting TTS\n");
-    
-    /**
-     * TODO:
-     * This is not implemented yet.  We can play TTS using using the playOnSay function
-     * because jambonz can generate local audio files using TTS vendors.
-     * However, we should probably at least implement support for elevenlabs streaming api
-     * here because it is so much faster.
-     * 
-     */
+  switch_status_t say_dub_track(struct cap_cb* cb, char* trackName, char* text, int gain) {
+    std::vector<std::string> headers;
+    std::string url, body;
+    Track* track = find_track_by_name(cb->tracks, trackName);

-    /*
-    track->state = DUB_TRACK_STATE_ACTIVE;
-    track->generatorId = id;
-    track->generator = DUB_GENERATOR_TYPE_TTS;
-    track->gain = gain;
-    */
+    if (!track) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "play_dub_track: track %s not found\n", trackName);
+      return SWITCH_STATUS_FALSE;
+    }
+    if (tts_vendor_parse_text(text, url, body, headers) != SWITCH_STATUS_SUCCESS) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "say_dub_track: failed to parse text\n");
+      return SWITCH_STATUS_FALSE;
+    }
+    track->queueHttpPostAudio(url, body, headers, gain);
    return SWITCH_STATUS_SUCCESS;
  }


  /* module load and unload */
  switch_status_t dub_init() {
-    switch_status_t status;
-    status = init_audio_downloader();
-    if (status == SWITCH_STATUS_SUCCESS) {
-      status = init_file_loader();
-    }
-    return status;
+    return SWITCH_STATUS_SUCCESS;
  }

  switch_status_t dub_cleanup() {
-    switch_status_t status;
-    status = deinit_audio_downloader();
-    if (status == SWITCH_STATUS_SUCCESS) {
-      status = deinit_file_loader();
-    }
-    return status;
+    return SWITCH_STATUS_SUCCESS;
  }

  switch_status_t dub_session_cleanup(switch_core_session_t *session, int channelIsClosing, switch_media_bug_t *bug) {
@@ -144,17 +132,18 @@ extern "C" {

      if (!switch_channel_get_private(channel, MY_BUG_NAME)) {
        // race condition
-        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s Bug is not attached (race).\n", switch_channel_get_name(channel));
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "%s Bug is not attached (race).\n", switch_channel_get_name(channel));
        switch_mutex_unlock(cb->mutex);
        return SWITCH_STATUS_FALSE;
      }
      switch_channel_set_private(channel, MY_BUG_NAME, NULL);

      for (int i = 0; i < MAX_DUB_TRACKS; i++) {
-        dub_track_t* track = &cb->tracks[i];
-        if (track->state != DUB_TRACK_STATE_INACTIVE) {
-    			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "dub_session_cleanup: cleared track %d:%s\n", i, track->trackName);
-          remove_dub_track(track);
+        Track* track = static_cast<Track*>(cb->tracks[i]);
+        if (track) {
+          track->removeAllAudio();
+          delete track;
+          cb->tracks[i] = nullptr;
        }
      }

@@ -162,7 +151,7 @@ extern "C" {
        switch_core_media_bug_remove(session, &bug);
      }

-			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "dub_session_cleanup: removed bug and cleared tracks\n");
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "dub_session_cleanup: removed bug and cleared tracks\n");
 			switch_mutex_unlock(cb->mutex);
    }
    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s dub_session_cleanup: Bug is not attached.\n", switch_channel_get_name(channel));
@@ -175,12 +164,17 @@ extern "C" {

    if (switch_mutex_trylock(cb->mutex) == SWITCH_STATUS_SUCCESS) {

-      /* check if any tracks are actively pushing audio */
-      int trackCount = 0;
+      /* check if any tracks have audio to contribute */
+      std::vector<Track*> activeTracks;
+      activeTracks.reserve(MAX_DUB_TRACKS);
      for (int i = 0; i < MAX_DUB_TRACKS; i++) {
-        if (cb->tracks[i].state == DUB_TRACK_STATE_ACTIVE) trackCount++;
+        if (cb->tracks[i]) {
+          auto track = static_cast<Track*>(cb->tracks[i]);
+          if (track->hasAudio_NoLock()) activeTracks.push_back(static_cast<Track*>(cb->tracks[i]));
+        }
      }
-      if (trackCount == 0 && cb->gain == 0) {
+
+      if (activeTracks.size() == 0 && cb->gain == 0) {
        switch_mutex_unlock(cb->mutex);
        return SWITCH_TRUE;
      }
@@ -189,31 +183,24 @@ extern "C" {
      int16_t *fp = reinterpret_cast<int16_t*>(rframe->data);

      rframe->channels = 1;
-      rframe->datalen = rframe->samples * rframe->channels * sizeof(int16_t);
+      rframe->datalen = rframe->samples * sizeof(int16_t);

      /* apply gain to audio in main channel if requested*/
      if (cb->gain != 0) {
-        switch_change_sln_volume_granular(fp, rframe->samples, cb->gain);
+        vector_change_sln_volume_granular(fp, rframe->samples, cb->gain);
      }

      /* now mux in the data from tracks */
-      for (int i = 0; i < rframe->samples; i++) {
-        int16_t input = fp[i];
-        int16_t value = input;
-        for (int j = 0; j < MAX_DUB_TRACKS; j++) {
-          dub_track_t* track = &cb->tracks[j];
-          if (track->state == DUB_TRACK_STATE_ACTIVE) {
-            CircularBuffer_t* buffer = reinterpret_cast<CircularBuffer_t*>(track->circularBuffer);
-            if (buffer && !buffer->empty()) {
-              int16_t sample = buffer->front();
-              buffer->pop_front();
-              value += sample;
-            }
-          }
+      for (auto track : activeTracks) {
+        int16_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
+        memset(data, 0, sizeof(data));
+        auto samples = track->retrieveAndClearAudio(data, rframe->samples);
+        if (samples > 0) {
+          vector_add(fp, data, rframe->samples);
        }
-        switch_normalize_to_16bit(value);
-        fp[i] = (int16_t) value;
      }
+      vector_normalize(fp, rframe->samples);
+
      switch_core_media_bug_set_write_replace_frame(bug, rframe);
      switch_mutex_unlock(cb->mutex);
    }