support playht3.0 (#119)

* support playht3.0

* update language

* wip

* update top_p and repetition_penalty

---------

Co-authored-by: root <root@af6633a5cfe8>

LICENSE_MIT

@@ -2,7 +2,7 @@ Under specific conditions that are described here: https://github.com/jambonz/fr
 
 The MIT License (MIT)
 
-Copyright (c) 2023 Drachtio Communications Services, LLC
+Copyright (c) 2024 FirstFive8, Inc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

mod_audio_fork/audio_pipe.cpp

@@ -65,13 +65,17 @@ int AudioPipe::lws_callback(struct lws *wsi,
           std::string username, password;
 
           ap->getBasicAuth(username, password);
-          lwsl_notice("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_APPEND_HANDSHAKE_HEADER username: %s, password: xxxxxx\n", username.c_str());
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_APPEND_HANDSHAKE_HEADER username: %s, password: xxxxxx\n", username.c_str());
           if (dch_lws_http_basic_auth_gen(username.c_str(), password.c_str(), b, sizeof(b))) break;
           if (lws_add_http_header_by_token(wsi, WSI_TOKEN_HTTP_AUTHORIZATION, (unsigned char *)b, strlen(b), p, end)) return -1;
         }
       }
       break;
 
+    case LWS_CALLBACK_WS_CLIENT_DROP_PROTOCOL:
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "AudioPipe::lws_service_thread LWS_CALLBACK_WS_CLIENT_DROP_PROTOCOL\n");
+      break;
+
     case LWS_CALLBACK_EVENT_WAIT_CANCELLED:
       processPendingConnects(vhd);
       processPendingDisconnects(vhd);
@@ -81,13 +85,13 @@ int AudioPipe::lws_callback(struct lws *wsi,
       {
         AudioPipe* ap = findAndRemovePendingConnect(wsi);
         int rc = lws_http_client_http_response(wsi);
-        lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_CONNECTION_ERROR: %s, response status %d\n", in ? (char *)in : "(null)", rc); 
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_CONNECTION_ERROR: %s, response status %d\n", in ? (char *)in : "(null)", rc); 
         if (ap) {
           ap->m_state = LWS_CLIENT_FAILED;
           ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECT_FAIL, (char *) in, NULL, len);
         }
         else {
-          lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_CONNECTION_ERROR unable to find wsi %p..\n", wsi); 
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_CONNECTION_ERROR unable to find wsi %p..\n", wsi); 
         }
       }      
       break;
@@ -102,7 +106,7 @@ int AudioPipe::lws_callback(struct lws *wsi,
           ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECT_SUCCESS, NULL, NULL, len);
         }
         else {
-          lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_ESTABLISHED %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_ESTABLISHED %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
         }
       }      
       break;
@@ -110,7 +114,7 @@ int AudioPipe::lws_callback(struct lws *wsi,
       {
         AudioPipe* ap = *ppAp;
         if (!ap) {
-          lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_CLOSED %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_CLOSED %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
           return 0;
         }
         if (ap->m_state == LWS_CLIENT_DISCONNECTING) {
@@ -119,7 +123,7 @@ int AudioPipe::lws_callback(struct lws *wsi,
         }
         else if (ap->m_state == LWS_CLIENT_CONNECTED) {
           // closed by far end
-          lwsl_notice("%s socket closed by far end\n", ap->m_uuid.c_str());
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,"%s socket closed by far end\n", ap->m_uuid.c_str());
           ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECTION_DROPPED, NULL, NULL, len);
         }
         ap->m_state = LWS_CLIENT_DISCONNECTED;
@@ -137,60 +141,68 @@ int AudioPipe::lws_callback(struct lws *wsi,
         
         AudioPipe* ap = *ppAp;
         if (!ap) {
-          lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
           return 0;
         }
         
-        if (lws_frame_is_binary(wsi)) {
-          if (ap->is_bidirectional_audio_stream()) {
-            ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::BINARY, NULL, (char *) in, len);
-          } else {
-            lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE received binary frame, discarding.\n");
-          }
+        if (ap->m_state == LWS_CLIENT_DISCONNECTING) {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,"AudioPipe::lws_service_thread race condition: got incoming message while closing the connection.\n");
           return 0;
         }
 
-        if (lws_is_first_fragment(wsi)) {
-          // allocate a buffer for the entire chunk of memory needed
-          assert(nullptr == ap->m_recv_buf);
-          ap->m_recv_buf_len = len + lws_remaining_packet_payload(wsi);
-          ap->m_recv_buf = (uint8_t*) malloc(ap->m_recv_buf_len);
-          ap->m_recv_buf_ptr = ap->m_recv_buf;
-        }
-
-        size_t write_offset = ap->m_recv_buf_ptr - ap->m_recv_buf;
-        size_t remaining_space = ap->m_recv_buf_len - write_offset;
-        if (remaining_space < len) {
-          lwsl_notice("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE buffer realloc needed.\n");
-          size_t newlen = ap->m_recv_buf_len + RECV_BUF_REALLOC_SIZE;
-          if (newlen > MAX_RECV_BUF_SIZE) {
-            free(ap->m_recv_buf);
-            ap->m_recv_buf = ap->m_recv_buf_ptr = nullptr;
-            ap->m_recv_buf_len = 0;
-            lwsl_notice("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE max buffer exceeded, truncating message.\n");
+        if (lws_frame_is_binary(wsi)) {
+          if (len > 0 && ap->is_bidirectional_audio_stream()) {
+            ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::BINARY, NULL, (char *) in, len);
+          } else if (len > 0) {
+            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE received unexpected binary frame, discarding.\n");
           }
           else {
-            ap->m_recv_buf = (uint8_t*) realloc(ap->m_recv_buf, newlen);
-            if (nullptr != ap->m_recv_buf) {
-              ap->m_recv_buf_len = newlen;
-              ap->m_recv_buf_ptr = ap->m_recv_buf + write_offset;
-            }
+            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE received zero length binary frame, discarding.\n");
           }
         }
-
-        if (nullptr != ap->m_recv_buf) {
-          if (len > 0) {
-            memcpy(ap->m_recv_buf_ptr, in, len);
-            ap->m_recv_buf_ptr += len;
+        else {
+          if (lws_is_first_fragment(wsi)) {
+            // allocate a buffer for the entire chunk of memory needed
+            assert(nullptr == ap->m_recv_buf);
+            ap->m_recv_buf_len = len + lws_remaining_packet_payload(wsi);
+            ap->m_recv_buf = (uint8_t*) malloc(ap->m_recv_buf_len);
+            ap->m_recv_buf_ptr = ap->m_recv_buf;
           }
-          if (lws_is_final_fragment(wsi)) {
-            if (nullptr != ap->m_recv_buf) {
-              std::string msg((char *)ap->m_recv_buf, ap->m_recv_buf_ptr - ap->m_recv_buf);
-              ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::MESSAGE, msg.c_str(), NULL, len);
-              if (nullptr != ap->m_recv_buf) free(ap->m_recv_buf);
+
+          size_t write_offset = ap->m_recv_buf_ptr - ap->m_recv_buf;
+          size_t remaining_space = ap->m_recv_buf_len - write_offset;
+          if (remaining_space < len) {
+            //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE buffer realloc needed.\n");
+            size_t newlen = ap->m_recv_buf_len + RECV_BUF_REALLOC_SIZE;
+            if (newlen > MAX_RECV_BUF_SIZE) {
+              free(ap->m_recv_buf);
+              ap->m_recv_buf = ap->m_recv_buf_ptr = nullptr;
+              ap->m_recv_buf_len = 0;
+              switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE max buffer exceeded, truncating message.\n");
+            }
+            else {
+              ap->m_recv_buf = (uint8_t*) realloc(ap->m_recv_buf, newlen);
+              if (nullptr != ap->m_recv_buf) {
+                ap->m_recv_buf_len = newlen;
+                ap->m_recv_buf_ptr = ap->m_recv_buf + write_offset;
+              }
+            }
+          }
+
+          if (nullptr != ap->m_recv_buf) {
+            if (len > 0) {
+              memcpy(ap->m_recv_buf_ptr, in, len);
+              ap->m_recv_buf_ptr += len;
+            }
+            if (lws_is_final_fragment(wsi)) {
+              if (nullptr != ap->m_recv_buf) {
+                std::string msg((char *)ap->m_recv_buf, ap->m_recv_buf_ptr - ap->m_recv_buf);
+                ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::MESSAGE, msg.c_str(), NULL, len);
+                if (nullptr != ap->m_recv_buf) free(ap->m_recv_buf);
+              }
+              ap->m_recv_buf = ap->m_recv_buf_ptr = nullptr;
+              ap->m_recv_buf_len = 0;
             }
-            ap->m_recv_buf = ap->m_recv_buf_ptr = nullptr;
-            ap->m_recv_buf_len = 0;
           }
         }
       }
@@ -200,13 +212,13 @@ int AudioPipe::lws_callback(struct lws *wsi,
       {
         AudioPipe* ap = *ppAp;
         if (!ap) {
-          lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_WRITEABLE %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_WRITEABLE %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
           return 0;
         }
 
         // check for graceful close - send a zero length binary frame
         if (ap->isGracefulShutdown()) {
-          lwsl_notice("%s graceful shutdown - sending zero length binary frame to flush any final responses\n", ap->m_uuid.c_str());
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"%s graceful shutdown - sending zero length binary frame to flush any final responses\n", ap->m_uuid.c_str());
           std::lock_guard<std::mutex> lk(ap->m_audio_mutex);
           int sent = lws_write(wsi, (unsigned char *) ap->m_audio_buffer + LWS_PRE, 0, LWS_WRITE_BINARY);
           return 0;
@@ -215,20 +227,21 @@ int AudioPipe::lws_callback(struct lws *wsi,
         // check for text frames to send
         {
           std::lock_guard<std::mutex> lk(ap->m_text_mutex);
-          if (ap->m_metadata.length() > 0) {
-            uint8_t buf[ap->m_metadata.length() + LWS_PRE];
-            memcpy(buf + LWS_PRE, ap->m_metadata.c_str(), ap->m_metadata.length());
-            int n = ap->m_metadata.length();
+          if (!ap->m_metadata_list.empty()) {
+            const std::string& message = ap->m_metadata_list.front();
+            uint8_t buf[message.length() + LWS_PRE];
+            memcpy(buf + LWS_PRE, message.c_str(), message.length());
+            int n = message.length();
             int m = lws_write(wsi, buf + LWS_PRE, n, LWS_WRITE_TEXT);
-            ap->m_metadata.clear();
+
             if (m < n) {
-              return -1;
+              return -1; // Failed to send the full message
             }
 
-            // there may be audio data, but only one write per writeable event
-            // get it next time
+            // Remove the message that was successfully sent
+            ap->m_metadata_list.pop_front();
+            // Request another writable event if there are more messages
             lws_callback_on_writable(wsi);
-
             return 0;
           }
         }
@@ -245,7 +258,7 @@ int AudioPipe::lws_callback(struct lws *wsi,
             size_t datalen = ap->m_audio_buffer_write_offset - LWS_PRE;
             int sent = lws_write(wsi, (unsigned char *) ap->m_audio_buffer + LWS_PRE, datalen, LWS_WRITE_BINARY);
             if (sent < datalen) {
-              lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_WRITEABLE %s attemped to send %lu only sent %d wsi %p..\n", 
+              switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_WRITEABLE %s attemped to send %lu only sent %d wsi %p..\n", 
                 ap->m_uuid.c_str(), datalen, sent, wsi); 
             }
             ap->m_audio_buffer_write_offset = LWS_PRE;
@@ -374,7 +387,7 @@ void AudioPipe::addPendingConnect(AudioPipe* ap) {
   {
     std::lock_guard<std::mutex> guard(mutex_connects);
     pendingConnects.push_back(ap);
-    lwsl_notice("%s after adding connect there are %lu pending connects\n", 
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,"%s after adding connect there are %lu pending connects\n", 
       ap->m_uuid.c_str(), pendingConnects.size());
   }
   lws_cancel_service(context);
@@ -384,7 +397,7 @@ void AudioPipe::addPendingDisconnect(AudioPipe* ap) {
   {
     std::lock_guard<std::mutex> guard(mutex_disconnects);
     pendingDisconnects.push_back(ap);
-    lwsl_notice("%s after adding disconnect there are %lu pending disconnects\n", 
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,"%s after adding disconnect there are %lu pending disconnects\n", 
       ap->m_uuid.c_str(), pendingDisconnects.size());
   }
   lws_cancel_service(ap->m_vhd->context);
@@ -423,11 +436,11 @@ bool AudioPipe::lws_service_thread() {
   info.timeout_secs_ah_idle = 10;       // secs to allow a client to hold an ah without using it
   info.retry_and_idle_policy = &retry;
 
-  lwsl_notice("AudioPipe::lws_service_thread creating context\n");
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,"AudioPipe::lws_service_thread creating context\n");
 
   context = lws_create_context(&info);
   if (!context) {
-    lwsl_err("AudioPipe::lws_service_thread failed creating context\n"); 
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread failed creating context\n"); 
     return false;
   }
 
@@ -444,16 +457,16 @@ bool AudioPipe::lws_service_thread() {
 
 void AudioPipe::initialize(const char* protocol, int loglevel, log_emit_function logger) {
   protocolName = protocol;
-  //lws_set_log_level(loglevel, logger);
+  lws_set_log_level(loglevel, logger);
 
-  lwsl_notice("AudioPipe::initialize starting\n"); 
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE,"AudioPipe::initialize starting\n"); 
   std::lock_guard<std::mutex> lock(mapMutex);
   stopFlag = false;
   serviceThread = std::thread(&AudioPipe::lws_service_thread);
 }
 
 bool AudioPipe::deinitialize() {
-  lwsl_notice("AudioPipe::deinitialize\n"); 
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE,"AudioPipe::deinitialize\n"); 
   std::lock_guard<std::mutex> lock(mapMutex);
   stopFlag = true;
   if (serviceThread.joinable()) {
@@ -508,7 +521,7 @@ bool AudioPipe::connect_client(struct lws_per_vhost_data *vhd) {
   m_vhd = vhd;
 
   m_wsi = lws_client_connect_via_info(&i);
-  lwsl_notice("%s attempting connection, wsi is %p\n", m_uuid.c_str(), m_wsi);
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,"%s attempting connection, wsi is %p\n", m_uuid.c_str(), m_wsi);
 
   return nullptr != m_wsi;
 }
@@ -517,7 +530,7 @@ void AudioPipe::bufferForSending(const char* text) {
   if (m_state != LWS_CLIENT_CONNECTED) return;
   {
     std::lock_guard<std::mutex> lk(m_text_mutex);
-    m_metadata.append(text);
+    m_metadata_list.emplace_back(text);
   }
   addPendingWrite(this);
 }

mod_audio_fork/audio_pipe.hpp

@@ -130,7 +130,7 @@ namespace drachtio {
     std::string m_bugname;
     unsigned int m_port;
     std::string m_path;
-    std::string m_metadata;
+    std::list<std::string> m_metadata_list;
     std::mutex m_text_mutex;
     std::mutex m_audio_mutex;
     int m_sslFlags;

mod_audio_fork/lws_glue.cpp

@@ -21,11 +21,14 @@
 
 #include <boost/circular_buffer.hpp>
 
+
 typedef boost::circular_buffer<uint16_t> CircularBuffer_t;
 
 #define RTP_PACKETIZATION_PERIOD 20
 #define FRAME_SIZE_8000  320 /*which means each 20ms frame as 320 bytes at 8 khz (1 channel only)*/
-#define BUFFER_GROW_SIZE (8192)
+#define BUFFER_GROW_SIZE (16384)
+#define AUDIO_MARKER 0xFFFF
+#define MAX_MARKS (30)
 
 namespace {
   static const char *requestedBufferSecs = std::getenv("MOD_AUDIO_FORK_BUFFER_SECS");
@@ -37,54 +40,174 @@ namespace {
   static unsigned int idxCallCount = 0;
   static uint32_t playCount = 0;
 
+  static bool markCountExceeded(private_t* tech_pvt) {
+    if (nullptr != tech_pvt->pVecMarksInUse) {
+      std::deque<std::string>* pVecMarksInUse = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInUse);
+      std::deque<std::string>* pVecMarksInInventory = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInInventory);
+      return pVecMarksInUse->size()+ pVecMarksInInventory->size() >= MAX_MARKS;
+    }
+    return false;
+  }
+
   switch_status_t processIncomingBinary(private_t* tech_pvt, switch_core_session_t* session, const char* message, size_t dataLength) {
-    CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->circularBuffer;
-    uint8_t* data = reinterpret_cast<uint8_t*>(const_cast<char*>(message));
-    uint16_t* data_uint16 = reinterpret_cast<uint16_t*>(data);
-    std::vector<uint16_t> pcm_data(data_uint16, data_uint16 + dataLength / sizeof(uint16_t));
+    std::vector<uint8_t> data;
 
+    // Prepend the set-aside byte if there is one
+    if (tech_pvt->has_set_aside_byte) {
+        data.push_back(tech_pvt->set_aside_byte);
+        tech_pvt->has_set_aside_byte = false;
+    }
 
-    if (tech_pvt->bidirectional_audio_resampler) {
-      std::vector<int16_t> in(pcm_data.begin(), pcm_data.end());
+    // Append the new incoming message
+    data.insert(data.end(), message, message + dataLength);
 
-      std::vector<int16_t> out(dataLength);
-      spx_uint32_t in_len = pcm_data.size();
-      spx_uint32_t out_len = out.size();
-      speex_resampler_process_interleaved_int(tech_pvt->bidirectional_audio_resampler, in.data(), &in_len, out.data(), &out_len);
+    // Check if the total data length is now odd
+    if (data.size() % 2 != 0) {
+        // Set aside the last byte
+        tech_pvt->set_aside_byte = data.back();
+        tech_pvt->has_set_aside_byte = true;
+        data.pop_back(); // Remove the last byte from the data vector
+    }
 
-      if (out_len > out.size()) {
-        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Resampler output exceeded maximum buffer size!\n");
+    // Convert the data to 16-bit elements
+    const uint16_t* data_uint16 = reinterpret_cast<const uint16_t*>(data.data());
+    size_t numSamples = data.size() / sizeof(uint16_t);
+
+    // Access the prebuffer
+    CircularBuffer_t* cBuffer = static_cast<CircularBuffer_t*>(tech_pvt->streamingPreBuffer);
+
+    int numMarkers = 0;
+    std::deque<std::string>* pVecMarksInInventory = nullptr;
+    std::deque<std::string>* pVecMarksInUse = nullptr;
+    if (nullptr != tech_pvt->pVecMarksInInventory) {
+      pVecMarksInInventory = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInInventory);
+      pVecMarksInUse = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInUse);
+      numMarkers = pVecMarksInInventory->size();
+
+      // move inventory to in-use
+      pVecMarksInUse->insert(pVecMarksInUse->end(), pVecMarksInInventory->begin(), pVecMarksInInventory->end());
+      pVecMarksInInventory->clear();
+    }
+  
+    // Ensure the prebuffer has enough capacity
+    if (cBuffer->capacity() - cBuffer->size() < numSamples + numMarkers) {
+        size_t newCapacity = cBuffer->size() + std::max(numSamples + numMarkers, (size_t)BUFFER_GROW_SIZE);
+        cBuffer->set_capacity(newCapacity);
+    }
+
+    // prepend any markers
+    while (numMarkers-- > 0) {
+      cBuffer->push_back(AUDIO_MARKER);
+    }
+
+    // Append the data to the prebuffer
+    cBuffer->insert(cBuffer->end(), data_uint16, data_uint16 + numSamples);
+    //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Appended %zu 16-bit samples to the prebuffer.\n", numSamples);
+
+    // if we haven't reached threshold amount of prebuffered data, return
+    if (cBuffer->size() < tech_pvt->streamingPreBufSize) {
+        //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Prebuffered data is below threshold %u, returning.\n", tech_pvt->streamingPreBufSize);
+        return SWITCH_STATUS_SUCCESS;
+    }
+
+    //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Prebuffered data samples %u is above threshold %u, prepare to playout.\n", cBuffer->size(), tech_pvt->streamingPreBufSize);
+
+    // after initial pre-buffering, rachet down the threshold to 40ms
+    tech_pvt->streamingPreBufSize = 320 * tech_pvt->downscale_factor * 2;
+
+    // Check for downsampling factor
+    size_t downsample_factor = tech_pvt->downscale_factor;
+
+    // Calculate the number of samples that can be evenly divided by the downsample factor
+    size_t numCompleteSamples = (cBuffer->size() / downsample_factor) * downsample_factor;
+
+    // Handle leftover samples
+    std::vector<uint16_t> leftoverSamples;
+    size_t numLeftoverSamples = cBuffer->size() - numCompleteSamples;
+    if (numLeftoverSamples > 0) {
+        leftoverSamples.assign(cBuffer->end() - numLeftoverSamples, cBuffer->end());
+        cBuffer->resize(numCompleteSamples);
+        //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Temporarily removing %u leftover samples due to downsampling.\n", numLeftoverSamples);
+    }
+
+    // resample if necessary
+    std::vector<int16_t> out;
+    try {
+      if (tech_pvt->bidirectional_audio_resampler) {
+        // Improvement: Use assign to convert circular buffer to vector for resampling
+        std::vector<int16_t> in;
+        in.assign(cBuffer->begin(), cBuffer->end()); 
+        out.resize(in.size() * 6); // max upsampling would be from 8k -> 48k
+
+        spx_uint32_t in_len = in.size();
+        spx_uint32_t out_len = out.size();
+
+        //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Resampling %u samples into a buffer that can hold %u samples\n", in.size(), out_len);
+
+        speex_resampler_process_interleaved_int(tech_pvt->bidirectional_audio_resampler, in.data(), &in_len, out.data(), &out_len);
+
+        // Resize the output buffer to match the output length from resampler
+        //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Resizing output buffer from %u to %u samples\n", in.size(), out_len);
+
+        out.resize(out_len);
+      }
+      else {
+        // If no resampling is needed, copy the data from the prebuffer to the output buffer
+        out.assign(cBuffer->begin(), cBuffer->end());
+      }
+    } catch (const std::exception& e) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error resampling incoming binary message: %s\n", e.what());
+      return SWITCH_STATUS_FALSE;
+    } catch (...) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error resampling incoming binary message\n");
+      return SWITCH_STATUS_FALSE;
+    }
+
+    if (nullptr != tech_pvt->mutex && switch_mutex_trylock(tech_pvt->mutex) == SWITCH_STATUS_SUCCESS) {
+      CircularBuffer_t *playoutBuffer = (CircularBuffer_t *) tech_pvt->streamingPlayoutBuffer;
+
+      try {
+        // Resize the buffer if necessary
+        if (playoutBuffer->capacity() - playoutBuffer->size() < out.size()) {
+          size_t newCapacity = playoutBuffer->size() + std::max(out.size(), (size_t)BUFFER_GROW_SIZE);
+          playoutBuffer->set_capacity(newCapacity);
+          //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Resized playout buffer to new capacity: %zu\n", newCapacity);
+        }
+        // Push the data into the buffer.
+        playoutBuffer->insert(playoutBuffer->end(), out.begin(), out.end());
+        //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Appended %zu 16-bit samples to the playout buffer.\n", out.size());
+      } catch (const std::exception& e) {
+        switch_mutex_unlock(tech_pvt->mutex);
+        cBuffer->clear();
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error processing incoming binary message: %s\n", e.what());
+        return SWITCH_STATUS_FALSE;
+      } catch (...) {
+        switch_mutex_unlock(tech_pvt->mutex);
+        cBuffer->clear();
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error processing incoming binary message\n");
         return SWITCH_STATUS_FALSE;
       }
+      switch_mutex_unlock(tech_pvt->mutex);
+      cBuffer->clear();
 
-      // Resize the pcm_data to match the output length from resampler, and then copy the resampled data into it.
-      pcm_data.resize(out_len);
-      memcpy(pcm_data.data(), out.data(), out_len * sizeof(int16_t));
+      // Put the leftover samples back in the prebuffer for the next time
+      if (!leftoverSamples.empty()) {
+          cBuffer->insert(cBuffer->end(), leftoverSamples.begin(), leftoverSamples.end());
+          //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Put back %u leftover samples into the prebuffer.\n", leftoverSamples.size());
+      }
+      return SWITCH_STATUS_SUCCESS;
     }
-    switch_mutex_lock(tech_pvt->mutex);
-
-    // Resize the buffer if necessary
-    size_t bytesResampled = pcm_data.size() * sizeof(uint16_t);
-    if (cBuffer->capacity() - cBuffer->size() < bytesResampled / sizeof(uint16_t)) {
-      // If buffer exceeds some max size, you could return SWITCH_STATUS_FALSE to abort the transfer
-      // if (cBuffer->size() + std::max(bytesResampled / sizeof(uint16_t), (size_t)BUFFER_GROW_SIZE) > MAX_BUFFER_SIZE) return SWITCH_STATUS_FALSE;
-
-      cBuffer->set_capacity(cBuffer->size() + std::max(bytesResampled / sizeof(uint16_t), (size_t)BUFFER_GROW_SIZE));
-    }
-    // Push the data into the buffer.
-    cBuffer->insert(cBuffer->end(), pcm_data.begin(), pcm_data.end());
-
-    switch_mutex_unlock(tech_pvt->mutex);
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Failed to get mutext (temp)\n");
 
     return SWITCH_STATUS_SUCCESS;
-}
+  }
 
   void processIncomingMessage(private_t* tech_pvt, switch_core_session_t* session, const char* message) {
     std::string msg = message;
     std::string type;
     cJSON* json = parse_json(session, msg, type) ;
     if (json) {
-      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "(%u) processIncomingMessage - received %s message\n", tech_pvt->id, type.c_str());
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "(%u) processIncomingMessage - received %s message %s\n", tech_pvt->id, type.c_str(), message);
       cJSON* jsonData = cJSON_GetObjectItem(json, "data");
       if (0 == type.compare("playAudio") &&
         // playAudio is enabled and there is no bidirectional audio from stream is enabled.
@@ -171,6 +294,66 @@ namespace {
         // kill any current playback on the channel
         switch_channel_t *channel = switch_core_session_get_channel(session);
         switch_channel_set_flag_value(channel, CF_BREAK, 2);
+
+        // this will dump buffered incoming audio
+        tech_pvt->clear_bidirectional_audio_buffer = true;
+      }
+      else if (0 == type.compare("mark")) {
+        cJSON* data = cJSON_GetObjectItem(json, "data");
+        if (data) {
+          cJSON* name = cJSON_GetObjectItem(data, "name");
+          if (cJSON_IsString(name) && name->valuestring) {
+            if (markCountExceeded(tech_pvt)) {
+              switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "(%u) processIncomingMessage - mark count exceeded, discarding mark %s\n", tech_pvt->id, cJSON_GetStringValue(name));
+            }
+            else {
+              if (nullptr == tech_pvt->pVecMarksInInventory) {
+                tech_pvt->pVecMarksInInventory = static_cast<void *>(new std::deque<std::string>());
+                tech_pvt->pVecMarksInUse = static_cast<void *>(new std::deque<std::string>());
+                tech_pvt->pVecMarksCleared = static_cast<void *>(new std::deque<std::string>());
+              }
+              std::deque<std::string>* pVec = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInInventory);
+              pVec->push_back(name->valuestring);
+            }
+          }
+        }
+      }
+      else if (0 == type.compare("clearMarks")) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "(%u) processIncomingMessage - received clearMarks\n", tech_pvt->id);
+        if (nullptr != tech_pvt->pVecMarksInInventory) {
+          std::deque<std::string>* pVecMarksInInventory = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInInventory);
+          std::deque<std::string>* pVecMarksInUse = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInUse);
+          std::deque<std::string>* pVecMarksCleared = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksCleared);
+          pVecMarksCleared->insert(pVecMarksCleared->end(), pVecMarksInUse->begin(), pVecMarksInUse->end());
+          pVecMarksCleared->insert(pVecMarksCleared->end(), pVecMarksInInventory->begin(), pVecMarksInInventory->end());
+          pVecMarksInInventory->clear();
+          pVecMarksInUse->clear();
+        }
+      }
+      else if (0 == type.compare("transcription")) {
+        char* jsonString = cJSON_PrintUnformatted(jsonData);
+        tech_pvt->responseHandler(session, EVENT_TRANSCRIPTION, jsonString);
+        free(jsonString);        
+      }
+      else if (0 == type.compare("transfer")) {
+        char* jsonString = cJSON_PrintUnformatted(jsonData);
+        tech_pvt->responseHandler(session, EVENT_TRANSFER, jsonString);
+        free(jsonString);                
+      }
+      else if (0 == type.compare("disconnect")) {
+        char* jsonString = cJSON_PrintUnformatted(jsonData);
+        tech_pvt->responseHandler(session, EVENT_DISCONNECT, jsonString);
+        free(jsonString);        
+      }
+      else if (0 == type.compare("error")) {
+        char* jsonString = cJSON_PrintUnformatted(jsonData);
+        tech_pvt->responseHandler(session, EVENT_ERROR, jsonString);
+        free(jsonString);        
+      }
+      else if (0 == type.compare("json")) {
+        char* jsonString = cJSON_PrintUnformatted(json);
+        tech_pvt->responseHandler(session, EVENT_JSON, jsonString);
+        free(jsonString);
       }
       else if (0 == type.compare("transcription")) {
         char* jsonString = cJSON_PrintUnformatted(jsonData);
@@ -290,10 +473,23 @@ namespace {
     tech_pvt->buffer_overrun_notified = 0;
     tech_pvt->audio_paused = 0;
     tech_pvt->graceful_shutdown = 0;
-    tech_pvt->circularBuffer = (void *) new CircularBuffer_t(8192);
+    tech_pvt->streamingPlayoutBuffer = (void *) new CircularBuffer_t(8192);
     tech_pvt->bidirectional_audio_enable = bidirectional_audio_enable;
     tech_pvt->bidirectional_audio_stream = bidirectional_audio_stream;
     tech_pvt->bidirectional_audio_sample_rate = bidirectional_audio_sample_rate;
+    tech_pvt->clear_bidirectional_audio_buffer = false;
+    tech_pvt->has_set_aside_byte = 0;
+    tech_pvt->downscale_factor = 1;
+    if (bidirectional_audio_sample_rate > sampling) {
+      tech_pvt->downscale_factor = bidirectional_audio_sample_rate / sampling;
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "downscale_factor is %d\n", tech_pvt->downscale_factor);
+    }
+    tech_pvt->streamingPreBufSize = 320 * tech_pvt->downscale_factor * 4; // min 80ms prebuffer
+    tech_pvt->streamingPreBuffer = (void *) new CircularBuffer_t(8192);
+    tech_pvt->pVecMarksInInventory = nullptr;
+    tech_pvt->pVecMarksInUse = nullptr;
+    tech_pvt->pVecMarksCleared = nullptr;
+
     strncpy(tech_pvt->bugname, bugname, MAX_BUG_LEN);
     if (metadata) strncpy(tech_pvt->initialMetadata, metadata, MAX_METADATA_LEN);
     
@@ -323,8 +519,8 @@ namespace {
     }
 
     if (bidirectional_audio_sample_rate && sampling != bidirectional_audio_sample_rate) {
-      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) bidirectional audio resampling from %u to %u\n", tech_pvt->id, bidirectional_audio_sample_rate, sampling);
-      tech_pvt->bidirectional_audio_resampler = speex_resampler_init(channels, bidirectional_audio_sample_rate, sampling, SWITCH_RESAMPLE_QUALITY, &err);
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) bidirectional audio resampling from %u to %u, channels %d\n", tech_pvt->id, bidirectional_audio_sample_rate, sampling, channels);
+      tech_pvt->bidirectional_audio_resampler = speex_resampler_init(1, bidirectional_audio_sample_rate, sampling, SWITCH_RESAMPLE_QUALITY, &err);
       if (0 != err) {
         switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error initializing bidirectional audio resampler: %s.\n", speex_resampler_strerror(err));
         return SWITCH_STATUS_FALSE;
@@ -350,10 +546,38 @@ namespace {
       switch_mutex_destroy(tech_pvt->mutex);
       tech_pvt->mutex = nullptr;
     }
-    if (tech_pvt->circularBuffer) {
-      CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->circularBuffer;
+    if (tech_pvt->streamingPlayoutBuffer) {
+      CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->streamingPlayoutBuffer;
       delete cBuffer;
-      tech_pvt->circularBuffer = nullptr;
+      tech_pvt->streamingPlayoutBuffer = nullptr;
+    }
+    if (tech_pvt->streamingPreBuffer) {
+      CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->streamingPreBuffer;
+      delete cBuffer;
+      tech_pvt->streamingPreBuffer = nullptr;
+    }
+
+    if (tech_pvt->pVecMarksInInventory) {
+      delete static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInInventory);
+      tech_pvt->pVecMarksInInventory = nullptr;
+      delete static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInUse);
+      tech_pvt->pVecMarksInUse = nullptr;
+      delete static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksCleared);
+      tech_pvt->pVecMarksCleared = nullptr;
+    }
+  }
+
+  static void send_mark_event(private_t* tech_pvt, const char* name, int cleared = false) {
+    drachtio::AudioPipe *pAudioPipe = static_cast<drachtio::AudioPipe *>(tech_pvt->pAudioPipe);
+    std::ostringstream json;
+    json << "{\"type\": \"mark\", \"data\": {\"name\":\"" << name << "\", ";
+    if (cleared) json << "\"event\": \"cleared\"}}";
+    else json << "\"event\": \"playout\"}}";
+
+    if (pAudioPipe) {
+      std::string str = json.str();
+      pAudioPipe->bufferForSending(str.c_str());
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "(%u) send_mark_event: %s\n", tech_pvt->id, str.c_str());
     }
   }
 
@@ -453,8 +677,8 @@ extern "C" {
     switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_audio_fork: audio buffer (in secs):    %d secs\n", nAudioBufferSecs);
     switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_audio_fork: sub-protocol:              %s\n", mySubProtocolName);
  
-    int logs = LLL_ERR | LLL_WARN | LLL_NOTICE ;
-     //LLL_INFO | LLL_PARSER | LLL_HEADER | LLL_EXT | LLL_CLIENT  | LLL_LATENCY | LLL_DEBUG ;
+    //int logs = LLL_ERR | LLL_WARN | LLL_NOTICE | LLL_INFO | LLL_PARSER | LLL_HEADER | LLL_EXT | LLL_CLIENT  | LLL_LATENCY | LLL_DEBUG ;
+    int logs = LLL_ERR | LLL_WARN | LLL_NOTICE;
     drachtio::AudioPipe::initialize(mySubProtocolName, logs, lws_logger);
     switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_audio_fork successfully initialized\n");
     return SWITCH_STATUS_SUCCESS;
@@ -706,28 +930,122 @@ extern "C" {
   }
 
   switch_bool_t dub_speech_frame(switch_media_bug_t *bug, private_t* tech_pvt) {
-    CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->circularBuffer;
+    CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->streamingPlayoutBuffer;
     if (switch_mutex_trylock(tech_pvt->mutex) == SWITCH_STATUS_SUCCESS) {
-      switch_frame_t* rframe = switch_core_media_bug_get_write_replace_frame(bug);
-      int16_t *fp = reinterpret_cast<int16_t*>(rframe->data);
 
-      rframe->channels = 1;
-      rframe->datalen = rframe->samples * sizeof(int16_t);
+      // if flag was set to clear the buffer, do so and clear the flag
+      if (tech_pvt->clear_bidirectional_audio_buffer) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "(%u) dub_speech_frame - clearing buffer\n", tech_pvt->id); 
+        cBuffer->clear();
+        tech_pvt->clear_bidirectional_audio_buffer = false;
 
-      int16_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
-      memset(data, 0, sizeof(data));
+        // send "mark" event for any queued markers
+        if (nullptr != tech_pvt->pVecMarksInInventory) {
+          std::deque<std::string>* pVecMarksInInventory = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInInventory);
+          std::deque<std::string>* pVecMarksInUse = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInUse);
+          if (pVecMarksInInventory->size() + pVecMarksInUse->size() > 0) {
+            std::deque<std::string> vec = *pVecMarksInUse;
+            vec.insert(vec.end(), pVecMarksInInventory->begin(), pVecMarksInInventory->end());
+            for (auto it = vec.begin(); it != vec.end(); ++it) {
+              switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "(%u) dub_speech_frame - Marker %s cleared\n",
+                  tech_pvt->id, it->c_str());
+              send_mark_event(tech_pvt, it->c_str(), true);
+            }
 
-      int samplesToCopy = std::min(static_cast<int>(cBuffer->size()), static_cast<int>(rframe->samples));
+            // put the "in-use" ones into the "cleared" queue so we dont notify again when they eventually come through
+            std::deque<std::string>* pVecMarksCleared = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksCleared);
+            pVecMarksCleared->insert(pVecMarksCleared->end(), pVecMarksInUse->begin(), pVecMarksInUse->end());
 
-      std::copy_n(cBuffer->begin(), samplesToCopy, data);
-      cBuffer->erase(cBuffer->begin(), cBuffer->begin() + samplesToCopy);
-
-      if (samplesToCopy > 0) {
-        vector_add(fp, data, rframe->samples);
+            pVecMarksInUse->clear();
+            pVecMarksInInventory->clear();
+          }
+        }
       }
-      vector_normalize(fp, rframe->samples);
+      else {
+        switch_frame_t* rframe = switch_core_media_bug_get_write_replace_frame(bug);
+        int16_t *fp = reinterpret_cast<int16_t*>(rframe->data);
 
-      switch_core_media_bug_set_write_replace_frame(bug, rframe);
+        rframe->channels = 1;
+        rframe->datalen = rframe->samples * sizeof(int16_t);
+
+        int16_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
+        memset(data, 0, sizeof(data));
+
+        int samplesToCopy = std::min(static_cast<int>(cBuffer->size()), static_cast<int>(rframe->samples));
+
+        //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "(%u) dub_speech_frame - samples to copy %u\n", tech_pvt->id, samplesToCopy); 
+
+        bool hasMarkers = false;
+        std::deque<std::string>* pVecInventory = nullptr;
+        std::deque<std::string>* pVecInUse = nullptr;
+        std::deque<std::string>* pVecCleared = nullptr;
+        if (tech_pvt->pVecMarksInInventory && tech_pvt->pVecMarksInUse && tech_pvt->pVecMarksCleared) {
+          pVecInventory = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInInventory);
+          pVecInUse = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksInUse);
+          pVecCleared = static_cast<std::deque<std::string>*>(tech_pvt->pVecMarksCleared);
+          hasMarkers = pVecInUse->size() + pVecCleared->size() >  0;
+        }
+
+        if (hasMarkers) {
+          /* discard markers and send notifications */
+          auto bufferIter = cBuffer->begin();
+          auto dataIter = data;
+          for (int i = 0; i < samplesToCopy; ++i) {
+            if (*bufferIter == AUDIO_MARKER) {
+              // Marker detected, discard it and send a notice unless it was previously cleared
+              auto * pVec = pVecCleared->size() > 0 ? pVecCleared : pVecInUse;
+              if (!pVec->empty()) {
+                auto name = pVec->front();
+                pVec->pop_front();
+
+                if (pVec == pVecInUse) {
+                  send_mark_event(tech_pvt, name.c_str());
+                  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "(%u) dub_speech_frame - Marker %s detected in playout\n",
+                    tech_pvt->id, name.c_str());
+                }
+                else {
+                  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "(%u) dub_speech_frame - Marker %s detected in playout but previously cleared\n",
+                    tech_pvt->id, name.c_str());
+                }
+              }
+            } else {
+              // Copy valid audio samplewhat 
+              *dataIter = *bufferIter;
+              ++dataIter;
+            }
+            ++bufferIter;
+          }
+
+          // Remove the processed samples (including discarded markers) from the buffer
+          cBuffer->erase(cBuffer->begin(), cBuffer->begin() + samplesToCopy);
+
+          // Adjust the number of samples copied to the output frame
+          int validSamplesCopied = std::distance(data, dataIter);
+
+          if (validSamplesCopied > 0) {
+            vector_add(fp, data, validSamplesCopied);
+          }
+        }
+        else {
+          std::copy_n(cBuffer->begin(), samplesToCopy, data);
+          cBuffer->erase(cBuffer->begin(), cBuffer->begin() + samplesToCopy);
+
+          if (samplesToCopy > 0) {
+            vector_add(fp, data, rframe->samples);
+          } else if (pVecInventory && pVecInventory->size()) {
+            // no bidirectional audio to dub but still have some mark in inventory, send them now
+            auto name = pVecInventory->front();
+            pVecInventory->pop_front();
+
+            send_mark_event(tech_pvt, name.c_str());
+            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "(%u) dub_speech_frame - Marker %s detected in inventory\n",
+              tech_pvt->id, name.c_str());
+          }
+        }
+        vector_normalize(fp, rframe->samples);
+
+        switch_core_media_bug_set_write_replace_frame(bug, rframe);
+      }
       switch_mutex_unlock(tech_pvt->mutex);
     }
     return SWITCH_TRUE;
@@ -742,7 +1060,7 @@ extern "C" {
     }
     private_t* tech_pvt = (private_t*) switch_core_media_bug_get_user_data(bug);
 
-    CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->circularBuffer;
+    CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->streamingPlayoutBuffer;
 
     if (switch_mutex_trylock(tech_pvt->mutex) == SWITCH_STATUS_SUCCESS) {
       if (cBuffer != nullptr) {

mod_audio_fork/mod_audio_fork.h

@@ -52,11 +52,24 @@ struct private_data {
   int audio_paused:1;
   int graceful_shutdown:1;
   char initialMetadata[8192];
-  void *circularBuffer;
+
+  // for "mark" feature of bidirectional audio
+  void *pVecMarksInInventory;
+  void *pVecMarksInUse;
+  void *pVecMarksCleared;
+
+  // bidirectional audio
+  void *streamingPlayoutBuffer;
+  void *streamingPreBuffer;
+  int streamingPreBufSize;
+  uint8_t set_aside_byte;
+  int has_set_aside_byte;
+  int downscale_factor;
   SpeexResamplerState *bidirectional_audio_resampler;
   int bidirectional_audio_enable;
 	int bidirectional_audio_stream;
   int bidirectional_audio_sample_rate;
+  int clear_bidirectional_audio_buffer;
 };
 
 typedef struct private_data private_t;

mod_aws_lex/aws_lex_glue.cpp

@@ -173,13 +173,12 @@ public:
 		char *metadata,
 		const char* awsAccessKeyId, 
 		const char* awsSecretAccessKey,
+		const char* awsSessionToken,
 		responseHandler_t responseHandler,
 		errorHandler_t  errorHandler) : 
 	m_bot(bot), m_alias(alias), m_region(region), m_sessionId(sessionId), m_finished(false), m_finishing(false), m_packets(0),
 	m_pStream(nullptr), m_bPlayDone(false), m_bDiscardAudio(false)
 	{
-		Aws::String key(awsAccessKeyId);
-		Aws::String secret(awsSecretAccessKey);
 		Aws::String awsLocale(locale);
 		Aws::Client::ClientConfiguration config;
 		config.region = region;
@@ -190,8 +189,11 @@ public:
 		for (int i = 4; i < 20; i++) keySnippet[i] = 'x';
 		keySnippet[19] = '\0';
 
-		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p ACCESS_KEY_ID %s\n", this, keySnippet);		
-		if (*awsAccessKeyId && *awsSecretAccessKey) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p ACCESS_KEY_ID %s\n", this, keySnippet);
+		if (*awsAccessKeyId && *awsSecretAccessKey && *awsSessionToken) {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "using AWS creds %s %s %s\n", awsAccessKeyId, awsSecretAccessKey, awsSessionToken);	
+			m_client = Aws::MakeUnique<LexRuntimeV2Client>(ALLOC_TAG, AWSCredentials(awsAccessKeyId, awsSecretAccessKey, awsSessionToken), config);
+		} else if (*awsAccessKeyId && *awsSecretAccessKey) {
 			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "using AWS creds %s %s\n", awsAccessKeyId, awsSecretAccessKey);	
 			m_client = Aws::MakeUnique<LexRuntimeV2Client>(ALLOC_TAG, AWSCredentials(awsAccessKeyId, awsSecretAccessKey), config);
 		}
@@ -540,7 +542,7 @@ static void *SWITCH_THREAD_FUNC lex_thread(switch_thread_t *thread, void *obj) {
 	bool ok = true;
 	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "lex_thread: starting cb %p\n", (void *) cb);
 	GStreamer* pStreamer = new GStreamer(cb->sessionId, cb->bot, cb->alias, cb->region, cb->locale, 
-		cb->intent, cb->metadata, cb->awsAccessKeyId, cb->awsSecretAccessKey, 
+		cb->intent, cb->metadata, cb->awsAccessKeyId, cb->awsSecretAccessKey, cb->awsSessionToken,
 		cb->responseHandler, cb->errorHandler);
 	if (!pStreamer) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "lex_thread: Error allocating streamer\n");
@@ -641,6 +643,7 @@ extern "C" {
 		memset(cb, sizeof(cb), 0);
 		const char* awsAccessKeyId = switch_channel_get_variable(channel, "AWS_ACCESS_KEY_ID");
 		const char* awsSecretAccessKey = switch_channel_get_variable(channel, "AWS_SECRET_ACCESS_KEY");
+		const char* awsSessionToken = switch_channel_get_variable(channel, "AWS_SESSION_TOKEN");
 
 		if (!hasDefaultCredentials && (!awsAccessKeyId || !awsSecretAccessKey)) {
 			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, 
@@ -654,6 +657,7 @@ extern "C" {
 		if (awsAccessKeyId && awsSecretAccessKey) {
 			strncpy(cb->awsAccessKeyId, awsAccessKeyId, 128);
 			strncpy(cb->awsSecretAccessKey, awsSecretAccessKey, 128);
+			if (awsSessionToken) strncpy(cb->awsSessionToken, awsSessionToken, 1024);
 		}
 		else {
 			strncpy(cb->awsAccessKeyId, std::getenv("AWS_ACCESS_KEY_ID"), 128);

mod_aws_lex/mod_aws_lex.h

@@ -30,6 +30,7 @@ struct cap_cb {
 	char sessionId[256];
   char awsAccessKeyId[128];
   char awsSecretAccessKey[128];
+	char awsSessionToken[1024];
   SpeexResamplerState *resampler;
 	void* streamer;
 	responseHandler_t responseHandler;

mod_aws_transcribe/aws_transcribe_glue.cpp

@@ -43,18 +43,17 @@ public:
     const char *sessionId,
 		const char *bugname,
 		u_int16_t channels,
-    char *lang, 
+    char *lang,
     int interim,
 		uint32_t samples_per_second,
-		const char* region, 
-		const char* awsAccessKeyId, 
+		const char* region,
+		const char* awsAccessKeyId,
 		const char* awsSecretAccessKey,
+		const char* awsSessionToken,
 		responseHandler_t responseHandler
   ) : m_sessionId(sessionId), m_bugname(bugname), m_finished(false), m_interim(interim), m_finishing(false), m_connected(false), m_connecting(false),
-	 		m_packets(0), m_responseHandler(responseHandler), m_pStream(nullptr), 
+	 		m_packets(0), m_responseHandler(responseHandler), m_pStream(nullptr),
 			m_audioBuffer(320 * (samples_per_second == 8000 ? 1 : 2), 15) {
-		Aws::String key(awsAccessKeyId);
-		Aws::String secret(awsSecretAccessKey);
 		Aws::Client::ClientConfiguration config;
 		if (region != nullptr && strlen(region) > 0) config.region = region;
 		char keySnippet[20];
@@ -63,14 +62,16 @@ public:
 		for (int i = 4; i < 20; i++) keySnippet[i] = 'x';
 		keySnippet[19] = '\0';
 
-		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p ACCESS_KEY_ID %s, region %s\n", this, keySnippet, region);		
-		if (*awsAccessKeyId && *awsSecretAccessKey) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p ACCESS_KEY_ID %s, region %s\n", this, keySnippet, region);
+		if (*awsAccessKeyId && *awsSecretAccessKey && *awsSessionToken) {
+			m_client = Aws::MakeUnique<TranscribeStreamingServiceClient>(ALLOC_TAG, AWSCredentials(awsAccessKeyId, awsSecretAccessKey, awsSessionToken), config);
+		} else if (*awsAccessKeyId && *awsSecretAccessKey) {
 			m_client = Aws::MakeUnique<TranscribeStreamingServiceClient>(ALLOC_TAG, AWSCredentials(awsAccessKeyId, awsSecretAccessKey), config);
 		}
 		else {
 			m_client = Aws::MakeUnique<TranscribeStreamingServiceClient>(ALLOC_TAG, config);
 		}
-	
+
     m_handler.SetTranscriptEventCallback([this](const TranscriptEvent& ev)
     {
 			switch_core_session_t* psession = switch_core_session_locate(m_sessionId.c_str());
@@ -131,7 +132,7 @@ public:
 
 				// send any buffered audio
 				int nFrames = m_audioBuffer.getNumItems();
-				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p got stream ready, %d buffered frames\n", this, nFrames);	
+				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p got stream ready, %d buffered frames\n", this, nFrames);
 				if (nFrames) {
 					char *p;
 					do {
@@ -141,19 +142,19 @@ public:
 						}
 					} while (p);
 				}
-	
+
 				switch_core_session_rwunlock(psession);
 			}
     };
-    auto OnResponseCallback = [this](const TranscribeStreamingServiceClient* pClient, 
-			const Model::StartStreamTranscriptionRequest& request, 
-			const Model::StartStreamTranscriptionOutcome& outcome, 
+    auto OnResponseCallback = [this](const TranscribeStreamingServiceClient* pClient,
+			const Model::StartStreamTranscriptionRequest& request,
+			const Model::StartStreamTranscriptionOutcome& outcome,
 			const std::shared_ptr<const Aws::Client::AsyncCallerContext>& context)
     {
  			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p stream got final response\n", this);
 			switch_core_session_t* psession = switch_core_session_locate(m_sessionId.c_str());
 			if (psession) {
-				if (!outcome.IsSuccess()) {				
+				if (!outcome.IsSuccess()) {
 					const TranscribeStreamingServiceError& err = outcome.GetError();
 					auto message = err.GetMessage();
 					auto exception = err.GetExceptionName();
@@ -185,7 +186,7 @@ public:
 
 
 	~GStreamer() {
-		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer::~GStreamer wrote %u packets %p\n", m_packets, this);		
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer::~GStreamer wrote %u packets %p\n", m_packets, this);
 	}
 
 	bool write(void* data, uint32_t datalen) {
@@ -227,7 +228,7 @@ public:
 		bool shutdownInitiated = false;
 		while (true) {
 			std::unique_lock<std::mutex> lk(m_mutex);
-			m_cond.wait(lk, [&, this] { 
+			m_cond.wait(lk, [&, this] {
 				return (!m_deqAudio.empty() && !m_finishing)  || m_transcript.TranscriptHasBeenSet() || m_finished  || (m_finishing && !shutdownInitiated);
 			});
 
@@ -263,7 +264,7 @@ public:
 						m_responseHandler(psession, s.str().c_str(), m_bugname.c_str());
 					}
 					TranscriptEvent empty;
-					m_transcript = empty; 
+					m_transcript = empty;
 
 					switch_core_session_rwunlock(psession);
 				}
@@ -320,8 +321,8 @@ static void *SWITCH_THREAD_FUNC aws_transcribe_thread(switch_thread_t *thread, v
 	struct cap_cb *cb = (struct cap_cb *) obj;
 	bool ok = true;
 	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "transcribe_thread: starting cb %p\n", (void *) cb);
-	GStreamer* pStreamer = new GStreamer(cb->sessionId, cb->bugname, cb->channels, cb->lang, cb->interim, cb->samples_per_second, cb->region, cb->awsAccessKeyId, cb->awsSecretAccessKey, 
-		cb->responseHandler);
+	GStreamer* pStreamer = new GStreamer(cb->sessionId, cb->bugname, cb->channels, cb->lang, cb->interim, cb->samples_per_second,
+		cb->region, cb->awsAccessKeyId, cb->awsSecretAccessKey, cb->awsSessionToken, cb->responseHandler);
 	if (!pStreamer) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "transcribe_thread: Error allocating streamer\n");
 		return nullptr;
@@ -361,7 +362,7 @@ extern "C" {
 		const char* secretAccessKey = std::getenv("AWS_SECRET_ACCESS_KEY");
 		const char* region = std::getenv("AWS_REGION");
 		if (NULL == accessKeyId && NULL == secretAccessKey) {
-			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, 
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE,
 				"\"AWS_ACCESS_KEY_ID\"  and/or \"AWS_SECRET_ACCESS_KEY\" env var not set; authentication will expect channel variables of same names to be set\n");
 		}
 		else {
@@ -369,7 +370,7 @@ extern "C" {
 
 		}
     Aws::SDKOptions options;
-/*		
+/*
     options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Trace;
 
 		Aws::Utils::Logging::InitializeAWSLogging(
@@ -380,7 +381,7 @@ extern "C" {
 
 		return SWITCH_STATUS_SUCCESS;
 	}
-	
+
 	switch_status_t aws_transcribe_cleanup() {
 		Aws::SDKOptions options;
 		/*
@@ -393,7 +394,7 @@ extern "C" {
 	}
 
 	// start transcribe on a channel
-	switch_status_t aws_transcribe_session_init(switch_core_session_t *session, responseHandler_t responseHandler, 
+	switch_status_t aws_transcribe_session_init(switch_core_session_t *session, responseHandler_t responseHandler,
           uint32_t samples_per_second, uint32_t channels, char* lang, int interim, char* bugname, void **ppUserData
 	) {
 		switch_status_t status = SWITCH_STATUS_SUCCESS;
@@ -408,6 +409,7 @@ extern "C" {
 		memset(cb, sizeof(cb), 0);
 		const char* awsAccessKeyId = switch_channel_get_variable(channel, "AWS_ACCESS_KEY_ID");
 		const char* awsSecretAccessKey = switch_channel_get_variable(channel, "AWS_SECRET_ACCESS_KEY");
+		const char* awsSessionToken = switch_channel_get_variable(channel, "AWS_SESSION_TOKEN");
 		const char* awsRegion = switch_channel_get_variable(channel, "AWS_REGION");
 		cb->channels = channels;
 		LanguageCode code = LanguageCodeMapper::GetLanguageCodeForName(lang);
@@ -419,19 +421,19 @@ extern "C" {
 		strncpy(cb->sessionId, switch_core_session_get_uuid(session), MAX_SESSION_ID);
 		strncpy(cb->bugname, bugname, MAX_BUG_LEN);
 
-		if (awsAccessKeyId && awsSecretAccessKey && awsRegion) {
+    if (awsRegion) strncpy(cb->region, awsRegion, MAX_REGION);
+		if (awsAccessKeyId && awsSecretAccessKey) {
 			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Using channel vars for aws authentication\n");
 			strncpy(cb->awsAccessKeyId, awsAccessKeyId, 128);
 			strncpy(cb->awsSecretAccessKey, awsSecretAccessKey, 128);
-			strncpy(cb->region, awsRegion, MAX_REGION);
-
+			if (awsSessionToken) strncpy(cb->awsSessionToken, awsSessionToken, 1024);
 		}
 		else if (std::getenv("AWS_ACCESS_KEY_ID") &&
 			std::getenv("AWS_SECRET_ACCESS_KEY") &&
 			std::getenv("AWS_REGION")) {
 			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Using env vars for aws authentication\n");
 			strncpy(cb->awsAccessKeyId, std::getenv("AWS_ACCESS_KEY_ID"), 128);
-			strncpy(cb->awsSecretAccessKey, std::getenv("AWS_SECRET_ACCESS_KEY"), 128);		
+			strncpy(cb->awsSecretAccessKey, std::getenv("AWS_SECRET_ACCESS_KEY"), 128);
 			strncpy(cb->region, std::getenv("AWS_REGION"), MAX_REGION);
 		}
 		else {
@@ -443,7 +445,7 @@ extern "C" {
 		if (switch_mutex_init(&cb->mutex, SWITCH_MUTEX_NESTED, pool) != SWITCH_STATUS_SUCCESS) {
 			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error initializing mutex\n");
 			status = SWITCH_STATUS_FALSE;
-			goto done; 
+			goto done;
 		}
 
 		cb->interim = interim;
@@ -453,7 +455,7 @@ extern "C" {
 		if (sampleRate != 8000) {
 			cb->resampler = speex_resampler_init(1, sampleRate, 16000, SWITCH_RESAMPLE_QUALITY, &err);
 			if (0 != err) {
-				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing resampler: %s.\n", 
+				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing resampler: %s.\n",
 							switch_channel_get_name(channel), speex_resampler_strerror(err));
 				status = SWITCH_STATUS_FALSE;
 				goto done;
@@ -486,7 +488,7 @@ extern "C" {
 				switch_vad_set_param(cb->vad, "silence_ms", silence_ms);
 				switch_vad_set_param(cb->vad, "voice_ms", voice_ms);
 				switch_vad_set_param(cb->vad, "debug", debug);
-				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "%s: delaying connection until vad, voice_ms %d, mode %d\n", 
+				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "%s: delaying connection until vad, voice_ms %d, mode %d\n",
 					switch_channel_get_name(channel), voice_ms, mode);
 			}
 		}
@@ -497,7 +499,7 @@ extern "C" {
 		switch_thread_create(&cb->thread, thd_attr, aws_transcribe_thread, cb, pool);
 
 		*ppUserData = cb;
-	
+
 	done:
 		return status;
 	}
@@ -519,7 +521,7 @@ extern "C" {
       do {
         streamer = (GStreamer *) cb->streamer;
         if (streamer) break;
-        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, 
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG,
           "aws_transcribe_session_stop: waiting for streamer to come online..%s\n", bugname);
         switch_yield(10000); // wait 10ms
       } while (i++ < 3);
@@ -555,7 +557,7 @@ extern "C" {
 		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s Bug is not attached.\n", switch_channel_get_name(channel));
 		return SWITCH_STATUS_FALSE;
 	}
-	
+
 	switch_bool_t aws_transcribe_frame(switch_media_bug_t *bug, void* user_data) {
 		switch_core_session_t *session = switch_core_media_bug_get_session(bug);
 		uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
@@ -585,7 +587,7 @@ extern "C" {
 						}
 
 						if (cb->resampler) {
-							speex_resampler_process_interleaved_int(cb->resampler, (const spx_int16_t *) frame.data, (spx_uint32_t *) &in_len, &out[0], &out_len);						
+							speex_resampler_process_interleaved_int(cb->resampler, (const spx_int16_t *) frame.data, (spx_uint32_t *) &in_len, &out[0], &out_len);
 							streamer->write( &out[0], sizeof(spx_int16_t) * out_len);
 						}
 						else {
@@ -595,11 +597,11 @@ extern "C" {
 				}
 			}
 			else {
-				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, 
+				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG,
 					"aws_transcribe_frame: not sending audio because aws channel has been closed\n");
 			}
 			switch_mutex_unlock(cb->mutex);
 		}
 		return SWITCH_TRUE;
 	}
-}
+}

mod_aws_transcribe/mod_aws_transcribe.h

@@ -28,6 +28,7 @@ struct cap_cb {
 	char sessionId[MAX_SESSION_ID+1];
   char awsAccessKeyId[128];
   char awsSecretAccessKey[128];
+	char awsSessionToken[1024];
 	uint32_t channels;
   SpeexResamplerState *resampler;
 	void* streamer;

mod_aws_transcribe_ws/Makefile.am

@@ -0,0 +1,10 @@
+include $(top_srcdir)/build/modmake.rulesam
+MODNAME=mod_aws_transcribe_ws
+
+mod_LTLIBRARIES = mod_aws_transcribe_ws.la
+mod_aws_transcribe_ws_la_SOURCES  = mod_aws_transcribe_ws.c aws_transcribe_glue.cpp transcribe_manager.cpp audio_pipe.cpp
+mod_aws_transcribe_ws_la_CFLAGS   = $(AM_CFLAGS)
+mod_aws_transcribe_ws_la_CXXFLAGS = $(AM_CXXFLAGS) -std=c++11 -I${switch_srcdir}/libs/aws-sdk-cpp/aws-cpp-sdk-core/include -I${switch_srcdir}/libs/aws-sdk-cpp/aws-cpp-sdk-transcribestreaming/include -I${switch_srcdir}/libs/aws-sdk-cpp/build/.deps/install/include
+
+mod_aws_transcribe_ws_la_LIBADD   = $(switch_builddir)/libfreeswitch.la
+mod_aws_transcribe_ws_la_LDFLAGS  = -avoid-version -module -no-undefined -shared `pkg-config --libs libwebsockets` 

mod_aws_transcribe_ws/README.md

@@ -0,0 +1,58 @@
+# mod_aws_transcribe
+
+A Freeswitch module that generates real-time transcriptions on a Freeswitch channel by using AWS streaming transcription API
+
+## API
+
+### Commands
+The freeswitch module exposes the following API commands:
+
+```
+aws_transcribe <uuid> start <lang-code> [interim]
+```
+Attaches media bug to channel and performs streaming recognize request.
+- `uuid` - unique identifier of Freeswitch channel
+- `lang-code` - a valid AWS [language code](https://docs.aws.amazon.com/transcribe/latest/dg/what-is-transcribe.html) that is supported for streaming transcription
+- `interim` - If the 'interim' keyword is present then both interim and final transcription results will be returned; otherwise only final transcriptions will be returned
+
+```
+aws_transcribe <uuid> stop
+```
+Stop transcription on the channel.
+
+### Authentication
+The plugin will first look for channel variables, then environment variables.  If neither are found, then the default AWS profile on the server will be used.
+
+The names of the channel variables and environment variables are:
+
+| variable | Description |
+| --- | ----------- |
+| AWS_ACCESS_KEY_ID | The Aws access key ID |
+| AWS_SECRET_ACCESS_KEY | The Aws secret access key |
+| AWS_REGION | The Aws region |
+
+
+### Events
+`aws_transcribe::transcription` - returns an interim or final transcription.  The event contains a JSON body describing the transcription result:
+```js
+[
+  {
+    "is_final": true,
+    "alternatives": [{
+      "transcript": "Hello. Can you hear me?"
+    }]
+  }
+]
+```
+
+## Usage
+When using [drachtio-fsrmf](https://www.npmjs.com/package/drachtio-fsmrf), you can access this API command via the api method on the 'endpoint' object.
+```js
+ep.api('aws_transcribe', `${ep.uuid} start en-US interim`);  
+```
+
+## Building
+This uses the AWS websocket api.
+
+## Examples
+[aws_transcribe.js](../../examples/aws_transcribe.js)

mod_aws_transcribe_ws/audio_pipe.cpp

@@ -0,0 +1,612 @@
+#include "audio_pipe.hpp"
+#include "transcribe_manager.hpp"
+#include "crc.h"
+
+#include <switch.h>
+
+#include <cassert>
+#include <iostream>
+#include <netinet/in.h>
+
+#include <fstream>
+#include <string>
+
+/* discard incoming text messages over the socket that are longer than this */
+#define MAX_RECV_BUF_SIZE (65 * 1024 * 10)
+#define RECV_BUF_REALLOC_SIZE (8 * 1024)
+#define AWS_PRELUDE_PLUS_HDRS_LEN (100)
+
+using namespace aws;
+
+namespace {
+  static const char *requestedTcpKeepaliveSecs = std::getenv("MOD_AUDIO_FORK_TCP_KEEPALIVE_SECS");
+  static int nTcpKeepaliveSecs = requestedTcpKeepaliveSecs ? ::atoi(requestedTcpKeepaliveSecs) : 55;
+  static uint8_t aws_prelude_and_headers[AWS_PRELUDE_PLUS_HDRS_LEN];
+
+  void writeToFile(const char* buffer, size_t bufferSize) {
+    static int writeCounter = 0; // Static variable to keep track of write count
+
+    // Write only the first three times
+    if (writeCounter >= 4) {
+        return;
+    }
+
+    // Generate a unique file name using the writeCounter
+    std::string filename = "/tmp/audio_data_" + std::to_string(writeCounter) + ".bin";
+
+    // Open a file in binary mode
+    std::ofstream outFile(filename, std::ios::binary);
+
+    // Check if the file is open
+    if (outFile.is_open()) {
+        // Write the buffer to the file
+        outFile.write(buffer, bufferSize);
+        outFile.close();
+
+        // Increment the write counter
+        writeCounter++;
+    } else {
+        // Handle error in file opening
+        std::cerr << "Unable to open file: " << filename << std::endl;
+    }
+  }
+}
+
+
+int AudioPipe::aws_lws_callback(struct lws *wsi, 
+  enum lws_callback_reasons reason,
+  void *user, void *in, size_t len) {
+
+  struct AudioPipe::lws_per_vhost_data *vhd = 
+    (struct AudioPipe::lws_per_vhost_data *) lws_protocol_vh_priv_get(lws_get_vhost(wsi), lws_get_protocol(wsi));
+
+  struct lws_vhost* vhost = lws_get_vhost(wsi);
+  AudioPipe ** ppAp = (AudioPipe **) user;
+
+  switch (reason) {
+    case LWS_CALLBACK_PROTOCOL_INIT:
+      vhd = (struct AudioPipe::lws_per_vhost_data *) lws_protocol_vh_priv_zalloc(lws_get_vhost(wsi), lws_get_protocol(wsi), sizeof(struct AudioPipe::lws_per_vhost_data));
+      vhd->context = lws_get_context(wsi);
+      vhd->protocol = lws_get_protocol(wsi);
+      vhd->vhost = lws_get_vhost(wsi);
+      break;
+
+    case LWS_CALLBACK_EVENT_WAIT_CANCELLED:
+      processPendingConnects(vhd);
+      processPendingDisconnects(vhd);
+      processPendingWrites();
+      break;
+    case LWS_CALLBACK_CLIENT_CONNECTION_ERROR:
+      {
+        AudioPipe* ap = findAndRemovePendingConnect(wsi);
+        int rc = lws_http_client_http_response(wsi);
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_CONNECTION_ERROR: %s, response status %d\n", in ? (char *)in : "(null)", rc); 
+        if (ap) {
+          ap->m_state = LWS_CLIENT_FAILED;
+          ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECT_FAIL, (char *) in, ap->isFinished());
+        }
+        else {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_ESTABLISHED %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
+        }
+      }      
+      break;
+
+    case LWS_CALLBACK_CLIENT_ESTABLISHED:
+      {
+        AudioPipe* ap = findAndRemovePendingConnect(wsi);
+        if (ap) {
+          *ppAp = ap;
+          ap->m_vhd = vhd;
+          ap->m_state = LWS_CLIENT_CONNECTED;
+          ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECT_SUCCESS, NULL,  ap->isFinished());
+          //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,"%s connected\n", ap->m_uuid.c_str());
+        }
+        else {
+          lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_ESTABLISHED %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
+        }
+      }      
+      break;
+    case LWS_CALLBACK_CLIENT_CLOSED:
+      {
+        AudioPipe* ap = *ppAp;
+        if (!ap) {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_CLOSED %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
+          return 0;
+        }
+        if (ap->m_state == LWS_CLIENT_DISCONNECTING) {
+          // closed by us
+
+          lwsl_debug("%s socket closed by us\n", ap->m_uuid.c_str());
+          ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECTION_CLOSED_GRACEFULLY, NULL,  ap->isFinished());
+        }
+        else if (ap->m_state == LWS_CLIENT_CONNECTED) {
+          // closed by far end
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,"%s socket closed by far end\n", ap->m_uuid.c_str());
+          ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECTION_DROPPED, NULL,  ap->isFinished());
+        }
+        ap->m_state = LWS_CLIENT_DISCONNECTED;
+        ap->setClosed();
+    
+        //NB: after receiving any of the events above, any holder of a 
+        //pointer or reference to this object must treat is as no longer valid
+
+        //*ppAp = NULL;
+        //delete ap;
+      }
+      break;
+
+    case LWS_CALLBACK_CLIENT_RECEIVE:
+      {
+        AudioPipe* ap = *ppAp;
+        if (!ap) {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
+          return 0;
+        }
+
+        if (lws_is_first_fragment(wsi)) {
+          // allocate a buffer for the entire chunk of memory needed
+          assert(nullptr == ap->m_recv_buf);
+          ap->m_recv_buf_len = len + lws_remaining_packet_payload(wsi);
+          ap->m_recv_buf = (uint8_t*) malloc(ap->m_recv_buf_len);
+          ap->m_recv_buf_ptr = ap->m_recv_buf;
+        }
+
+        size_t write_offset = ap->m_recv_buf_ptr - ap->m_recv_buf;
+        size_t remaining_space = ap->m_recv_buf_len - write_offset;
+        if (remaining_space < len) {
+          lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE buffer realloc needed.\n");
+          size_t newlen = ap->m_recv_buf_len + RECV_BUF_REALLOC_SIZE;
+          if (newlen > MAX_RECV_BUF_SIZE) {
+            free(ap->m_recv_buf);
+            ap->m_recv_buf = ap->m_recv_buf_ptr = nullptr;
+            ap->m_recv_buf_len = 0;
+            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE max buffer exceeded, truncating message.\n");
+          }
+          else {
+            ap->m_recv_buf = (uint8_t*) realloc(ap->m_recv_buf, newlen);
+            if (nullptr != ap->m_recv_buf) {
+              ap->m_recv_buf_len = newlen;
+              ap->m_recv_buf_ptr = ap->m_recv_buf + write_offset;
+            }
+          }
+        }
+
+        if (nullptr != ap->m_recv_buf) {
+          if (len > 0) {
+            memcpy(ap->m_recv_buf_ptr, in, len);
+            ap->m_recv_buf_ptr += len;
+          }
+          if (lws_is_final_fragment(wsi)) {
+            //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "AudioPipe::lws_service_thread - LWS_CALLBACK_CLIENT_RECEIVE received %d bytes.\n", len);
+            if (nullptr != ap->m_recv_buf) {
+              bool isError = false;
+              std::string payload;
+              std::string msg((char *)ap->m_recv_buf, ap->m_recv_buf_ptr - ap->m_recv_buf);
+
+              TranscribeManager::parseResponse(msg, payload, isError, true);
+              //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE payload: %s.\n", payload.c_str());
+              //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE response %s.\n", msg.c_str());
+
+
+              if (0 != payload.compare("{\"Transcript\":{\"Results\":[]}}")) {
+                ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::MESSAGE, payload.c_str(),  ap->isFinished());
+              }
+              if (nullptr != ap->m_recv_buf) free(ap->m_recv_buf);
+            }
+            ap->m_recv_buf = ap->m_recv_buf_ptr = nullptr;
+            ap->m_recv_buf_len = 0;
+          }
+        }
+      }
+      break;
+
+    case LWS_CALLBACK_CLIENT_WRITEABLE:
+      {
+        AudioPipe* ap = *ppAp;
+        if (!ap) {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_WRITEABLE %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi); 
+          return 0;
+        }
+
+        if (ap->m_state == LWS_CLIENT_DISCONNECTING) {
+          lws_close_reason(wsi, LWS_CLOSE_STATUS_NORMAL, NULL, 0);
+          return -1;
+        }
+
+        // check for audio packets
+        {
+          std::lock_guard<std::mutex> lk(ap->m_audio_mutex);
+          if (ap->m_audio_buffer_write_offset > LWS_PRE + AWS_PRELUDE_PLUS_HDRS_LEN || ap->isFinished()) {
+
+            // send a zero length audio packet to indicate end of stream
+            if (ap->isFinished()) {
+              ap->m_audio_buffer_write_offset = LWS_PRE + AWS_PRELUDE_PLUS_HDRS_LEN;
+            }
+            /**
+             * fill in
+             * [0..3] = total byte length
+             * [8..11] = prelude crc
+             * following the audio data: 4 bytes of Message CRC
+             * 
+             */
+
+            // copy in the prelude and headers
+            memcpy(ap->m_audio_buffer + LWS_PRE, aws_prelude_and_headers, AWS_PRELUDE_PLUS_HDRS_LEN);
+
+            // fill in the total byte length
+            uint32_t totalLen = ap->m_audio_buffer_write_offset - LWS_PRE + 4; // for the trailing Message CRC which is 4 bytes
+            //lwsl_err("AudioPipe - total length %u (decimal), 0x%X (hex)\n", totalLen, totalLen);
+            totalLen = htonl(totalLen);
+            //lwsl_err("AudioPipe - total length in network byte order %u (decimal), 0x%X (hex)\n", totalLen, totalLen);
+            memcpy(ap->m_audio_buffer + LWS_PRE, &totalLen, sizeof(uint32_t));
+
+            // fill in the prelude crc
+            uint32_t preludeCRC = CRC::Calculate(ap->m_audio_buffer + LWS_PRE, 8, CRC::CRC_32());
+            //lwsl_err("AudioPipe - prelude CRC %u (decimal), 0x%X (hex)\n", preludeCRC, preludeCRC);
+            preludeCRC = htonl(preludeCRC);
+            //lwsl_err("AudioPipe - prelude CRC in network order %u (decimal), 0x%X (hex)\n", preludeCRC, preludeCRC);
+            memcpy(ap->m_audio_buffer + LWS_PRE + 8, &preludeCRC, sizeof(uint32_t));
+
+            // fill in the message crc
+            uint32_t messageCRC = CRC::Calculate(ap->m_audio_buffer + LWS_PRE, ap->m_audio_buffer_write_offset - LWS_PRE, CRC::CRC_32());
+            messageCRC = htonl(messageCRC);
+            memcpy(ap->m_audio_buffer + ap->m_audio_buffer_write_offset, &messageCRC, sizeof(uint32_t));
+            ap->m_audio_buffer_write_offset + sizeof(uint32_t);
+
+            size_t datalen = ap->m_audio_buffer_write_offset - LWS_PRE + 4;
+            
+            //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,"%s writing data length %lu\n", ap->m_uuid.c_str(), datalen);
+
+            // TMP: write data to a file
+            //writeToFile((const char *) ap->m_audio_buffer + LWS_PRE, datalen);
+
+            int sent = lws_write(wsi, (unsigned char *) ap->m_audio_buffer + LWS_PRE, datalen, LWS_WRITE_BINARY);
+            if (sent < datalen) {
+              switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_WRITEABLE %s attemped to send %lu only sent %d wsi %p..\n", 
+                ap->m_uuid.c_str(), datalen, sent, wsi); 
+            }
+            ap->binaryWritePtrResetToZero();
+          }
+        }
+
+        return 0;
+      }
+      break;
+
+    default:
+      break;
+  }
+  return lws_callback_http_dummy(wsi, reason, user, in, len);
+}
+
+
+// static members
+static const lws_retry_bo_t retry = {
+    nullptr,   // retry_ms_table
+    0,         // retry_ms_table_count
+    0,         // conceal_count
+    UINT16_MAX,         // secs_since_valid_ping
+    UINT16_MAX,        // secs_since_valid_hangup
+    0          // jitter_percent
+};
+
+struct lws_context *AudioPipe::contexts[] = {
+  nullptr, nullptr, nullptr, nullptr, nullptr,
+  nullptr, nullptr, nullptr, nullptr, nullptr
+};
+unsigned int AudioPipe::numContexts = 0;
+unsigned int AudioPipe::nchild = 0;
+std::string AudioPipe::protocolName;
+std::mutex AudioPipe::mutex_connects;
+std::mutex AudioPipe::mutex_disconnects;
+std::mutex AudioPipe::mutex_writes;
+std::list<AudioPipe*> AudioPipe::pendingConnects;
+std::list<AudioPipe*> AudioPipe::pendingDisconnects;
+std::list<AudioPipe*> AudioPipe::pendingWrites;
+AudioPipe::log_emit_function AudioPipe::logger;
+std::mutex AudioPipe::mapMutex;
+std::unordered_map<std::thread::id, bool> AudioPipe::stopFlags;
+std::queue<std::thread::id> AudioPipe::threadIds;
+
+
+void AudioPipe::processPendingConnects(lws_per_vhost_data *vhd) {
+  std::list<AudioPipe*> connects;
+  {
+    std::lock_guard<std::mutex> guard(mutex_connects);
+    for (auto it = pendingConnects.begin(); it != pendingConnects.end(); ++it) {
+      if ((*it)->m_state == LWS_CLIENT_IDLE) {
+        connects.push_back(*it);
+        (*it)->m_state = LWS_CLIENT_CONNECTING;
+      }
+    }
+  }
+  for (auto it = connects.begin(); it != connects.end(); ++it) {
+    AudioPipe* ap = *it;
+    ap->connect_client(vhd);   
+  }
+}
+
+void AudioPipe::processPendingDisconnects(lws_per_vhost_data *vhd) {
+  std::list<AudioPipe*> disconnects;
+  {
+    std::lock_guard<std::mutex> guard(mutex_disconnects);
+    for (auto it = pendingDisconnects.begin(); it != pendingDisconnects.end(); ++it) {
+      if ((*it)->m_state == LWS_CLIENT_DISCONNECTING) disconnects.push_back(*it);
+    }
+    pendingDisconnects.clear();
+  }
+  for (auto it = disconnects.begin(); it != disconnects.end(); ++it) {
+    AudioPipe* ap = *it;
+    lws_callback_on_writable(ap->m_wsi); 
+  }
+}
+
+void AudioPipe::processPendingWrites() {
+  std::list<AudioPipe*> writes;
+  {
+    std::lock_guard<std::mutex> guard(mutex_writes);
+    for (auto it = pendingWrites.begin(); it != pendingWrites.end(); ++it) {
+       if ((*it)->m_state == LWS_CLIENT_CONNECTED) writes.push_back(*it);
+    }  
+    pendingWrites.clear();
+  }
+  for (auto it = writes.begin(); it != writes.end(); ++it) {
+    AudioPipe* ap = *it;
+    lws_callback_on_writable(ap->m_wsi);
+  }
+}
+
+AudioPipe* AudioPipe::findAndRemovePendingConnect(struct lws *wsi) {
+  AudioPipe* ap = NULL;
+  std::lock_guard<std::mutex> guard(mutex_connects);
+  std::list<AudioPipe* > toRemove;
+
+  for (auto it = pendingConnects.begin(); it != pendingConnects.end() && !ap; ++it) {
+    int state = (*it)->m_state;
+
+    if ((*it)->m_wsi == nullptr)
+      toRemove.push_back(*it);
+
+    if ((state == LWS_CLIENT_CONNECTING) &&
+      (*it)->m_wsi == wsi) ap = *it;
+  }
+
+  for (auto it = toRemove.begin(); it != toRemove.end(); ++it)
+    pendingConnects.remove(*it);
+
+  if (ap) {
+    pendingConnects.remove(ap);
+  }
+
+  return ap;
+}
+
+AudioPipe* AudioPipe::findPendingConnect(struct lws *wsi) {
+  AudioPipe* ap = NULL;
+  std::lock_guard<std::mutex> guard(mutex_connects);
+
+  for (auto it = pendingConnects.begin(); it != pendingConnects.end() && !ap; ++it) {
+    int state = (*it)->m_state;
+    if ((state == LWS_CLIENT_CONNECTING) &&
+      (*it)->m_wsi == wsi) ap = *it;
+  }
+  return ap;
+}
+
+void AudioPipe::addPendingConnect(AudioPipe* ap) {
+  {
+    std::lock_guard<std::mutex> guard(mutex_connects);
+    pendingConnects.push_back(ap);
+    lwsl_debug("%s after adding connect there are %lu pending connects\n", 
+      ap->m_uuid.c_str(), pendingConnects.size());
+  }
+  lws_cancel_service(contexts[nchild++ % numContexts]);
+}
+void AudioPipe::addPendingDisconnect(AudioPipe* ap) {
+  ap->m_state = LWS_CLIENT_DISCONNECTING;
+  {
+    std::lock_guard<std::mutex> guard(mutex_disconnects);
+    pendingDisconnects.push_back(ap);
+    lwsl_debug("%s after adding disconnect there are %lu pending disconnects\n", 
+      ap->m_uuid.c_str(), pendingDisconnects.size());
+  }
+  lws_cancel_service(ap->m_vhd->context);
+}
+void AudioPipe::addPendingWrite(AudioPipe* ap) {
+  {
+    std::lock_guard<std::mutex> guard(mutex_writes);
+    pendingWrites.push_back(ap);
+  }
+  lws_cancel_service(ap->m_vhd->context);
+}
+
+void AudioPipe::binaryWritePtrResetToZero(void) {
+    m_audio_buffer_write_offset = LWS_PRE + AWS_PRELUDE_PLUS_HDRS_LEN;
+}
+
+bool AudioPipe::lws_service_thread(unsigned int nServiceThread) {
+  struct lws_context_creation_info info;
+  std::thread::id this_id = std::this_thread::get_id();
+
+  const struct lws_protocols protocols[] = {
+    {
+      "",
+      AudioPipe::aws_lws_callback,
+      sizeof(void *),
+      1024,
+    },
+    { NULL, NULL, 0, 0 }
+  };
+
+  memset(&info, 0, sizeof info); 
+  info.port = CONTEXT_PORT_NO_LISTEN; 
+  info.options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT;
+  info.protocols = protocols;
+  info.ka_time = nTcpKeepaliveSecs;                    // tcp keep-alive timer
+  info.ka_probes = 4;                   // number of times to try ka before closing connection
+  info.ka_interval = 5;                 // time between ka's
+  info.timeout_secs = 10;                // doc says timeout for "various processes involving network roundtrips"
+  info.keepalive_timeout = 5;           // seconds to allow remote client to hold on to an idle HTTP/1.1 connection 
+  info.timeout_secs_ah_idle = 10;       // secs to allow a client to hold an ah without using it
+  info.retry_and_idle_policy = &retry;
+
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,"AudioPipe::lws_service_thread creating context\n");
+
+  contexts[nServiceThread] = lws_create_context(&info);
+  if (!contexts[nServiceThread]) {
+    lwsl_err("AudioPipe::lws_service_thread failed creating context in service thread %d..\n", nServiceThread); 
+    return false;
+  }
+
+  int n;
+  do {
+    n = lws_service(contexts[nServiceThread], 0);
+  } while (n >= 0 && !stopFlags[this_id]);
+
+  // Cleanup once work is done or stopped
+  {
+      std::lock_guard<std::mutex> lock(mapMutex);
+      stopFlags.erase(this_id);
+  }
+
+  lwsl_notice("AudioPipe::lws_service_thread ending in service thread %d\n", nServiceThread); 
+  return true;
+}
+
+void AudioPipe::initialize(unsigned int nThreads, int loglevel, log_emit_function logger) {
+  assert(nThreads > 0 && nThreads <= 10);
+
+  numContexts = nThreads;
+  lws_set_log_level(loglevel, logger);
+
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE,"AudioPipe::initialize starting\n"); 
+  for (unsigned int i = 0; i < numContexts; i++) {
+    std::lock_guard<std::mutex> lock(mapMutex);
+    std::thread t(&AudioPipe::lws_service_thread, i);
+    stopFlags[t.get_id()] = false;
+    threadIds.push(t.get_id());
+    t.detach();
+  }
+}
+
+bool AudioPipe::deinitialize() {
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE,"AudioPipe::deinitialize\n"); 
+  std::lock_guard<std::mutex> lock(mapMutex);
+  if (!threadIds.empty()) {
+      std::thread::id id = threadIds.front();
+      threadIds.pop();
+      stopFlags[id] = true;
+  }
+  for (unsigned int i = 0; i < numContexts; i++)
+  {
+    lwsl_notice("AudioPipe::deinitialize destroying context %d of %d\n", i + 1, numContexts);
+    lws_context_destroy(contexts[i]);
+  }
+  std::this_thread::sleep_for(std::chrono::seconds(2));
+  return true;
+}
+
+// instance members
+AudioPipe::AudioPipe(const char* uuid, const char* bugname, const char* host, unsigned int port, const char* path,
+  size_t bufLen, size_t minFreespace, notifyHandler_t callback) :
+  m_uuid(uuid), m_host(host), m_port(port), m_path(path), m_finished(false), m_bugname(bugname),
+  m_audio_buffer_min_freespace(minFreespace), m_audio_buffer_max_len(bufLen), m_gracefulShutdown(false),
+  m_recv_buf(nullptr), m_recv_buf_ptr(nullptr),
+  m_state(LWS_CLIENT_IDLE), m_wsi(nullptr), m_vhd(nullptr), m_callback(callback) {
+
+  char headerBuffer[88];
+  char* buffer = headerBuffer;
+  m_audio_buffer = new uint8_t[m_audio_buffer_max_len];
+
+  // stamp out the template for the prelude and headers 
+  memset(aws_prelude_and_headers, 0, AWS_PRELUDE_PLUS_HDRS_LEN);
+
+  // aws_prelude_and_headers[0..3] = total byte length (not known till message send time)
+
+  // aws_prelude_and_headers[4..7] = headers byte length
+  uint32_t headerLen = sizeof(headerBuffer);
+  headerLen = htonl(headerLen);
+  memcpy(&aws_prelude_and_headers[4], &headerLen, sizeof(uint32_t));
+
+  // aws_prelude_and_headers[8..11] = prelude crc (not known till message send time)
+
+  // aws_prelude_and_headers[12..99] = headers
+  TranscribeManager::writeHeader(&buffer, ":content-type", "application/octet-stream");
+  TranscribeManager::writeHeader(&buffer, ":event-type", "AudioEvent");
+  TranscribeManager::writeHeader(&buffer, ":message-type", "event");
+
+  memcpy(&aws_prelude_and_headers[12], headerBuffer, sizeof(headerBuffer));
+
+  // following this will be the audio data and a final message CRC (not known till message send time)
+
+  memcpy(m_audio_buffer + LWS_PRE, aws_prelude_and_headers, AWS_PRELUDE_PLUS_HDRS_LEN);
+  m_audio_buffer_write_offset = LWS_PRE + AWS_PRELUDE_PLUS_HDRS_LEN;
+
+  //writeToFile((const char *) m_audio_buffer + LWS_PRE, AWS_PRELUDE_PLUS_HDRS_LEN);
+
+}
+
+AudioPipe::~AudioPipe() {
+  if (m_audio_buffer) delete [] m_audio_buffer;
+  if (m_recv_buf) delete [] m_recv_buf;
+}
+
+void AudioPipe::connect(void) {
+  addPendingConnect(this);
+}
+
+bool AudioPipe::connect_client(struct lws_per_vhost_data *vhd) {
+  assert(m_audio_buffer != nullptr);
+  assert(m_vhd == nullptr);
+  struct lws_client_connect_info i;
+
+  memset(&i, 0, sizeof(i));
+  i.context = vhd->context;
+  i.port = m_port;
+  i.address = m_host.c_str();
+  i.path = m_path.c_str();
+  i.host = i.address;
+  i.origin = i.address;
+  i.ssl_connection = LCCSCF_USE_SSL;
+  //i.protocol = protocolName.c_str();
+  i.pwsi = &(m_wsi);
+
+  m_state = LWS_CLIENT_CONNECTING;
+  m_vhd = vhd;
+
+  m_wsi = lws_client_connect_via_info(&i);
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,"%s attempting connection, wsi is %p\n", m_uuid.c_str(), m_wsi);
+
+  return nullptr != m_wsi;
+}
+
+void AudioPipe::bufferForSending(const char* text) {
+  if (m_state != LWS_CLIENT_CONNECTED) return;
+  {
+    std::lock_guard<std::mutex> lk(m_text_mutex);
+    m_metadata.append(text);
+  }
+  addPendingWrite(this);
+}
+
+void AudioPipe::unlockAudioBuffer() {
+  if (m_audio_buffer_write_offset > LWS_PRE) addPendingWrite(this);
+  m_audio_mutex.unlock();
+}
+
+void AudioPipe::close() {
+  if (m_state != LWS_CLIENT_CONNECTED) return;
+  addPendingDisconnect(this);
+}
+
+void AudioPipe::finish() {
+  if (m_finished || m_state != LWS_CLIENT_CONNECTED) return;
+  m_finished = true;
+  addPendingWrite(this);
+}
+
+void AudioPipe::waitForClose() {
+  std::shared_future<void> sf(m_promise.get_future());
+  sf.wait();
+  return;
+}

mod_aws_transcribe_ws/audio_pipe.hpp

@@ -0,0 +1,144 @@
+#ifndef __AWS_AUDIO_PIPE_HPP__
+#define __AWS_AUDIO_PIPE_HPP__
+
+#include <string>
+#include <list>
+#include <mutex>
+#include <future>
+#include <queue>
+#include <unordered_map>
+#include <thread>
+
+#include <libwebsockets.h>
+
+namespace aws {
+
+class AudioPipe {
+public:
+  enum LwsState_t {
+    LWS_CLIENT_IDLE,
+    LWS_CLIENT_CONNECTING,
+    LWS_CLIENT_CONNECTED,
+    LWS_CLIENT_FAILED,
+    LWS_CLIENT_DISCONNECTING,
+    LWS_CLIENT_DISCONNECTED
+  };
+  enum NotifyEvent_t {
+    CONNECT_SUCCESS,
+    CONNECT_FAIL,
+    CONNECTION_DROPPED,
+    CONNECTION_CLOSED_GRACEFULLY,
+    MESSAGE
+  };
+  typedef void (*log_emit_function)(int level, const char *line);
+  typedef void (*notifyHandler_t)(const char *sessionId, const char* bugname, NotifyEvent_t event, const char* message, bool finished);
+
+  struct lws_per_vhost_data {
+    struct lws_context *context;
+    struct lws_vhost *vhost;
+    const struct lws_protocols *protocol;
+  };
+
+  static void initialize(unsigned int nThreads, int loglevel, log_emit_function logger);
+  static bool deinitialize();
+  static bool lws_service_thread(unsigned int nServiceThread);
+
+  // constructor
+  AudioPipe(const char* uuid, const char* bugname, const char* host, unsigned int port, const char* path, 
+    size_t bufLen, size_t minFreespace, notifyHandler_t callback);
+  ~AudioPipe();  
+
+  LwsState_t getLwsState(void) { return m_state; }
+  std::string& getApiKey(void) {
+    return m_apiKey;
+  }
+  void connect(void);
+  void bufferForSending(const char* text);
+  size_t binarySpaceAvailable(void) {
+    return m_audio_buffer_max_len - m_audio_buffer_write_offset;
+  }
+  size_t binaryMinSpace(void) {
+    return m_audio_buffer_min_freespace;
+  }
+  char * binaryWritePtr(void) { 
+    return (char *) m_audio_buffer + m_audio_buffer_write_offset;
+  }
+  void binaryWritePtrAdd(size_t len) {
+    m_audio_buffer_write_offset += len;
+  }
+  void binaryWritePtrResetToZero(void);
+  void lockAudioBuffer(void) {
+    m_audio_mutex.lock();
+  }
+  void unlockAudioBuffer(void) ;
+
+  void close() ;
+  void finish();
+  void waitForClose();
+  void setClosed() { m_promise.set_value(); }
+  bool isFinished() { return m_finished;}
+
+  // no default constructor or copying
+  AudioPipe() = delete;
+  AudioPipe(const AudioPipe&) = delete;
+  void operator=(const AudioPipe&) = delete;
+
+private:
+
+  static int aws_lws_callback(struct lws *wsi, enum lws_callback_reasons reason, void *user, void *in, size_t len); 
+  static unsigned int nchild;
+  static struct lws_context *contexts[];
+  static unsigned int numContexts;
+  static std::string protocolName;
+  static std::mutex mutex_connects;
+  static std::mutex mutex_disconnects;
+  static std::mutex mutex_writes;
+  static std::list<AudioPipe*> pendingConnects;
+  static std::list<AudioPipe*> pendingDisconnects;
+  static std::list<AudioPipe*> pendingWrites;
+  static log_emit_function logger;
+
+  static std::mutex mapMutex;
+  static std::unordered_map<std::thread::id, bool> stopFlags;
+  static std::queue<std::thread::id> threadIds;
+
+  static AudioPipe* findAndRemovePendingConnect(struct lws *wsi);
+  static AudioPipe* findPendingConnect(struct lws *wsi);
+  static void addPendingConnect(AudioPipe* ap);
+  static void addPendingDisconnect(AudioPipe* ap);
+  static void addPendingWrite(AudioPipe* ap);
+  static void processPendingConnects(lws_per_vhost_data *vhd);
+  static void processPendingDisconnects(lws_per_vhost_data *vhd);
+  static void processPendingWrites(void);
+  
+  bool connect_client(struct lws_per_vhost_data *vhd);
+
+  LwsState_t m_state;
+  std::string m_uuid;
+  std::string m_host;
+  unsigned int m_port;
+  std::string m_path;
+  std::string m_metadata;
+  std::mutex m_text_mutex;
+  std::mutex m_audio_mutex;
+  int m_sslFlags;
+  struct lws *m_wsi;
+  uint8_t *m_audio_buffer;
+  size_t m_audio_buffer_max_len;
+  size_t m_audio_buffer_write_offset;
+  size_t m_audio_buffer_min_freespace;
+  uint8_t* m_recv_buf;
+  uint8_t* m_recv_buf_ptr;
+  size_t m_recv_buf_len;
+  struct lws_per_vhost_data* m_vhd;
+  notifyHandler_t m_callback;
+  log_emit_function m_logger;
+  std::string m_apiKey;
+  bool m_gracefulShutdown;
+  bool m_finished;
+  std::string m_bugname;
+  std::promise<void> m_promise;
+};
+
+} // namespace deepgram
+#endif

mod_aws_transcribe_ws/aws_transcribe_glue.cpp

@@ -0,0 +1,415 @@
+#include <switch.h>
+#include <switch_json.h>
+#include <string.h>
+#include <string>
+#include <mutex>
+#include <thread>
+#include <list>
+#include <algorithm>
+#include <functional>
+#include <cassert>
+#include <cstdlib>
+#include <fstream>
+#include <sstream>
+#include <regex>
+#include <iostream>
+#include <unordered_map>
+
+#include "mod_aws_transcribe_ws.h"
+#include "simple_buffer.h"
+//#include "parser.hpp"
+#include "audio_pipe.hpp"
+#include "transcribe_manager.hpp"
+
+#define RTP_PACKETIZATION_PERIOD 20
+#define FRAME_SIZE_8000  320 /*which means each 20ms frame as 320 bytes at 8 khz (1 channel only)*/
+
+namespace {
+  static bool hasDefaultCredentials = false;
+  static const char* defaultApiKey = nullptr;
+  static const char *requestedBufferSecs = std::getenv("MOD_AUDIO_FORK_BUFFER_SECS");
+  static int nAudioBufferSecs = std::max(1, std::min(requestedBufferSecs ? ::atoi(requestedBufferSecs) : 2, 5));
+  static const char *requestedNumServiceThreads = std::getenv("MOD_AUDIO_FORK_SERVICE_THREADS");
+  static unsigned int nServiceThreads = std::max(1, std::min(requestedNumServiceThreads ? ::atoi(requestedNumServiceThreads) : 1, 5));
+  static unsigned int idxCallCount = 0;
+  static uint32_t playCount = 0;
+
+  static const char* emptyTranscript = "{\"Transcript\":{\"Results\":[]}}";
+  static const char* messageStart = "{\"Message\":";
+
+  static void reaper(private_t *tech_pvt) {
+    std::shared_ptr<aws::AudioPipe> pAp;
+    pAp.reset((aws::AudioPipe *)tech_pvt->pAudioPipe);
+    tech_pvt->pAudioPipe = nullptr;
+
+    std::thread t([pAp, tech_pvt]{
+      pAp->finish();
+      pAp->waitForClose();
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s (%u) got remote close\n", tech_pvt->sessionId, tech_pvt->id);
+    });
+    t.detach();
+  }
+
+  static void destroy_tech_pvt(private_t *tech_pvt) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "%s (%u) destroy_tech_pvt\n", tech_pvt->sessionId, tech_pvt->id);
+    if (tech_pvt) {
+      if (tech_pvt->pAudioPipe) {
+        aws::AudioPipe* p = (aws::AudioPipe *) tech_pvt->pAudioPipe;
+        delete p;
+        tech_pvt->pAudioPipe = nullptr;
+      }
+      if (tech_pvt->resampler) {
+          speex_resampler_destroy(tech_pvt->resampler);
+          tech_pvt->resampler = NULL;
+      }
+
+      if (tech_pvt->vad) {
+        switch_vad_destroy(&tech_pvt->vad);
+        tech_pvt->vad = nullptr;
+      }
+    }
+  }
+
+  static void eventCallback(const char* sessionId, const char* bugname, 
+    aws::AudioPipe::NotifyEvent_t event, const char* message, bool finished) {
+    switch_core_session_t* session = switch_core_session_locate(sessionId);
+    if (session) {
+      switch_channel_t *channel = switch_core_session_get_channel(session);
+      switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, bugname);
+      if (bug) {
+        private_t* tech_pvt = (private_t*) switch_core_media_bug_get_user_data(bug);
+        if (tech_pvt) {
+          switch (event) {
+            case aws::AudioPipe::CONNECT_SUCCESS:
+              switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "connection successful\n");
+              tech_pvt->responseHandler(session, TRANSCRIBE_EVENT_CONNECT_SUCCESS, NULL, tech_pvt->bugname, finished);
+            break;
+            case aws::AudioPipe::CONNECT_FAIL:
+            {
+              // first thing: we can no longer access the AudioPipe
+              std::stringstream json;
+              json << "{\"reason\":\"" << message << "\"}";
+              tech_pvt->pAudioPipe = nullptr;
+              tech_pvt->responseHandler(session, TRANSCRIBE_EVENT_CONNECT_FAIL, (char *) json.str().c_str(), tech_pvt->bugname, finished);
+              switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_NOTICE, "connection failed: %s\n", message);
+            }
+            break;
+            case aws::AudioPipe::CONNECTION_DROPPED:
+              // first thing: we can no longer access the AudioPipe
+              tech_pvt->pAudioPipe = nullptr;
+              tech_pvt->responseHandler(session, TRANSCRIBE_EVENT_DISCONNECT, NULL, tech_pvt->bugname, finished);
+              switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connection dropped from far end\n");
+            break;
+            case aws::AudioPipe::CONNECTION_CLOSED_GRACEFULLY:
+              // first thing: we can no longer access the AudioPipe
+              tech_pvt->pAudioPipe = nullptr;
+              switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connection closed gracefully\n");
+            break;
+            case aws::AudioPipe::MESSAGE:
+              if( strstr(message, emptyTranscript)) {
+                switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "discarding empty aws transcript\n");
+              }
+              else if (0 == strncmp( message, messageStart, strlen(messageStart))) {
+                tech_pvt->responseHandler(session, TRANSCRIBE_EVENT_ERROR, message, tech_pvt->bugname, finished);
+                switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "error message from aws: %s\n", message);
+              }
+              else {
+                tech_pvt->responseHandler(session, TRANSCRIBE_EVENT_RESULTS, message, tech_pvt->bugname, finished);
+                switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "aws message: %s.\n", message);
+              }
+            break;
+
+            default:
+              switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_NOTICE, "got unexpected msg from aws %d:%s\n", event, message);
+              break;
+          }
+        }
+      }
+      switch_core_session_rwunlock(session);
+    }
+  }
+
+  void lws_logger(int level, const char *line) {
+    switch_log_level_t llevel = SWITCH_LOG_DEBUG;
+
+    switch (level) {
+      case LLL_ERR: llevel = SWITCH_LOG_ERROR; break;
+      case LLL_WARN: llevel = SWITCH_LOG_WARNING; break;
+      case LLL_NOTICE: llevel = SWITCH_LOG_NOTICE; break;
+      case LLL_INFO: llevel = SWITCH_LOG_INFO; break;
+      break;
+    }
+	  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "%s\n", line);
+  }
+}
+
+
+extern "C" {
+  switch_status_t aws_transcribe_init() {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_aws_transcribe: audio buffer (in secs):    %d secs\n", nAudioBufferSecs);
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_aws_transcribe: lws service threads:       %d\n", nServiceThreads);
+ 
+    int logs = LLL_ERR | LLL_WARN | LLL_NOTICE || LLL_INFO | LLL_PARSER | LLL_HEADER | LLL_EXT | LLL_CLIENT  | LLL_LATENCY | LLL_DEBUG ;
+    
+    aws::AudioPipe::initialize(nServiceThreads, logs, lws_logger);
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "AudioPipe::initialize completed\n");
+
+		return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t aws_transcribe_cleanup() {
+    bool cleanup = false;
+    cleanup = aws::AudioPipe::deinitialize();
+    if (cleanup == true) {
+        return SWITCH_STATUS_SUCCESS;
+    }
+    return SWITCH_STATUS_FALSE;
+  }
+	
+  	// start transcribe on a channel
+	switch_status_t aws_transcribe_session_init(switch_core_session_t *session, responseHandler_t responseHandler, 
+          uint32_t samples_per_second, uint32_t channels, char* lang, int interim, char* bugname, void **ppUserData
+	) {
+		switch_status_t status = SWITCH_STATUS_SUCCESS;
+		switch_channel_t *channel = switch_core_session_get_channel(session);
+		int err;
+    uint32_t desiredSampling = 8000;
+		switch_threadattr_t *thd_attr = NULL;
+		switch_memory_pool_t *pool = switch_core_session_get_pool(session);
+		auto read_codec = switch_core_session_get_read_codec(session);
+		uint32_t sampleRate = read_codec->implementation->actual_samples_per_second;
+    switch_codec_implementation_t read_impl;
+    switch_core_session_get_read_impl(session, &read_impl);
+
+		private_t* tech_pvt = (private_t *) switch_core_session_alloc(session, sizeof(private_t));
+		memset(tech_pvt, sizeof(tech_pvt), 0);
+		const char* awsAccessKeyId = switch_channel_get_variable(channel, "AWS_ACCESS_KEY_ID");
+		const char* awsSecretAccessKey = switch_channel_get_variable(channel, "AWS_SECRET_ACCESS_KEY");
+		const char* awsRegion = switch_channel_get_variable(channel, "AWS_REGION");
+		const char* awsSessionToken = switch_channel_get_variable(channel, "AWS_SECURITY_TOKEN");
+		tech_pvt->channels = channels;
+		strncpy(tech_pvt->sessionId, switch_core_session_get_uuid(session), MAX_SESSION_ID);
+		strncpy(tech_pvt->bugname, bugname, MAX_BUG_LEN);
+
+		if (awsAccessKeyId && awsSecretAccessKey && awsRegion && awsSessionToken) {
+			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Using channel vars for aws authentication\n");
+			strncpy(tech_pvt->awsAccessKeyId, awsAccessKeyId, 128);
+			strncpy(tech_pvt->awsSecretAccessKey, awsSecretAccessKey, 128);
+			strncpy(tech_pvt->awsSessionToken, awsSessionToken, MAX_SESSION_TOKEN_LEN);
+			strncpy(tech_pvt->region, awsRegion, MAX_REGION);
+		}
+		else if (std::getenv("AWS_ACCESS_KEY_ID") &&
+			std::getenv("AWS_SECRET_ACCESS_KEY") &&
+			std::getenv("AWS_REGION")) {
+			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Using env vars for aws authentication\n");
+			strncpy(tech_pvt->awsAccessKeyId, std::getenv("AWS_ACCESS_KEY_ID"), 128);
+			strncpy(tech_pvt->awsSecretAccessKey, std::getenv("AWS_SECRET_ACCESS_KEY"), 128);		
+			strncpy(tech_pvt->region, std::getenv("AWS_REGION"), MAX_REGION);
+		}
+		else {
+			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "No channel vars or env vars for aws authentication..will use default profile if found\n");
+		}
+
+		tech_pvt->responseHandler = responseHandler;
+
+		tech_pvt->interim = interim;
+		strncpy(tech_pvt->lang, lang, MAX_LANG);
+		tech_pvt->samples_per_second = sampleRate;
+		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "sample rate of rtp stream is %d\n", samples_per_second);
+
+    const char* vocabularyName = switch_channel_get_variable(channel, "AWS_VOCABULARY_NAME");
+    const char* vocabularyFilterName = switch_channel_get_variable(channel, "AWS_VOCABULARY_FILTER_NAME");
+    const char* vocabularyFilterMethod = switch_channel_get_variable(channel, "AWS_VOCABULARY_FILTER_METHOD");
+    const char* piiEntityTypes = switch_channel_get_variable(channel, "AWS_PII_ENTITY_TYPES");
+    int shouldIdentifyPII = switch_true(switch_channel_get_variable(channel, "AWS_PII_IDENTIFY_ENTITIES"));
+    const char* languageModelName = switch_channel_get_variable(channel, "AWS_LANGUAGE_MODEL_NAME");
+
+    std::string host, path;
+    TranscribeManager::getSignedWebsocketUrl(
+      host,
+      path,
+      tech_pvt->awsAccessKeyId,
+      tech_pvt->awsSecretAccessKey, 
+      tech_pvt->awsSessionToken, 
+      tech_pvt->region,
+      lang,
+      vocabularyName,
+      vocabularyFilterName,
+      vocabularyFilterMethod,
+      piiEntityTypes,
+      shouldIdentifyPII,
+      languageModelName
+    );
+
+    host = host.substr(0, host.find(':'));
+		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connecting to host %s, path %s\n", host.c_str(), path.c_str());
+
+    strncpy(tech_pvt->sessionId, switch_core_session_get_uuid(session), MAX_SESSION_ID);
+    strncpy(tech_pvt->host, host.c_str(), MAX_WS_URL_LEN);
+    tech_pvt->port = 8443;
+    strncpy(tech_pvt->path, path.c_str(), MAX_PATH_LEN);    
+    tech_pvt->responseHandler = responseHandler;
+    tech_pvt->channels = channels;
+    tech_pvt->id = ++idxCallCount;
+    tech_pvt->buffer_overrun_notified = 0;
+
+    size_t buflen = LWS_PRE + (FRAME_SIZE_8000 * desiredSampling / 8000 * channels * 1000 / RTP_PACKETIZATION_PERIOD * nAudioBufferSecs);
+
+    aws::AudioPipe* ap = new aws::AudioPipe(tech_pvt->sessionId, tech_pvt->bugname, tech_pvt->host, tech_pvt->port, tech_pvt->path, 
+      buflen, read_impl.decoded_bytes_per_packet, eventCallback);
+    if (!ap) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error allocating AudioPipe\n");
+      return SWITCH_STATUS_FALSE;
+    }
+
+    tech_pvt->pAudioPipe = static_cast<void *>(ap);
+
+		if (switch_mutex_init(&tech_pvt->mutex, SWITCH_MUTEX_NESTED, pool) != SWITCH_STATUS_SUCCESS) {
+			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error initializing mutex\n");
+			status = SWITCH_STATUS_FALSE;
+			goto done; 
+		}
+		if (sampleRate != 8000) {
+			tech_pvt->resampler = speex_resampler_init(1, sampleRate, 16000, SWITCH_RESAMPLE_QUALITY, &err);
+			if (0 != err) {
+				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing resampler: %s.\n", 
+							switch_channel_get_name(channel), speex_resampler_strerror(err));
+				status = SWITCH_STATUS_FALSE;
+				goto done;
+			}
+		}
+
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connecting now\n");
+    ap->connect();
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connection in progress\n");
+
+
+		*ppUserData = tech_pvt;
+	
+	done:
+		return status;
+	}
+
+	switch_status_t aws_transcribe_session_stop(switch_core_session_t *session,int channelIsClosing, char* bugname) {
+    switch_channel_t *channel = switch_core_session_get_channel(session);
+    switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, MY_BUG_NAME);
+    if (!bug) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "aws_transcribe_session_stop: no bug - websocket conection already closed\n");
+      return SWITCH_STATUS_FALSE;
+    }
+    private_t* tech_pvt = (private_t*) switch_core_media_bug_get_user_data(bug);
+    uint32_t id = tech_pvt->id;
+
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) aws_transcribe_session_stop\n", id);
+
+    if (!tech_pvt) return SWITCH_STATUS_FALSE;
+      
+    // close connection and get final responses
+    switch_mutex_lock(tech_pvt->mutex);
+    switch_channel_set_private(channel, bugname, NULL);
+    if (!channelIsClosing) switch_core_media_bug_remove(session, &bug);
+
+    aws::AudioPipe *pAudioPipe = static_cast<aws::AudioPipe *>(tech_pvt->pAudioPipe);
+    if (pAudioPipe) reaper(tech_pvt);
+    destroy_tech_pvt(tech_pvt);
+    switch_mutex_unlock(tech_pvt->mutex);
+    switch_mutex_destroy(tech_pvt->mutex);
+    tech_pvt->mutex = nullptr;
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) aws_transcribe_session_stop\n", id);
+    return SWITCH_STATUS_SUCCESS;
+  }
+	
+	switch_bool_t aws_transcribe_frame(switch_media_bug_t *bug, void* user_data) {
+    switch_core_session_t *session = switch_core_media_bug_get_session(bug);
+    private_t* tech_pvt = (private_t*) switch_core_media_bug_get_user_data(bug);
+    size_t inuse = 0;
+    bool dirty = false;
+    char *p = (char *) "{\"msg\": \"buffer overrun\"}";
+
+    if (!tech_pvt) return SWITCH_TRUE;
+    
+    if (switch_mutex_trylock(tech_pvt->mutex) == SWITCH_STATUS_SUCCESS) {
+      if (!tech_pvt->pAudioPipe) {
+        switch_mutex_unlock(tech_pvt->mutex);
+        return SWITCH_TRUE;
+      }
+      aws::AudioPipe *pAudioPipe = static_cast<aws::AudioPipe *>(tech_pvt->pAudioPipe);
+      if (pAudioPipe->getLwsState() != aws::AudioPipe::LWS_CLIENT_CONNECTED) {
+        switch_mutex_unlock(tech_pvt->mutex);
+        return SWITCH_TRUE;
+      }
+
+      pAudioPipe->lockAudioBuffer();
+      size_t available = pAudioPipe->binarySpaceAvailable();
+      if (NULL == tech_pvt->resampler) {
+        switch_frame_t frame = { 0 };
+        frame.data = pAudioPipe->binaryWritePtr();
+        frame.buflen = available;
+        while (true) {
+
+          // check if buffer would be overwritten; dump packets if so
+          if (available < pAudioPipe->binaryMinSpace()) {
+            if (!tech_pvt->buffer_overrun_notified) {
+              tech_pvt->buffer_overrun_notified = 1;
+              tech_pvt->responseHandler(session, TRANSCRIBE_EVENT_BUFFER_OVERRUN, NULL, tech_pvt->bugname, 0);
+            }
+            switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "(%u) dropping packets!\n", 
+              tech_pvt->id);
+            pAudioPipe->binaryWritePtrResetToZero();
+
+            frame.data = pAudioPipe->binaryWritePtr();
+            frame.buflen = available = pAudioPipe->binarySpaceAvailable();
+          }
+
+          switch_status_t rv = switch_core_media_bug_read(bug, &frame, SWITCH_TRUE);
+          if (rv != SWITCH_STATUS_SUCCESS) break;
+          if (frame.datalen) {
+            pAudioPipe->binaryWritePtrAdd(frame.datalen);
+            frame.buflen = available = pAudioPipe->binarySpaceAvailable();
+            frame.data = pAudioPipe->binaryWritePtr();
+            dirty = true;
+          }
+        }
+      }
+      else {
+        uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
+        switch_frame_t frame = { 0 };
+        frame.data = data;
+        frame.buflen = SWITCH_RECOMMENDED_BUFFER_SIZE;
+        while (switch_core_media_bug_read(bug, &frame, SWITCH_TRUE) == SWITCH_STATUS_SUCCESS) {
+          if (frame.datalen) {
+            spx_uint32_t out_len = available >> 1;  // space for samples which are 2 bytes
+            spx_uint32_t in_len = frame.samples;
+
+            speex_resampler_process_interleaved_int(tech_pvt->resampler, 
+              (const spx_int16_t *) frame.data, 
+              (spx_uint32_t *) &in_len, 
+              (spx_int16_t *) ((char *) pAudioPipe->binaryWritePtr()),
+              &out_len);
+
+            if (out_len > 0) {
+              // bytes written = num samples * 2 * num channels
+              size_t bytes_written = out_len << tech_pvt->channels;
+              pAudioPipe->binaryWritePtrAdd(bytes_written);
+              available = pAudioPipe->binarySpaceAvailable();
+              dirty = true;
+            }
+            if (available < pAudioPipe->binaryMinSpace()) {
+              if (!tech_pvt->buffer_overrun_notified) {
+                tech_pvt->buffer_overrun_notified = 1;
+                switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "(%u) dropping packets!\n", 
+                  tech_pvt->id);
+                tech_pvt->responseHandler(session, TRANSCRIBE_EVENT_BUFFER_OVERRUN, NULL, tech_pvt->bugname, 0);
+              }
+              break;
+            }
+          }
+        }
+      }
+
+      pAudioPipe->unlockAudioBuffer();
+      switch_mutex_unlock(tech_pvt->mutex);
+    }
+    return SWITCH_TRUE;
+  }
+}

mod_aws_transcribe_ws/aws_transcribe_glue.h

@@ -0,0 +1,11 @@
+#ifndef __AWS_GLUE_H__
+#define __AWS_GLUE_H__
+
+switch_status_t aws_transcribe_init();
+switch_status_t aws_transcribe_cleanup();
+switch_status_t aws_transcribe_session_init(switch_core_session_t *session, responseHandler_t responseHandler, 
+		uint32_t samples_per_second, uint32_t channels, char* lang, int interim, char* bugname, void **ppUserData);
+switch_status_t aws_transcribe_session_stop(switch_core_session_t *session, int channelIsClosing, char* bugname);
+switch_bool_t aws_transcribe_frame(switch_media_bug_t *bug, void* user_data);
+
+#endif

mod_aws_transcribe_ws/mod_aws_transcribe_ws.c

@@ -0,0 +1,212 @@
+/* 
+ *
+ * mod_aws_transcribe.c -- Freeswitch module for using aws streaming transcribe api
+ *
+ */
+#include "mod_aws_transcribe_ws.h"
+#include "aws_transcribe_glue.h"
+
+/* Prototypes */
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_aws_transcribe_ws_shutdown);
+SWITCH_MODULE_LOAD_FUNCTION(mod_aws_transcribe_ws_load);
+
+SWITCH_MODULE_DEFINITION(mod_aws_transcribe_ws, mod_aws_transcribe_ws_load, mod_aws_transcribe_ws_shutdown, NULL);
+
+static switch_status_t do_stop(switch_core_session_t *session, char* bugname);
+
+static void responseHandler(switch_core_session_t* session, 
+	const char* eventName, const char * json, const char* bugname, int finished) {
+	switch_event_t *event;
+	switch_channel_t *channel = switch_core_session_get_channel(session);
+
+	switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, eventName);
+	switch_channel_event_set_data(channel, event);
+	switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "transcription-vendor", "deepgram");
+	switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "transcription-session-finished", finished ? "true" : "false");
+	if (finished) {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "responseHandler returning event %s, from finished recognition session\n", eventName);
+	}
+	if (json) switch_event_add_body(event, "%s", json);
+	if (bugname) switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "media-bugname", bugname);
+	switch_event_fire(&event);
+}
+
+
+static switch_bool_t capture_callback(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type)
+{
+	switch_core_session_t *session = switch_core_media_bug_get_session(bug);
+
+	switch (type) {
+	case SWITCH_ABC_TYPE_INIT:
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got SWITCH_ABC_TYPE_INIT.\n");
+		break;
+
+	case SWITCH_ABC_TYPE_CLOSE:
+		{
+			private_t *tech_pvt = (private_t*) switch_core_media_bug_get_user_data(bug);
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got SWITCH_ABC_TYPE_CLOSE.\n");
+			aws_transcribe_session_stop(session, 1, tech_pvt->bugname);
+			//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Finished SWITCH_ABC_TYPE_CLOSE.\n");
+		}
+		break;
+	
+	case SWITCH_ABC_TYPE_READ:
+
+		return aws_transcribe_frame(bug, user_data);
+		break;
+
+	case SWITCH_ABC_TYPE_WRITE:
+	default:
+		break;
+	}
+
+	return SWITCH_TRUE;
+}
+
+static switch_status_t start_capture(switch_core_session_t *session, switch_media_bug_flag_t flags, 
+  char* lang, int interim, char* bugname)
+{
+	switch_channel_t *channel = switch_core_session_get_channel(session);
+	switch_media_bug_t *bug;
+	switch_status_t status;
+	switch_codec_implementation_t read_impl = { 0 };
+	void *pUserData;
+	uint32_t samples_per_second;
+
+	if (switch_channel_get_private(channel, bugname)) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing bug from previous transcribe\n");
+		do_stop(session, bugname);
+	}
+
+	switch_core_session_get_read_impl(session, &read_impl);
+
+	if (switch_channel_pre_answer(channel) != SWITCH_STATUS_SUCCESS) {
+		return SWITCH_STATUS_FALSE;
+	}
+
+	samples_per_second = !strcasecmp(read_impl.iananame, "g722") ? read_impl.actual_samples_per_second : read_impl.samples_per_second;
+
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, " initializing aws speech session.\n");
+	if (SWITCH_STATUS_FALSE == aws_transcribe_session_init(session, responseHandler, samples_per_second, 
+    flags & SMBF_STEREO ? 2 : 1, lang, interim, bugname, &pUserData)) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing aws speech session.\n");
+		return SWITCH_STATUS_FALSE;
+	}
+	if ((status = switch_core_media_bug_add(session, bugname, NULL, capture_callback, pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
+		return status;
+	}
+  switch_channel_set_private(channel, bugname, bug);
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "added media bug for aws transcribe\n");
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+static switch_status_t do_stop(switch_core_session_t *session, char* bugname)
+{
+	switch_status_t status = SWITCH_STATUS_SUCCESS;
+
+	switch_channel_t *channel = switch_core_session_get_channel(session);
+	switch_media_bug_t *bug = switch_channel_get_private(channel, bugname);
+
+	if (bug) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Received user command command to stop transcribe on %s.\n", bugname);
+		status = aws_transcribe_session_stop(session, 0, bugname);
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "stopped transcribe.\n");
+	}
+
+	return status;
+}
+
+#define TRANSCRIBE_API_SYNTAX "<uuid> [start|stop] lang-code [interim] [stereo|mono] [bugname]"
+SWITCH_STANDARD_API(aws_transcribe_function)
+{
+	char *mycmd = NULL, *argv[6] = { 0 };
+	int argc = 0;
+	switch_status_t status = SWITCH_STATUS_FALSE;
+	switch_media_bug_flag_t flags = SMBF_READ_STREAM /* | SMBF_WRITE_STREAM | SMBF_READ_PING */;
+
+	if (!zstr(cmd) && (mycmd = strdup(cmd))) {
+		argc = switch_separate_string(mycmd, ' ', argv, (sizeof(argv) / sizeof(argv[0])));
+	}
+
+	if (zstr(cmd) || 
+      (!strcasecmp(argv[1], "stop") && argc < 2) ||
+      (!strcasecmp(argv[1], "start") && argc < 3) ||
+      zstr(argv[0])) {
+		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error with command %s %s %s.\n", cmd, argv[0], argv[1]);
+		stream->write_function(stream, "-USAGE: %s\n", TRANSCRIBE_API_SYNTAX);
+		goto done;
+	} else {
+		switch_core_session_t *lsession = NULL;
+
+		if ((lsession = switch_core_session_locate(argv[0]))) {
+			if (!strcasecmp(argv[1], "stop")) {
+				char *bugname = argc > 2 ? argv[2] : MY_BUG_NAME;
+    		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "stop transcribing\n");
+				status = do_stop(lsession, bugname);
+			} else if (!strcasecmp(argv[1], "start")) {
+        char* lang = argv[2];
+        int interim = argc > 3 && !strcmp(argv[3], "interim");
+				char *bugname = argc > 5 ? argv[5] : MY_BUG_NAME;
+				if (argc > 4 && !strcmp(argv[4], "stereo")) {
+          flags |= SMBF_WRITE_STREAM ;
+          flags |= SMBF_STEREO;
+				}
+    		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "start transcribing %s %s %s\n", lang, interim ? "interim": "complete", bugname);
+				status = start_capture(lsession, flags, lang, interim, bugname);
+			}
+			switch_core_session_rwunlock(lsession);
+		}
+	}
+
+	if (status == SWITCH_STATUS_SUCCESS) {
+		stream->write_function(stream, "+OK Success\n");
+	} else {
+		stream->write_function(stream, "-ERR Operation Failed\n");
+	}
+
+  done:
+
+	switch_safe_free(mycmd);
+	return SWITCH_STATUS_SUCCESS;
+}
+
+
+SWITCH_MODULE_LOAD_FUNCTION(mod_aws_transcribe_ws_load)
+{
+	switch_api_interface_t *api_interface;
+
+	/* create/register custom event message type */
+	if (switch_event_reserve_subclass(TRANSCRIBE_EVENT_RESULTS) != SWITCH_STATUS_SUCCESS) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't register subclass %s!\n", TRANSCRIBE_EVENT_RESULTS);
+		return SWITCH_STATUS_TERM;
+	}
+
+	/* connect my internal structure to the blank pointer passed to me */
+	*module_interface = switch_loadable_module_create_module_interface(pool, modname);
+
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "AWS Speech Transcription API loading..\n");
+
+  if (SWITCH_STATUS_FALSE == aws_transcribe_init()) {
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Failed initializing aws speech interface\n");
+	}
+
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "AWS Speech Transcription API successfully loaded\n");
+
+	SWITCH_ADD_API(api_interface, "uuid_aws_transcribe", "AWS Speech Transcription API", aws_transcribe_function, TRANSCRIBE_API_SYNTAX);
+	switch_console_set_complete("add uuid_aws_transcribe start lang-code [interim|final] [stereo|mono]");
+	switch_console_set_complete("add uuid_aws_transcribe stop ");
+
+	/* indicate that the module should continue to be loaded */
+	return SWITCH_STATUS_SUCCESS;
+}
+
+/*
+  Called when the system shuts down
+  Macro expands to: switch_status_t mod_aws_transcribe_ws_shutdown() */
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_aws_transcribe_ws_shutdown)
+{
+	aws_transcribe_cleanup();
+	switch_event_free_subclass(TRANSCRIBE_EVENT_RESULTS);
+	return SWITCH_STATUS_SUCCESS;
+}

mod_aws_transcribe_ws/mod_aws_transcribe_ws.h

@@ -0,0 +1,60 @@
+#ifndef __MOD_AWS_TRANSCRIBE_WS_H__
+#define __MOD_AWS_TRANSCRIBE_WS_H__
+
+#include <switch.h>
+#include <speex/speex_resampler.h>
+
+#include <unistd.h>
+
+#define MY_BUG_NAME "aws_transcribe_ws"
+#define MAX_BUG_LEN (64)
+#define MAX_SESSION_ID (256)
+#define TRANSCRIBE_EVENT_RESULTS "aws_transcribe::transcription"
+#define TRANSCRIBE_EVENT_END_OF_TRANSCRIPT "aws_transcribe::end_of_transcript"
+#define TRANSCRIBE_EVENT_NO_AUDIO_DETECTED "aws_transcribe::no_audio_detected"
+#define TRANSCRIBE_EVENT_MAX_DURATION_EXCEEDED "aws_transcribe::max_duration_exceeded"
+#define TRANSCRIBE_EVENT_VAD_DETECTED "aws_transcribe::vad_detected"
+#define TRANSCRIBE_EVENT_CONNECT_SUCCESS "aws::connect"
+#define TRANSCRIBE_EVENT_CONNECT_FAIL    "aws::connect_failed"
+#define TRANSCRIBE_EVENT_DISCONNECT      "aws::disconnect"
+#define TRANSCRIBE_EVENT_BUFFER_OVERRUN  "aws::buffer_overrun"
+#define TRANSCRIBE_EVENT_ERROR      "jambonz_transcribe::error"
+
+#define MAX_LANG (12)
+#define MAX_REGION (32)
+#define MAX_WS_URL_LEN (512)
+#define MAX_PATH_LEN (4096)
+#define MAX_SESSION_TOKEN_LEN (4096)
+
+/* per-channel data */
+typedef void (*responseHandler_t)(switch_core_session_t* session, const char* eventName, const char* json, const char* bugname, int finished);
+
+struct private_data {
+	switch_mutex_t *mutex;
+	char sessionId[MAX_SESSION_ID+1];
+  char awsAccessKeyId[128];
+  char awsSecretAccessKey[128];
+  char awsSessionToken[MAX_SESSION_TOKEN_LEN+1];
+  SpeexResamplerState *resampler;
+	responseHandler_t responseHandler;
+	int interim;
+	char lang[MAX_LANG+1];
+	char region[MAX_REGION+1];
+	switch_vad_t * vad;
+	uint32_t samples_per_second;
+  void *pAudioPipe;
+  int ws_state;
+  char host[MAX_WS_URL_LEN+1];
+  unsigned int port;
+  char path[MAX_PATH_LEN+1];
+  char bugname[MAX_BUG_LEN+1];
+  int sampling;
+  int  channels;
+  unsigned int id;
+  int buffer_overrun_notified:1;
+  int is_finished:1;
+};
+
+typedef struct private_data private_t;
+
+#endif

mod_aws_transcribe_ws/simple_buffer.h

@@ -0,0 +1,51 @@
+/**
+ * (very) simple and limited circular buffer, 
+ * supporting only the use case of doing all of the adds
+ * and then subsquently retrieves.
+ * 
+ */
+class SimpleBuffer {
+  public:
+    SimpleBuffer(uint32_t chunkSize, uint32_t numChunks) : numItems(0),
+    m_numChunks(numChunks), m_chunkSize(chunkSize) {
+      m_pData = new char[chunkSize * numChunks];
+      m_pNextWrite = m_pData;
+    }
+    ~SimpleBuffer() {
+      delete [] m_pData;
+    }
+
+    void add(void *data, uint32_t datalen) {
+      if (datalen % m_chunkSize != 0) return;
+      int numChunks = datalen / m_chunkSize;
+      for (int i = 0; i < numChunks; i++) {
+        memcpy(m_pNextWrite, data, m_chunkSize);
+        data = static_cast<char*>(data) + m_chunkSize;
+        if (numItems < m_numChunks) numItems++;
+
+        uint32_t offset = (m_pNextWrite - m_pData) / m_chunkSize;
+        if (offset >= m_numChunks - 1) m_pNextWrite = m_pData;
+        else m_pNextWrite += m_chunkSize;
+      }
+    }
+
+    char* getNextChunk() {
+      if (numItems--) {
+        char *p = m_pNextWrite;
+        uint32_t offset = (m_pNextWrite - m_pData) / m_chunkSize;
+        if (offset >= m_numChunks - 1) m_pNextWrite = m_pData;
+        else m_pNextWrite += m_chunkSize;
+        return p;
+      }
+      return nullptr;
+    }
+
+    uint32_t getNumItems() { return numItems;}
+
+  private:
+    char *m_pData;
+    uint32_t numItems;
+    uint32_t m_chunkSize;
+    uint32_t m_numChunks;
+    char* m_pNextWrite;
+};

mod_aws_transcribe_ws/transcribe_manager.cpp

@@ -0,0 +1,324 @@
+#include "transcribe_manager.hpp"
+#include "crc.h"
+
+#include <switch.h>
+
+#include <openssl/sha.h>
+#include <openssl/hmac.h>
+#include <iomanip>
+#include <regex>
+#include <iostream>
+#include <cstring>
+#include <netinet/in.h>
+
+using namespace std;
+
+namespace {
+  std::string uri_encode(const std::string &value) {
+    std::string encoded;
+    char hex[4];
+    for (char c : value) {
+        if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') {
+            encoded += c;
+        } else {
+            sprintf(hex, "%%%02X", c);
+            encoded.append(hex);
+        }
+    }
+    return encoded;
+  }
+}
+// see
+// https://docs.aws.amazon.com/transcribe/latest/dg/websocket.html#websocket-url
+// https://docs.aws.amazon.com/transcribe/latest/dg/event-stream.html
+
+void TranscribeManager::getSignedWebsocketUrl(string& host, string& path, const string& accessKey,
+        const string& secretKey, const string& securityToken, const string& region, const std::string& lang, 
+        const char* vocabularyName, const char* vocabularyFilterName, const char* vocabularyFilterMethod,
+        const char* piiEntities, int shouldIdentifyPiiEntities, const char* languageModelName) {
+    string method = "GET";
+    string service = "transcribe";
+    string endpoint = "wss://transcribestreaming." + region + ".amazonaws.com";
+    host = "transcribestreaming." + region + ".amazonaws.com";
+
+    time_t now = time(0);
+    tm *gmtm = gmtime(&now);
+
+    char amzDate[21];
+    snprintf (amzDate, 21, "%04d%02d%02dT%02d%02d%02dZ",
+            1900 + gmtm->tm_year, 1 + gmtm->tm_mon, gmtm->tm_mday,
+            gmtm->tm_hour, gmtm->tm_min, gmtm->tm_sec);
+
+    char datestamp[9];
+    snprintf (datestamp, 9, "%04d%02d%02d", 1900 + gmtm->tm_year, 1 + gmtm->tm_mon, gmtm->tm_mday);
+
+    string canonical_uri = "/stream-transcription-websocket";
+    string canonical_headers = "host:" + host + "\n";
+    string signed_headers = "host";
+    string algorithm = "AWS4-HMAC-SHA256";
+    string credential_scope = string(datestamp) + "%2F" + region + "%2F" + service + "%2F" + "aws4_request";
+
+    // N.B.: The order of all of these query args are important!
+    // Otherwise, the signature will be invalid.
+    string canonical_querystring  = "X-Amz-Algorithm=" + algorithm;
+    canonical_querystring += "&X-Amz-Credential=" + accessKey + "%2F" + credential_scope;
+    canonical_querystring += "&X-Amz-Date=" + string(amzDate);
+    canonical_querystring += "&X-Amz-Expires=300";
+    canonical_querystring += "&X-Amz-Security-Token=" + uri_encode(securityToken);
+    canonical_querystring += "&X-Amz-SignedHeaders=" + signed_headers;
+
+    if (piiEntities && shouldIdentifyPiiEntities) {
+      canonical_querystring += "&content-redaction-type=PII";
+    }
+    
+    canonical_querystring += "&language-code=" + lang;
+    
+    if (languageModelName) {
+      std::string str(languageModelName);
+      canonical_querystring += "&language-model-name=" + uri_encode(str);
+    }
+
+    canonical_querystring += "&media-encoding=pcm";
+
+    if (piiEntities) {
+      std::string str(piiEntities);
+      canonical_querystring += "&pii-entitytypes=" + uri_encode(str);
+    }
+
+    canonical_querystring += "&sample-rate=8000";
+
+    // custom vocabulary and filter
+    if (vocabularyFilterMethod) {
+      std::string str(vocabularyFilterMethod);
+      canonical_querystring += "&vocabulary-filter-method=" + str;
+    }
+    if (vocabularyFilterName) {
+      std::string str(vocabularyFilterName);
+      canonical_querystring += "&vocabulary-filter-name=" + str;
+    }
+    if (vocabularyName) {
+      std::string str(vocabularyName);
+      canonical_querystring += "&vocabulary-name=" + str;
+    }
+
+    string payload_hash = getSha256("");
+
+    string canonical_request = method + '\n'
+       + canonical_uri + '\n'
+       + canonical_querystring + '\n'
+       + canonical_headers + '\n'
+       + signed_headers + '\n'
+       + payload_hash;
+
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,"TranscribeManager::getSignedWebsocketUrl canonical_request: %s\n", canonical_request.c_str());
+    string string_to_sign = algorithm + "\n"
+       + amzDate + "\n"
+       + regex_replace(credential_scope, regex("%2F"), "/") + "\n"
+       + getSha256(canonical_request);
+
+    unsigned char signing_key[SHA256_DIGEST_LENGTH];
+    getSignatureKey(signing_key, secretKey, datestamp, region, service);
+
+    unsigned char signatureBinary[SHA256_DIGEST_LENGTH];
+    getHMAC(signatureBinary, signing_key, SHA256_DIGEST_LENGTH, string_to_sign);
+    string signature = toHex(signatureBinary);
+
+    canonical_querystring += "&X-Amz-Signature=" + signature;
+    string request_url = endpoint + canonical_uri + "?" + canonical_querystring;
+    path = canonical_uri + "?" + canonical_querystring;
+
+    return;
+}
+
+string TranscribeManager::getSha256(string str) {
+    SHA256_CTX ctx;
+    SHA256_Init(&ctx);
+    SHA256_Update(&ctx, str.c_str(), str.length());
+    unsigned char hash[SHA256_DIGEST_LENGTH] = { 0 };
+    SHA256_Final(hash, &ctx);
+
+    ostringstream os;
+    os << hex << setfill('0');
+    for (int i = 0; i < SHA256_DIGEST_LENGTH; ++i) {
+        os << setw(2) << static_cast<unsigned int>(hash[i]);
+    }
+
+    return os.str();
+}
+
+void TranscribeManager::getSignatureKey(unsigned char *signatureKey, const string& secretKey,
+            const string& datestamp, const string& region, const string& service) {
+    string key = string("AWS4") + secretKey;
+    unsigned char kDate[SHA256_DIGEST_LENGTH];
+    unsigned char kRegion[SHA256_DIGEST_LENGTH];
+    unsigned char kService[SHA256_DIGEST_LENGTH];
+    unsigned char kSigning[SHA256_DIGEST_LENGTH];
+    getHMAC(kDate, (unsigned char *)key.c_str(), key.length(), datestamp);
+    getHMAC(kRegion, kDate, SHA256_DIGEST_LENGTH, region);
+    getHMAC(kService, kRegion, SHA256_DIGEST_LENGTH, service);
+    getHMAC(kSigning, kService, SHA256_DIGEST_LENGTH, "aws4_request");
+
+    memcpy(signatureKey, kSigning, SHA256_DIGEST_LENGTH);
+}
+
+void TranscribeManager::getHMAC(unsigned char *hmac, unsigned char *key, int keyLen, const string& str) {
+    unsigned char *data = (unsigned char*)str.c_str();
+    unsigned char *result = HMAC(EVP_sha256(), key, keyLen, data, strlen((char *)data), NULL, NULL);
+    memcpy(hmac, result, SHA256_DIGEST_LENGTH);
+}
+
+string TranscribeManager::toHex(unsigned char *hmac) {
+    ostringstream os;
+    os << hex << setfill('0');
+    for (int i = 0; i < SHA256_DIGEST_LENGTH; ++i) {
+        os << setw(2) << static_cast<unsigned int>(hmac[i]);
+    }
+
+    return os.str();
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+
+bool TranscribeManager::parseResponse(const string& response, string& payload, bool& isError, bool verbose) {
+    const char* buffer = response.c_str();
+
+    uint32_t totalLen;
+    memcpy(&totalLen, &buffer[0], sizeof(uint32_t));
+    totalLen = ntohl(totalLen);
+
+    uint32_t headerLen;
+    memcpy(&headerLen, &buffer[4], sizeof(uint32_t));
+    headerLen = ntohl(headerLen);
+
+    if (!verifyCRC(buffer, totalLen)) {
+        return false;
+    }
+
+    buffer += 12; // bytes 0 - 11 are prelude
+
+    const int numberOfHeaders = 3;
+    for (int i = 0; i < numberOfHeaders; i++) {
+        parseHeader(&buffer, isError, verbose);
+    }
+
+    payload = string(buffer, totalLen - headerLen - 4*4);
+
+    return true;
+}
+
+bool TranscribeManager::verifyCRC(const char* buffer, const uint32_t totalLength) {
+    uint32_t preludeCRC;
+    memcpy(&preludeCRC, &buffer[8], 4);
+    preludeCRC = ntohl(preludeCRC);
+
+    uint32_t calculatedPreludeCRC = CRC::Calculate(&buffer[0], 8, CRC::CRC_32());
+    if (calculatedPreludeCRC != preludeCRC) {
+        cout << "Prelude CRC didn't match!" << endl;
+        return false;
+    }
+
+    uint32_t messageCRC;
+    memcpy(&messageCRC, &buffer[totalLength - 4], 4);
+    messageCRC = ntohl(messageCRC);
+
+    uint32_t calculatedMessageCRC = CRC::Calculate(buffer, totalLength - 4, CRC::CRC_32());
+
+    if (calculatedMessageCRC != messageCRC) {
+        cout << "Message CRC didn't match!" << endl;
+        return false;
+    }
+
+    return true;
+}
+
+void TranscribeManager::parseHeader(const char** buffer, bool& isError, bool verbose) {
+    uint8_t headerNameLen;
+    memcpy(&headerNameLen, *buffer, sizeof(uint8_t));
+    (*buffer)++;
+
+    string headerName(*buffer, headerNameLen);
+    *buffer += headerNameLen;
+
+    uint8_t headerType;
+    memcpy(&headerType, *buffer, sizeof(uint8_t));
+    (*buffer)++;
+
+    uint16_t headerValLen;
+    memcpy(&headerValLen, *buffer, sizeof(uint16_t));
+    headerValLen = ntohs(headerValLen);
+    *buffer += 2;
+
+    string headerVal(*buffer, headerValLen);
+    *buffer += headerValLen;
+
+    if (headerVal == "exception") {
+        isError = true;
+    }
+    if (verbose) {
+        cout << headerName << "(" << (int)headerType << "): " << headerVal << endl;
+    }
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////
+
+bool TranscribeManager::makeRequest(string& request, const vector<uint8_t>& data) {
+    char preludeAndCrcBuffer[4*3];
+    char headerBuffer[88];
+    char messageCrcBuffer[4];
+
+    // prelude
+    uint32_t totalLen = sizeof(preludeAndCrcBuffer) + sizeof(headerBuffer) + data.size() + sizeof(messageCrcBuffer);
+    uint32_t headerLen = sizeof(headerBuffer);
+    totalLen = htonl(totalLen);
+    headerLen = htonl(headerLen);
+
+    memcpy(&preludeAndCrcBuffer[0], &totalLen, sizeof(uint32_t));
+    memcpy(&preludeAndCrcBuffer[4], &headerLen, sizeof(uint32_t));
+
+    uint32_t preludeCRC = CRC::Calculate(&preludeAndCrcBuffer[0], 8, CRC::CRC_32());
+    preludeCRC = htonl(preludeCRC);
+    memcpy(&preludeAndCrcBuffer[8], &preludeCRC, sizeof(uint32_t));
+
+    // header
+    char* buffer = headerBuffer;
+    writeHeader(&buffer, ":content-type", "application/octet-stream");
+    writeHeader(&buffer, ":event-type", "AudioEvent");
+    writeHeader(&buffer, ":message-type", "event");
+
+    // write everything to response string except for the message CRC
+    request.append(preludeAndCrcBuffer, sizeof(preludeAndCrcBuffer));
+    request.append(headerBuffer, sizeof(headerBuffer));
+    request.append(data.begin(), data.end());
+
+    // message CRC
+    uint32_t messageCRC = CRC::Calculate(request.c_str(), request.length(), CRC::CRC_32());
+    messageCRC = htonl(messageCRC);
+    memcpy(messageCrcBuffer, &messageCRC, sizeof(uint32_t));
+
+    // write message CRC to response string
+    request.append(messageCrcBuffer, sizeof(messageCrcBuffer));
+
+    return true;
+}
+
+void TranscribeManager::writeHeader(char** buffer, const char* key, const char* val) {
+    uint8_t keyLen = strlen(key);
+    uint16_t valueLen = strlen(val);
+
+    memcpy(*buffer, &keyLen, sizeof(uint8_t));
+    (*buffer)++;
+
+    memcpy(*buffer, key, keyLen);
+    (*buffer) += keyLen;
+
+    uint8_t valueType = 7;
+    memcpy(*buffer, &valueType, sizeof(uint8_t));
+    (*buffer)++;
+
+    uint16_t valLen = htons(valueLen);
+    memcpy(*buffer, &valLen, sizeof(uint16_t));
+    (*buffer) += 2;
+
+    memcpy(*buffer, val, valueLen);
+    (*buffer) += valueLen;
+}

mod_aws_transcribe_ws/transcribe_manager.hpp

@@ -0,0 +1,50 @@
+#ifndef TRANSCRIBEMANAGER_HPP_
+#define TRANSCRIBEMANAGER_HPP_
+
+#include <string>
+#include <vector>
+
+/** Usage
+ #include "transcribe_manager.hpp"
+
+ // get signed URL
+ const string url = TranscribeManager::getSignedWebsocketUrl(accessKey_, secretKey_, region_);
+
+ // connect to the url using a socket library (e.g. https://github.com/machinezone/IXWebSocket)
+
+ // build request string
+ string request;
+ TranscribeManager::makeRequest(request, audioData); // audioData is a const vector<uint8_t>
+
+ // send request to socket
+ * 
+ */
+
+using namespace std;
+
+class TranscribeManager {
+public:
+    static void getSignedWebsocketUrl(string& host, string& path,
+            const std::string& accessKey, const std::string& secretKey, const std::string& securityToken, 
+            const std::string& region, const std::string& lang, const char* vocabularyName,
+            const char* vocabularyFilterName, const char* vocabularyFilterMethod,
+            const char* piiEntities, int shouldIdentifyPiiEntities, const char* languageModelName);
+
+    static bool parseResponse(const std::string& response, std::string& payload, bool& isError, bool verbose = false);
+
+    static bool makeRequest(std::string& request, const std::vector<uint8_t>& data);
+    static void writeHeader(char** buffer, const char* key, const char* val);
+
+private:
+    static std::string getSha256(std::string str);
+    static void getSignatureKey(unsigned char *signatureKey, const std::string& secretKey,
+            const std::string& datestamp, const std::string& region, const std::string& service);
+    static void getHMAC(unsigned char *hmac, unsigned char *key, int keyLen, const std::string& str);
+    static std::string toHex(unsigned char *hmac);
+
+    static bool verifyCRC(const char* buffer, const uint32_t totalLength);
+    static void parseHeader(const char** buffer, bool& isError, bool verbose = false);
+
+};
+
+#endif /* TRANSCRIBEMANAGER_HPP_ */

mod_azure_transcribe/azure_transcribe_glue.cpp

@@ -32,11 +32,12 @@ static const char* proxyIP = std::getenv("JAMBONES_HTTP_PROXY_IP");
 static const char* proxyPort = std::getenv("JAMBONES_HTTP_PROXY_PORT");
 static const char* proxyUsername = std::getenv("JAMBONES_HTTP_PROXY_USERNAME");
 static const char* proxyPassword = std::getenv("JAMBONES_HTTP_PROXY_PASSWORD");
+static const bool use_single_connection = switch_true(std::getenv("AZURE_SPEECH_USE_SINGLE_CONNECTION"));
 
 class GStreamer {
 public:
 	GStreamer(
-    const char *sessionId,
+		const char *sessionId,
 		const char *bugname,
 		u_int16_t channels,
     char *lang, 
@@ -71,10 +72,7 @@ public:
 		if (switch_true(switch_channel_get_variable(channel, "AZURE_USE_OUTPUT_FORMAT_DETAILED"))) {
 			speechConfig->SetOutputFormat(OutputFormat::Detailed);
 		}
-		if (nullptr != endpointId) {
-      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(psession), SWITCH_LOG_DEBUG, "setting endpoint id: %s\n", endpointId);
-			speechConfig->SetEndpointId(endpointId);
-		}
+
 		if (!sdkInitialized && sdkLog) {
 			sdkInitialized = true;
 			speechConfig->SetProperty(PropertyId::Speech_LogFilename, sdkLog);
@@ -103,11 +101,31 @@ public:
 				languages.push_back( alt_langs[i]);
         switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(psession), SWITCH_LOG_DEBUG, "added alternative lang %s\n", alt_langs[i]);
       }
-			auto autoDetectSourceLanguageConfig = AutoDetectSourceLanguageConfig::FromLanguages(languages);
+
+			std::vector<std::shared_ptr<SourceLanguageConfig>> sourceLanguageConfigs;
+			for (const auto& language : languages) {
+					std::shared_ptr<SourceLanguageConfig> sourceLanguageConfig;
+					
+					if (endpointId != nullptr) {
+							sourceLanguageConfig = SourceLanguageConfig::FromLanguage(language, endpointId);
+					} else {
+							sourceLanguageConfig = SourceLanguageConfig::FromLanguage(language);
+					}
+
+					sourceLanguageConfigs.push_back(sourceLanguageConfig);
+			}
+
+			// Create AutoDetectSourceLanguageConfig from SourceLanguageConfigs
+			auto autoDetectSourceLanguageConfig = AutoDetectSourceLanguageConfig::FromSourceLanguageConfigs(sourceLanguageConfigs);
 			m_recognizer = SpeechRecognizer::FromConfig(speechConfig, autoDetectSourceLanguageConfig, audioConfig);
     }
 		else {
-			auto sourceLanguageConfig = SourceLanguageConfig::FromLanguage(lang);
+			std::shared_ptr<SourceLanguageConfig> sourceLanguageConfig;
+			if (endpointId != nullptr) {
+					sourceLanguageConfig = SourceLanguageConfig::FromLanguage(lang, endpointId);
+			} else {
+					sourceLanguageConfig = SourceLanguageConfig::FromLanguage(lang);
+			}
 			m_recognizer = SpeechRecognizer::FromConfig(speechConfig, sourceLanguageConfig, audioConfig);
 		}
 
@@ -141,7 +159,13 @@ public:
 			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(psession), SWITCH_LOG_DEBUG, "setting SpeechServiceConnection_LanguageIdMode to %s \n", languageIdMode);
 			properties.SetProperty(PropertyId::SpeechServiceConnection_LanguageIdMode, languageIdMode);
 		}
-
+		//https://learn.microsoft.com/en-us/javascript/api/microsoft-cognitiveservices-speech-sdk/propertyid?view=azure-node-latest
+		//PropertyId::SpeechServiceResponse_PostProcessingOption
+ 		const char* postProcessingOption = switch_channel_get_variable(channel, "AZURE_POST_PROCESSING_OPTION");
+		if (postProcessingOption) {
+			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(psession), SWITCH_LOG_DEBUG, "setting SpeechServiceResponse_PostProcessingOption to %s \n", postProcessingOption);
+			properties.SetProperty(PropertyId::SpeechServiceResponse_PostProcessingOption, postProcessingOption);
+		}
 		// recognition mode - readonly according to Azure docs: 
 		// https://docs.microsoft.com/en-us/javascript/api/microsoft-cognitiveservices-speech-sdk/propertyid?view=azure-node-latest
 		/*
@@ -246,6 +270,9 @@ public:
 		m_recognizer->Recognized += onRecognitionEvent;
 		m_recognizer->Canceled += onCanceled;
 
+		// Store the final configuration string
+    m_configuration_string = createConfigurationStr(channels, lang, interim, samples_per_second, region, subscriptionKey, psession);
+
 		switch_core_session_rwunlock(psession);
 	}
 
@@ -253,6 +280,10 @@ public:
 		//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer::~GStreamer %p\n", this);		
 	}
 
+	const char* configuration() {
+		return m_configuration_string.c_str();
+	}
+
 	void connect() {
 		if (m_connecting) return;
 		m_connecting = true;
@@ -318,13 +349,33 @@ public:
     return m_connecting;
   }
 
+	bool hasConfigurationChanged(
+		u_int16_t channels,
+		char *lang, 
+		int interim,
+		uint32_t samples_per_second,
+		const char* region, 
+		const char* subscriptionKey) {
+		switch_core_session_t* psession = switch_core_session_locate(m_sessionId.c_str());
+		if (!psession) throw std::invalid_argument( "session id no longer active" );
+
+		std::string newConfiguration = createConfigurationStr(channels, lang, interim, samples_per_second, region, subscriptionKey, psession);
+
+		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(psession), SWITCH_LOG_DEBUG,
+			"hasConfigurationChanged: old configurattion: %s, new configuration: %s\n", configuration(),  newConfiguration.c_str());
+
+		switch_core_session_rwunlock(psession);
+
+		return strcmp(newConfiguration.c_str(), configuration());
+	}
+
 private:
 	std::string m_sessionId;
 	std::string m_bugname;
 	std::string  m_region;
 	std::shared_ptr<SpeechRecognizer> m_recognizer;
 	std::shared_ptr<PushAudioInputStream> m_pushStream;
-
+  std::string m_configuration_string;
 	responseHandler_t m_responseHandler;
 	bool m_interim;
 	bool m_finished;
@@ -332,6 +383,64 @@ private:
 	bool m_connecting;
 	bool m_stopped;
 	SimpleBuffer m_audioBuffer;
+
+	std::string createConfigurationStr(
+		u_int16_t channels,
+		char *lang, 
+		int interim,
+		uint32_t samples_per_second,
+		const char* region, 
+		const char* subscriptionKey,
+		switch_core_session_t* psession
+	) {
+		switch_channel_t *channel = switch_core_session_get_channel(psession);
+		std::ostringstream configuration_stream;
+		configuration_stream << 
+			channels << ";" <<
+			lang << ";" <<
+			interim << ";" <<
+			samples_per_second << ";" <<
+			region << ";" <<
+			subscriptionKey << ";";
+
+		const char* endpoint = switch_channel_get_variable(channel, "AZURE_SERVICE_ENDPOINT");
+		const char* endpointId = switch_channel_get_variable(channel, "AZURE_SERVICE_ENDPOINT_ID");
+		configuration_stream << (endpoint ? endpoint : "") << ";"
+			<< (endpointId ? endpointId : "") << ";";
+		if (switch_true(switch_channel_get_variable(channel, "AZURE_USE_OUTPUT_FORMAT_DETAILED"))) {
+			configuration_stream << "output_format_detailed;";
+		}
+		if (switch_true(switch_channel_get_variable(channel, "AZURE_AUDIO_LOGGING"))) {
+			configuration_stream << "audio_logging;";
+		}
+		configuration_stream << (proxyIP ? proxyIP : "") << ";"
+			<< (proxyPort ? proxyPort : "") << ";"
+			<< (proxyUsername ? proxyUsername : "") << ";"
+			<< (proxyPassword ? proxyPassword : "") << ";";
+		const char* var;
+		if (var = switch_channel_get_variable(channel, "AZURE_SPEECH_ALTERNATIVE_LANGUAGE_CODES")) {
+			configuration_stream << var << ";";
+		}
+		if (var = switch_channel_get_variable(channel, "AZURE_PROFANITY_OPTION")) {
+			configuration_stream << var << ";";
+		}
+		if (var = switch_channel_get_variable(channel, "AZURE_REQUEST_SNR")) {
+			configuration_stream << var << ";";
+		}
+		if (var = switch_channel_get_variable(channel, "AZURE_INITIAL_SPEECH_TIMEOUT_MS")) {
+			configuration_stream << var << ";";
+		}
+		if (var = switch_channel_get_variable(channel, "AZURE_SPEECH_SEGMENTATION_SILENCE_TIMEOUT_MS")) {
+			configuration_stream << var << ";";
+		}
+		if (var = switch_channel_get_variable(channel, "AZURE_LANGUAGE_ID_MODE")) {
+			configuration_stream << var << ";";
+		}
+		if (var = switch_channel_get_variable(channel, "AZURE_SPEECH_HINTS")) {
+			configuration_stream << var << ";";
+		}
+		return configuration_stream.str();
+	}
 };
 
 static void reaper(struct cap_cb *cb) {
@@ -388,16 +497,40 @@ extern "C" {
 		GStreamer *streamer = NULL;
 		switch_status_t status = SWITCH_STATUS_SUCCESS;
 		switch_channel_t *channel = switch_core_session_get_channel(session);
+		switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, bugname);
+		const char* subscriptionKey = switch_channel_get_variable(channel, "AZURE_SUBSCRIPTION_KEY");
+		const char* region = switch_channel_get_variable(channel, "AZURE_REGION");
+		const char* sessionId = switch_core_session_get_uuid(session);
+		auto read_codec = switch_core_session_get_read_codec(session);
+		uint32_t sampleRate = read_codec->implementation->actual_samples_per_second;
+		if (bug) {
+			try {
+				struct cap_cb* existing_cb = (struct cap_cb*) switch_core_media_bug_get_user_data(bug);
+				GStreamer* existing_streamer = (GStreamer*) existing_cb->streamer;
+				existing_cb->is_keep_alive = 0;
+				if (!existing_streamer->hasConfigurationChanged(channels, lang, interim, sampleRate, region, subscriptionKey)) {
+					switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Reuse active azure connection.\n");
+					return SWITCH_STATUS_SUCCESS;
+				}
+				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Azure configuration is changed, destroy old and create new azure connection\n");
+				reaper(existing_cb);
+				streamer =  new GStreamer(sessionId, bugname, channels, lang, interim, sampleRate, region, subscriptionKey, responseHandler);
+				if (!existing_cb->vad) streamer->connect();
+				existing_cb->streamer = streamer;
+				*ppUserData = existing_cb;
+				return SWITCH_STATUS_SUCCESS;
+			} catch (std::exception& e) {
+				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing gstreamer: %s.\n", 
+					switch_channel_get_name(channel), e.what());
+				return SWITCH_STATUS_FALSE;
+			}
+			
+		}
 		int err;
 		switch_threadattr_t *thd_attr = NULL;
 		switch_memory_pool_t *pool = switch_core_session_get_pool(session);
-		auto read_codec = switch_core_session_get_read_codec(session);
-		uint32_t sampleRate = read_codec->implementation->actual_samples_per_second;
-		const char* sessionId = switch_core_session_get_uuid(session);
 		struct cap_cb* cb = (struct cap_cb *) switch_core_session_alloc(session, sizeof(*cb));
 		memset(cb, sizeof(cb), 0);
-		const char* subscriptionKey = switch_channel_get_variable(channel, "AZURE_SUBSCRIPTION_KEY");
-		const char* region = switch_channel_get_variable(channel, "AZURE_REGION");
 		cb->channels = channels;
 		strncpy(cb->sessionId, sessionId, MAX_SESSION_ID);
 		strncpy(cb->bugname, bugname, MAX_BUG_LEN);
@@ -418,15 +551,27 @@ extern "C" {
 
 		cb->responseHandler = responseHandler;
 
+		cb->interim = interim;
+		strncpy(cb->lang, lang, MAX_LANG);
+
+		try {
+			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "%s: initializing gstreamer with %s\n", 
+					switch_channel_get_name(channel), bugname);
+			streamer = new GStreamer(sessionId, bugname, channels, lang, interim, sampleRate, cb->region, subscriptionKey, responseHandler);
+			cb->streamer = streamer;
+			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "azure_transcribe_session_init: config: %s\n", streamer->configuration());
+		} catch (std::exception& e) {
+			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing gstreamer: %s.\n", 
+				switch_channel_get_name(channel), e.what());
+			return SWITCH_STATUS_FALSE;
+		}
+
 		if (switch_mutex_init(&cb->mutex, SWITCH_MUTEX_NESTED, pool) != SWITCH_STATUS_SUCCESS) {
 			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error initializing mutex\n");
 			status = SWITCH_STATUS_FALSE;
 			goto done; 
 		}
 
-		cb->interim = interim;
-		strncpy(cb->lang, lang, MAX_LANG);
-
 		/* determine if we need to resample the audio to 16-bit 8khz */
 		if (sampleRate != 8000) {
 			cb->resampler = speex_resampler_init(1, sampleRate, 8000, SWITCH_RESAMPLE_QUALITY, &err);
@@ -468,23 +613,10 @@ extern "C" {
 					switch_channel_get_name(channel), voice_ms, mode);
 			}
 		}
-
-		try {
-			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "%s: initializing gstreamer with %s\n", 
-					switch_channel_get_name(channel), bugname);
-			streamer = new GStreamer(sessionId, bugname, channels, lang, interim, sampleRate, cb->region, subscriptionKey, responseHandler);
-			cb->streamer = streamer;
-			if (!cb->vad) streamer->connect();
-		} catch (std::exception& e) {
-			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing gstreamer: %s.\n", 
-				switch_channel_get_name(channel), e.what());
-			return SWITCH_STATUS_FALSE;
-		}
-
-
-		*ppUserData = cb;
-	
+		if (!cb->vad) streamer->connect();
 	done:
+		*ppUserData = cb;
+		cb->is_keep_alive = 0;
 		return status;
 	}
 
@@ -496,6 +628,12 @@ extern "C" {
 			struct cap_cb *cb = (struct cap_cb *) switch_core_media_bug_get_user_data(bug);
 			switch_status_t st;
 
+			if (use_single_connection && !channelIsClosing) {
+				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "azure_transcribe_session_stop: call is running, use_single_connection is true, keep alive is activated\n");
+				cb->is_keep_alive = 1;
+				return SWITCH_STATUS_SUCCESS;
+			}
+
 			// close connection and get final responses
 			switch_mutex_lock(cb->mutex);
 			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "azure_transcribe_session_stop: locked session\n");
@@ -507,6 +645,7 @@ extern "C" {
 			if (streamer) reaper(cb);
 			killcb(cb);
 			switch_mutex_unlock(cb->mutex);
+			switch_mutex_destroy(cb->mutex);
 			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "azure_transcribe_session_stop: unlocked session\n");
 
 			return SWITCH_STATUS_SUCCESS;
@@ -524,7 +663,18 @@ extern "C" {
 
 		frame.data = data;
 		frame.buflen = SWITCH_RECOMMENDED_BUFFER_SIZE;
-
+		if (cb->is_keep_alive) {
+			// remove media bug buffered data
+ 			while (true) {
+				unsigned char data[SWITCH_RECOMMENDED_BUFFER_SIZE] = {0};
+				switch_frame_t frame = { 0 };
+				frame.data = data;
+				frame.buflen = SWITCH_RECOMMENDED_BUFFER_SIZE;
+				switch_status_t rv = switch_core_media_bug_read(bug, &frame, SWITCH_TRUE);
+				if (rv != SWITCH_STATUS_SUCCESS) break;
+			}
+			return SWITCH_TRUE;
+		}
 		if (switch_mutex_trylock(cb->mutex) == SWITCH_STATUS_SUCCESS) {
 			GStreamer* streamer = (GStreamer *) cb->streamer;
 			if (streamer) {

mod_azure_transcribe/mod_azure_transcribe.c

@@ -71,9 +71,9 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
 	switch_codec_implementation_t read_impl = { 0 };
 	void *pUserData;
 	uint32_t samples_per_second;
-
-
-	if (switch_channel_get_private(channel, bugname)) {
+	int use_single_connection = switch_true(getenv("AZURE_SPEECH_USE_SINGLE_CONNECTION"));
+	bug = switch_channel_get_private(channel, bugname);
+	if (bug && !use_single_connection) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing bug from previous transcribe\n");
 		do_stop(session, bugname);
 	}
@@ -91,12 +91,13 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing azure speech session.\n");
 		return SWITCH_STATUS_FALSE;
 	}
-	if ((status = switch_core_media_bug_add(session, bugname, NULL, capture_callback, pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
-		return status;
+	if (!bug || !use_single_connection) {
+		if ((status = switch_core_media_bug_add(session, bugname, NULL, capture_callback, pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
+			return status;
+		}
+		switch_channel_set_private(channel, bugname, bug);
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "added media bug for azure transcribe\n");
 	}
-  switch_channel_set_private(channel, bugname, bug);
-	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "added media bug for azure transcribe\n");
-
 	return SWITCH_STATUS_SUCCESS;
 }
 

mod_azure_transcribe/mod_azure_transcribe.h

@@ -37,6 +37,8 @@ struct cap_cb {
 	char lang[MAX_LANG];
 	char region[MAX_REGION];
 
+	int is_keep_alive;
+
 	switch_vad_t * vad;
 };
 

mod_azure_tts/azure_glue.cpp

@@ -15,11 +15,13 @@ typedef boost::circular_buffer<uint16_t> CircularBuffer_t;
 
 using namespace Microsoft::CognitiveServices::Speech;
 
+static const char* audioLogFile= std::getenv("AZURE_AUDIO_LOGGING");
+
 static std::string fullDirPath;
 
 static void start_synthesis(std::shared_ptr<SpeechSynthesizer> speechSynthesizer, const char* text, azure_t* a) {
     try {
-      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "start_synthesis calling \n");
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "start_synthesis calling, text %s\n", text);
       auto result = std::strncmp(text, "<speak", 6) == 0 ?
         speechSynthesizer->SpeakSsmlAsync(text).get() :
         speechSynthesizer->SpeakTextAsync(text).get();
@@ -43,6 +45,8 @@ static void start_synthesis(std::shared_ptr<SpeechSynthesizer> speechSynthesizer
         switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mod_azure_tts: Exception in start_synthesis %s\n",  e.what());
     }
     a->draining = 1;
+
+    free((void*) text);
 }
 
 extern "C" {
@@ -87,14 +91,7 @@ extern "C" {
 
   switch_status_t azure_speech_feed_tts(azure_t* a, char* text, switch_speech_flag_t *flags) {
     const int MAX_CHARS = 20;
-    char tempText[MAX_CHARS + 4]; // +4 for the ellipsis and null terminator
-
-    if (strlen(text) > MAX_CHARS) {
-        strncpy(tempText, text, MAX_CHARS);
-        strcpy(tempText + MAX_CHARS, "...");
-    } else {
-        strcpy(tempText, text);
-    }
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts %s\n", text);
 
     /* open cache file */
     if (a->cache_audio && fullDirPath.length() > 0) {
@@ -168,8 +165,14 @@ extern "C" {
 			speechConfig->SetEndpointId(a->endpointId);
 		}
 
+    if (audioLogFile) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts enabling audio logging to %s\n", audioLogFile);
+      speechConfig->SetProperty(PropertyId::Speech_LogFilename, audioLogFile);
+      speechConfig->EnableAudioLogging();
+    }
+
     try {
-      auto speechSynthesizer = SpeechSynthesizer::FromConfig(speechConfig);
+      auto speechSynthesizer = SpeechSynthesizer::FromConfig(speechConfig, nullptr);
 
       speechSynthesizer->SynthesisStarted += [a](const SpeechSynthesisEventArgs& e) {
           switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts SynthesisStarted\n");
@@ -179,23 +182,55 @@ extern "C" {
         if (a->flushed) return;
         bool fireEvent = false;
         CircularBuffer_t *cBuffer = (CircularBuffer_t *) a->circularBuffer;
+         size_t total_bytes_to_process;
 
         auto audioData = e.Result->GetAudioData();
+        auto bytes_received = audioData->size();
+        // Buffer to hold combined data if there is unprocessed byte from the last call.
+        std::unique_ptr<uint8_t[]> combinedData;
+        if (a->has_last_byte) {
+          a->has_last_byte = false;  // We'll handle the last_byte now, so toggle the flag off
+
+          // Allocate memory for the new data array
+          combinedData.reset(new uint8_t[bytes_received + 1]);
+
+          // Prepend the last byte from previous call
+          combinedData[0] = a->last_byte;
+
+          // Copy the new data following the prepended byte
+          memcpy(combinedData.get() + 1, audioData->data(), bytes_received);
+
+          total_bytes_to_process = bytes_received + 1;
+        } else {
+          // Allocate memory for the new data array
+          combinedData.reset(new uint8_t[bytes_received]);
+          memcpy(combinedData.get(), audioData->data(), bytes_received);
+          total_bytes_to_process = bytes_received;
+        }
+
+        // If we now have an odd total, save the last byte for next time
+        auto data = combinedData.get();
+        if ((total_bytes_to_process % sizeof(int16_t)) != 0) {
+          a->last_byte = data[total_bytes_to_process - 1];
+          a->has_last_byte = true;
+          total_bytes_to_process--;
+        }
+  
         if (a->file) {
-          fwrite(audioData->data(), 1, audioData->size(), a->file);
+          fwrite(data, 1, total_bytes_to_process, a->file);
         }
 
         /**
          * this sort of reinterpretation can be dangerous as a general rule, but in this case we know that the data
          * is 16-bit PCM, so it's safe to do this and its much faster than copying the data byte by byte
          */
-        const uint16_t* begin = reinterpret_cast<const uint16_t*>(audioData->data());
-        const uint16_t* end = reinterpret_cast<const uint16_t*>(audioData->data() + audioData->size());
+        const uint16_t* begin = reinterpret_cast<const uint16_t*>(data);
+        const uint16_t* end = reinterpret_cast<const uint16_t*>(data + total_bytes_to_process);
 
         /* lock as briefly as possible */
         switch_mutex_lock(a->mutex);
-        if (cBuffer->capacity() - cBuffer->size() < audioData->size()) {
-          cBuffer->set_capacity(cBuffer->size() + std::max( audioData->size(), (size_t)BUFFER_SIZE));
+        if (cBuffer->capacity() - cBuffer->size() < total_bytes_to_process) {
+          cBuffer->set_capacity(cBuffer->size() + std::max( total_bytes_to_process, (size_t)BUFFER_SIZE));
         }
         cBuffer->insert(cBuffer->end(), begin, end);
         switch_mutex_unlock(a->mutex);
@@ -223,6 +258,7 @@ extern "C" {
                   switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", a->cache_filename);
                 }
                 switch_event_fire(&event);
+                a->playback_start_sent = 1;
               } else {
                 switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "speechSynthesizer->Synthesizing: failed to create event\n");
               }
@@ -233,7 +269,8 @@ extern "C" {
         }
       };
 
-      std::thread(start_synthesis, speechSynthesizer, text, a).detach();
+      const char* dupText = strdup(text); // text will be freed in the thread
+      std::thread(start_synthesis, speechSynthesizer, dupText, a).detach();
       switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts sent synthesize request\n");
     } catch (const std::exception& e) {
       switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mod_azure_tts: Exception: %s\n", e.what());
@@ -315,16 +352,16 @@ extern "C" {
         }
         a->file = nullptr ;
       }
-
-      if (a->cache_filename) {
-        //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", a->cache_filename);
-        if (unlink(a->cache_filename) != 0) {
-          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
-            a->cache_filename, errno, strerror(errno));
-        }
-        free(a->cache_filename);
-        a->cache_filename = nullptr ;
+    }
+    // If playback_start has not been sent, delete the file
+    if (a->cache_filename && !a->playback_start_sent) {
+      //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", a->cache_filename);
+      if (unlink(a->cache_filename) != 0) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
+          a->cache_filename, errno, strerror(errno));
       }
+      free(a->cache_filename);
+      a->cache_filename = nullptr ;
     }
     if (a->session_id) {
       switch_core_session_t* session = switch_core_session_locate(a->session_id);

mod_azure_tts/mod_azure_tts.c

@@ -82,6 +82,10 @@ static switch_status_t a_speech_feed_tts(switch_speech_handle_t *sh, char *text,
   a->draining = 0;
   a->reads = 0;
   a->flushed = 0;
+  a->response_code = 0;
+  a->err_msg = NULL;
+  a->has_last_byte = 0;
+  a->playback_start_sent = 0;
 
   return azure_speech_feed_tts(a, text, flags);
 }

mod_azure_tts/mod_azure_tts.h

@@ -25,6 +25,7 @@ typedef struct azure_data {
   int reads;
   int cache_audio;
   int flushed;
+  int playback_start_sent;
 
   void *startTime;
 
@@ -32,6 +33,9 @@ typedef struct azure_data {
   SpeexResamplerState *resampler;
   void *circularBuffer;
   switch_mutex_t *mutex;
+
+  int has_last_byte;
+  uint8_t last_byte;
 } azure_t;
 
 #endif

mod_custom_tts/LICENSE

@@ -0,0 +1,8 @@
+Copyright 2023, Drachtio Communications Services, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

mod_custom_tts/Makefile.am

@@ -0,0 +1,8 @@
+include $(top_srcdir)/build/modmake.rulesam
+MODNAME=mod_custom_tts
+
+mod_LTLIBRARIES = mod_custom_tts.la
+mod_custom_tts_la_SOURCES  = mod_custom_tts.c custom_glue.cpp
+mod_custom_tts_la_CFLAGS   = $(AM_CFLAGS)
+mod_custom_tts_la_LIBADD   = $(switch_builddir)/libfreeswitch.la
+mod_custom_tts_la_LDFLAGS  = -avoid-version -module -no-undefined -shared -lstdc++ -lboost_system -lboost_thread -lmpg123

mod_custom_tts/README.md

@@ -0,0 +1,3 @@
+# mod_custom_tts
+
+A Freeswitch module that allows speak text to speech audio from custom vendor stream.

mod_custom_tts/custom_glue.cpp

@@ -0,0 +1,981 @@
+#include "mod_custom_tts.h"
+#include <switch.h>
+#include <switch_json.h>
+#include <curl/curl.h>
+#include <cstdlib>
+
+#include <boost/circular_buffer.hpp>
+#include <boost/thread.hpp>
+#include <boost/asio.hpp>
+#include <boost/asio/ssl.hpp>
+#include <boost/pool/object_pool.hpp>
+#include <boost/bind/bind.hpp>
+#include <boost/tokenizer.hpp>
+#include <boost/foreach.hpp>
+#include <boost/asio.hpp>
+#include <boost/assign/list_of.hpp>
+#include <boost/algorithm/string.hpp>
+
+#include "mpg123.h"
+
+#define BUFFER_GROW_SIZE (8192)
+#define MP3_DCACHE 8192 * 2
+
+typedef boost::circular_buffer<uint16_t> CircularBuffer_t;
+/* Global information, common to all connections */
+typedef struct
+{
+  CURLM *multi;
+  int still_running;
+} GlobalInfo_t;
+static GlobalInfo_t global;
+
+/* Information associated with a specific easy handle */
+typedef struct
+{
+  CURL *easy;
+  custom_t* custom;
+  char* body;
+  struct curl_slist *hdr_list;
+  GlobalInfo_t *global;
+  mpg123_handle *mh;
+  char error[CURL_ERROR_SIZE];
+  FILE* file;
+  std::chrono::time_point<std::chrono::high_resolution_clock> startTime;
+  bool flushed;
+} ConnInfo_t;
+
+
+static boost::object_pool<ConnInfo_t> pool ;
+static std::map<curl_socket_t, boost::asio::ip::tcp::socket *> socket_map;
+static boost::asio::io_service io_service;
+static boost::asio::deadline_timer timer(io_service);
+static std::string fullDirPath;
+static std::thread worker_thread;
+
+std::string secondsToMillisecondsString(double seconds) {
+    // Convert to milliseconds
+    double milliseconds = seconds * 1000.0;
+
+    // Truncate to remove fractional part
+    long milliseconds_long = static_cast<long>(milliseconds);
+
+    // Convert to string
+    return std::to_string(milliseconds_long);
+}
+
+static std::string getEnvVar(const std::string& varName) {
+    const char* val = std::getenv(varName.c_str());
+    return val ? std::string(val) : "";
+}
+
+static long getConnectionTimeout() {
+    std::string connectTimeoutStr = getEnvVar("CUSTOM_TTS_CURL_CONNECT_TIMEOUT");
+
+    if (connectTimeoutStr.empty()) {
+        connectTimeoutStr = getEnvVar("TTS_CURL_CONNECT_TIMEOUT");
+    }
+
+    long connectTimeout = 10L;
+
+    if (!connectTimeoutStr.empty()) {
+        connectTimeout = std::stol(connectTimeoutStr);
+    }
+
+    return connectTimeout;
+}
+
+static CURL* createEasyHandle(void) {
+  CURL* easy = curl_easy_init();
+  if(!easy) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "curl_easy_init() failed!\n");
+    return nullptr ;
+  }  
+
+  curl_easy_setopt(easy, CURLOPT_FOLLOWLOCATION, 1L);
+  curl_easy_setopt(easy, CURLOPT_USERAGENT, "jambonz/0.8.5");
+
+  // set connect timeout to 3 seconds and total timeout to 109 seconds
+  curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
+  curl_easy_setopt(easy, CURLOPT_TIMEOUT, getConnectionTimeout());
+
+  return easy ;    
+}
+
+static void cleanupConn(ConnInfo_t *conn) {
+  auto c = conn->custom;
+
+  if (conn->mh) {
+    mpg123_close(conn->mh);
+    mpg123_delete(conn->mh);
+  }
+
+  if( conn->hdr_list ) {
+    curl_slist_free_all(conn->hdr_list);
+    conn->hdr_list = nullptr ;
+  }
+  curl_easy_cleanup(conn->easy);
+
+  if (conn->file) {
+    if (fclose(conn->file) != 0) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error closing audio cache file\n");
+    }
+    conn->file = nullptr ;
+  }
+
+  c->conn = nullptr ;
+  c->draining = 1;
+
+  memset(conn, 0, sizeof(ConnInfo_t));
+  pool.destroy(conn) ;
+}
+
+/* Check for completed transfers, and remove their easy handles */
+void check_multi_info(GlobalInfo_t *g) {
+  CURLMsg *msg;
+  int msgs_left;
+  ConnInfo_t *conn;
+  CURL *easy;
+  CURLcode res;
+  
+  while((msg = curl_multi_info_read(g->multi, &msgs_left))) {
+    if(msg->msg == CURLMSG_DONE) {
+      long response_code;
+      double namelookup=0, connect=0, total=0 ;
+      char *ct = NULL ;
+
+      easy = msg->easy_handle;
+      res = msg->data.result;
+      curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
+      curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &response_code);
+      curl_easy_getinfo(easy, CURLINFO_CONTENT_TYPE, &ct);
+
+      curl_easy_getinfo(easy, CURLINFO_NAMELOOKUP_TIME, &namelookup);
+      curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connect);
+      curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &total);
+
+      auto c = conn->custom;
+      c->response_code = response_code;
+      if (ct) c->ct = strdup(ct);
+
+      std::string name_lookup_ms = secondsToMillisecondsString(namelookup);
+      std::string connect_ms = secondsToMillisecondsString(connect);
+      std::string final_response_time_ms = secondsToMillisecondsString(total);
+
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, 
+        "mod_custom_tts: response: %ld, content-type %s,"
+        "dns(ms): %"  CURL_FORMAT_CURL_OFF_T ".%06ld, "
+        "connect(ms): %"  CURL_FORMAT_CURL_OFF_T ".%06ld, "
+        "total(ms): %"  CURL_FORMAT_CURL_OFF_T ".%06ld\n",
+        response_code, ct,
+        (long)(namelookup), (long)(fmod(namelookup, 1.0) * 1000000),
+        (long)(connect), (long)(fmod(connect, 1.0) * 1000000),
+        (long)(total), (long)(fmod(total, 1.0) * 1000000));
+
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "name lookup time: %s\n", name_lookup_ms.c_str());
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "connect time: %s\n", connect_ms.c_str());
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "final response time: %s\n", final_response_time_ms.c_str());
+
+      c->name_lookup_time_ms = strdup(name_lookup_ms.c_str());
+      c->connect_time_ms = strdup(connect_ms.c_str());
+      c->final_response_time_ms = strdup(final_response_time_ms.c_str());
+
+      curl_multi_remove_handle(g->multi, easy);
+      cleanupConn(conn);
+    }
+  }
+}
+
+int mcode_test(const char *where, CURLMcode code) {
+  if(CURLM_OK != code) {
+    const char *s;
+    switch(code) {
+    case CURLM_CALL_MULTI_PERFORM:
+      s = "CURLM_CALL_MULTI_PERFORM";
+      break;
+    case CURLM_BAD_HANDLE:
+      s = "CURLM_BAD_HANDLE";
+      break;
+    case CURLM_BAD_EASY_HANDLE:
+      s = "CURLM_BAD_EASY_HANDLE";
+      break;
+    case CURLM_OUT_OF_MEMORY:
+      s = "CURLM_OUT_OF_MEMORY";
+      break;
+    case CURLM_INTERNAL_ERROR:
+      s = "CURLM_INTERNAL_ERROR";
+      break;
+    case CURLM_UNKNOWN_OPTION:
+      s = "CURLM_UNKNOWN_OPTION";
+      break;
+    case CURLM_LAST:
+      s = "CURLM_LAST";
+      break;
+    default:
+      s = "CURLM_unknown";
+      break;
+    case CURLM_BAD_SOCKET:
+      s = "CURLM_BAD_SOCKET";
+      break;
+    }
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mcode_test ERROR: %s returns %s:%d\n", where, s, code);
+
+    return -1;
+  }
+  return 0 ;
+}
+
+static void remsock(int *f, GlobalInfo_t *g) {
+  if(f) {
+    free(f);
+    f = NULL;
+  }
+}
+
+/* Called by asio when there is an action on a socket */
+static void event_cb(GlobalInfo_t *g, curl_socket_t s, int action, const boost::system::error_code & error, int *fdp) {
+  int f = *fdp;
+
+  //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X has action %d\n", s, action) ; 
+
+  // Socket already POOL REMOVED.
+  if (f == CURL_POLL_REMOVE) {
+    //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X removed\n", s); 
+    remsock(fdp, g);
+    return;
+  }
+
+  if(socket_map.find(s) == socket_map.end()) {
+    //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb: socket  %#X already closed\n, s");
+    return;
+  }
+
+  /* make sure the event matches what are wanted */
+  if(f == action || f == CURL_POLL_INOUT) {
+    if(error) {
+      action = CURL_CSELECT_ERR;
+    }
+    CURLMcode rc = curl_multi_socket_action(g->multi, s, action, &g->still_running);
+
+    mcode_test("event_cb: curl_multi_socket_action", rc);
+    check_multi_info(g);
+
+    if(g->still_running <= 0) {
+      timer.cancel();
+    }
+
+    /* keep on watching.
+      * the socket may have been closed and/or fdp may have been changed
+      * in curl_multi_socket_action(), so check them both */
+    if(!error && socket_map.find(s) != socket_map.end() &&
+        (f == action || f == CURL_POLL_INOUT)) {
+      boost::asio::ip::tcp::socket *tcp_socket = socket_map.find(s)->second;
+
+      if(action == CURL_POLL_IN) {
+        tcp_socket->async_read_some(boost::asio::null_buffers(),
+                                    boost::bind(&event_cb, g, s,
+                                                action, boost::placeholders::_1, fdp));
+      }
+      if(action == CURL_POLL_OUT) {
+        tcp_socket->async_write_some(boost::asio::null_buffers(),
+                                      boost::bind(&event_cb, g, s,
+                                                  action, boost::placeholders::_1, fdp));
+      } 
+    }
+  }
+}
+
+/* socket functions */
+static void setsock(int *fdp, curl_socket_t s, CURL *e, int act, int oldact, GlobalInfo_t *g) {
+  std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(s);
+
+  if(it == socket_map.end()) {
+    //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setsock: socket  %#X not found\n, s");
+    return;
+  }
+
+  boost::asio::ip::tcp::socket * tcp_socket = it->second;
+
+  *fdp = act;
+
+  if(act == CURL_POLL_IN) {
+    if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
+      tcp_socket->async_read_some(boost::asio::null_buffers(),
+                                  boost::bind(&event_cb, g, s,
+                                  CURL_POLL_IN, boost::placeholders::_1, fdp));
+    }
+  }
+  else if(act == CURL_POLL_OUT) {
+    if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
+      tcp_socket->async_write_some(boost::asio::null_buffers(),
+                                    boost::bind(&event_cb, g, s,
+                                    CURL_POLL_OUT, boost::placeholders::_1, fdp));
+    }
+  }
+  else if(act == CURL_POLL_INOUT) {
+    if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
+      tcp_socket->async_read_some(boost::asio::null_buffers(),
+                                  boost::bind(&event_cb, g, s,
+                                  CURL_POLL_IN, boost::placeholders::_1, fdp));
+    }
+    if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
+      tcp_socket->async_write_some(boost::asio::null_buffers(),
+                                    boost::bind(&event_cb, g, s,
+                                    CURL_POLL_OUT, boost::placeholders::_1, fdp));
+    }
+  }
+}
+
+static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo_t *g) {
+  /* fdp is used to store current action */
+  int *fdp = (int *) calloc(sizeof(int), 1);
+
+  setsock(fdp, s, easy, action, 0, g);
+  curl_multi_assign(g->multi, s, fdp);
+}
+
+static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) {
+  GlobalInfo_t *g = &global;
+
+  int *actionp = (int *) sockp;
+  static const char *whatstr[] = { "none", "IN", "OUT", "INOUT", "REMOVE"};
+
+  if(what == CURL_POLL_REMOVE) {
+    *actionp = what;
+  }
+  else {
+    if(!actionp) {
+      addsock(s, e, what, g);
+    }
+    else {
+      setsock(actionp, s, e, what, *actionp, g);
+    }
+  }
+  return 0;  
+}
+
+static void threadFunc() {      
+  /* to make sure the event loop doesn't terminate when there is no work to do */
+  io_service.reset() ;
+  boost::asio::io_service::work work(io_service);
+  
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_custom_tts threadFunc - starting\n");
+
+  for(;;) {
+      
+    try {
+      io_service.run() ;
+      break ;
+    }
+    catch( std::exception& e) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mod_custom_tts threadFunc - Error: %s\n", e.what());
+    }
+  }
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_custom_tts threadFunc - ending\n");
+}
+
+
+/* Called by asio when our timeout expires */
+static void timer_cb(const boost::system::error_code & error, GlobalInfo_t *g)
+{
+  //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "timer_cb\n");
+
+  if(!error) {
+    CURLMcode rc = curl_multi_socket_action(g->multi, CURL_SOCKET_TIMEOUT, 0, &g->still_running);
+    mcode_test("timer_cb: curl_multi_socket_action", rc);
+    check_multi_info(g);
+  }
+}
+
+int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo_t *g) {
+
+  /* cancel running timer */
+  timer.cancel();
+
+  if(timeout_ms >= 0) {
+    // from libcurl 7.88.1-10+deb12u4 does not allow call curl_multi_socket_action or curl_multi_perform in curl_multi callback directly
+    timer.expires_from_now(boost::posix_time::millisec(timeout_ms ? timeout_ms : 1));
+    timer.async_wait(boost::bind(&timer_cb, boost::placeholders::_1, g));
+  }
+
+  return 0;
+}
+
+static std::vector<uint16_t> convert_mp3_to_linear(ConnInfo_t *conn, uint8_t *data, size_t len) {
+  std::vector<uint16_t> linear_data;
+  int eof = 0;
+  int mp3err = 0;
+  unsigned char decode_buf[MP3_DCACHE];
+
+  if(mpg123_feed(conn->mh, data, len) == MPG123_OK) {
+    while(!eof) {
+      size_t usedlen = 0;
+      off_t frame_offset;
+      unsigned char* audio;
+
+      int decode_status = mpg123_decode_frame(conn->mh, &frame_offset, &audio, &usedlen);
+
+      switch(decode_status) {
+        case MPG123_NEW_FORMAT:
+          continue;
+
+        case MPG123_OK:
+          for(size_t i = 0; i < usedlen; i += 2) {
+            uint16_t value = reinterpret_cast<uint16_t*>(audio)[i / 2];
+            linear_data.push_back(value);
+          }
+          break;
+
+        case MPG123_DONE:
+        case MPG123_NEED_MORE:
+          eof = 1;
+          break;
+
+        case MPG123_ERR:
+        default:
+          if(++mp3err >= 5) {
+            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Decoder Error!\n");
+            eof = 1;
+          }
+      }
+
+      if (eof)
+        break;
+
+      mp3err = 0;
+    }
+  }
+
+  return linear_data;
+}
+/* CURLOPT_WRITEFUNCTION */
+static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
+  bool fireEvent = false;
+  uint8_t *data = (uint8_t *) ptr;
+  size_t bytes_received = size * nmemb;
+  auto c = conn->custom;
+  CircularBuffer_t *cBuffer = (CircularBuffer_t *) c->circularBuffer;
+  std::vector<uint16_t> pcm_data;
+  
+  if (conn->flushed || cBuffer == nullptr) {
+    /* this will abort the transfer */
+    return 0;
+  }
+  {
+    switch_mutex_lock(c->mutex);
+
+    if (c->response_code > 0 && c->response_code != 200) {
+      std::string body((char *) ptr, bytes_received);
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: received body %s\n", body.c_str());
+      c->err_msg = strdup(body.c_str());
+      switch_mutex_unlock(c->mutex);
+      return 0;
+    }
+
+    pcm_data = convert_mp3_to_linear(conn, data, bytes_received);
+    size_t bytesResampled = pcm_data.size() * sizeof(uint16_t);
+
+    /* cache same data to avoid streaming and cached audio quality is different*/
+    if (conn->file) fwrite(pcm_data.data(), sizeof(uint8_t), bytesResampled, conn->file);
+
+    // Resize the buffer if necessary
+    if (cBuffer->capacity() - cBuffer->size() < (bytesResampled / sizeof(uint16_t))) {
+      //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "write_cb growing buffer\n"); 
+
+      //TODO: if buffer exceeds some max size, return CURL_WRITEFUNC_ERROR to abort the transfer
+      cBuffer->set_capacity(cBuffer->size() + std::max((bytesResampled / sizeof(uint16_t)), (size_t)BUFFER_GROW_SIZE));
+    }
+    
+    /* Push the data into the buffer */
+    cBuffer->insert(cBuffer->end(), pcm_data.data(), pcm_data.data() + pcm_data.size());
+
+    if (0 == c->reads++) {
+      fireEvent = true;
+    }
+    switch_mutex_unlock(c->mutex);
+  }
+  if (fireEvent && c->session_id) {
+    auto endTime = std::chrono::high_resolution_clock::now();
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(endTime - conn->startTime);
+    auto time_to_first_byte_ms = std::to_string(duration.count());
+    switch_core_session_t* session = switch_core_session_locate(c->session_id);
+    if (session) {
+      switch_channel_t *channel = switch_core_session_get_channel(session);
+      switch_core_session_rwunlock(session);
+      if (channel) {
+        switch_event_t *event;
+        if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_START) == SWITCH_STATUS_SUCCESS) {
+          switch_channel_event_set_data(channel, event);
+
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: firing playback-started\n");
+
+          switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
+          if (c->name_lookup_time_ms) {
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_custom_name_lookup_time_ms", c->name_lookup_time_ms);
+          }
+          if (c->connect_time_ms) {
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_custom_connect_time_ms", c->connect_time_ms);
+          }
+          if (c->final_response_time_ms) {
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_custom_final_response_time_ms", c->final_response_time_ms);
+          }
+          if (c->voice_name) {
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_custom_voice_name", c->voice_name);
+          }
+          if (c->cache_filename) {
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", c->cache_filename);
+          }
+
+          switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
+          switch_event_fire(&event);
+        }
+        else {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
+        }
+      }
+      else {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
+      }
+    }
+    else {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: session %s not found\n", c->session_id);
+    }
+  }
+  return bytes_received;
+}
+
+static bool parseHeader(const std::string& str, std::string& header, std::string& value) {
+    std::vector<std::string> parts;
+    boost::split(parts, str, boost::is_any_of(":"), boost::token_compress_on);
+
+    if (parts.size() != 2)
+        return false;
+
+    header = boost::trim_copy(parts[0]);
+    value = boost::trim_copy(parts[1]);
+    return true;
+}
+
+static int extract_response_code(const std::string& input) {
+  std::size_t space_pos = input.find(' ');
+  if (space_pos == std::string::npos) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid HTTP response format %s\n", input.c_str());
+    return 0;
+  }
+
+  std::size_t code_start_pos = space_pos + 1;
+  std::size_t code_end_pos = input.find(' ', code_start_pos);
+  if (code_end_pos == std::string::npos) {
+    code_end_pos = input.length();
+  }
+
+  std::string code_str = input.substr(code_start_pos, code_end_pos - code_start_pos);
+  int response_code = std::stoi(code_str);
+  return response_code;
+}
+
+static size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo_t *conn) {
+  size_t bytes_received = size * nitems;
+  const std::string prefix = "HTTP/";
+  custom_t* c = conn->custom;
+  std::string header, value;
+  std::string input(buffer, bytes_received);
+  if (parseHeader(input, header, value)) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s with value %s\n", header.c_str(), value.c_str());
+  }
+  else {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: %s\n", input.c_str());
+    if (input.rfind(prefix, 0) == 0) {
+      try {
+        c->response_code = extract_response_code(input);
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: parsed response code: %ld\n", c->response_code);
+      } catch (const std::invalid_argument& e) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "header_callback: invalid response code %s\n", input.substr(prefix.length()).c_str());
+      }
+    }
+  }
+  return bytes_received;
+}
+
+/* CURLOPT_OPENSOCKETFUNCTION */
+static curl_socket_t opensocket(void *clientp, curlsocktype purpose, struct curl_sockaddr *address) {
+  curl_socket_t sockfd = CURL_SOCKET_BAD;
+
+  //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "opensocket: %d\n", purpose);
+  /* restrict to IPv4 */
+  if(purpose == CURLSOCKTYPE_IPCXN && address->family == AF_INET) {
+    /* create a tcp socket object */
+    boost::asio::ip::tcp::socket *tcp_socket = new boost::asio::ip::tcp::socket(io_service);
+
+    /* open it and get the native handle*/
+    boost::system::error_code ec;
+    tcp_socket->open(boost::asio::ip::tcp::v4(), ec);
+
+    if(ec) {
+      /* An error occurred */
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't open socket [%ld][%s]\n", ec, ec.message().c_str());
+    }
+    else {
+      sockfd = tcp_socket->native_handle();
+
+      /* save it for monitoring */
+      socket_map.insert(std::pair<curl_socket_t, boost::asio::ip::tcp::socket *>(sockfd, tcp_socket));
+    }
+  }
+  return sockfd;
+}
+
+/* CURLOPT_CLOSESOCKETFUNCTION */
+static int close_socket(void *clientp, curl_socket_t item) {
+  //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "close_socket : %#X\n", item);
+
+  std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(item);
+  if(it != socket_map.end()) {
+    delete it->second;
+    socket_map.erase(it);
+  }
+  return 0;
+}
+
+
+extern "C" {
+  switch_status_t custom_speech_load() {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "custom_speech_loading..\n");
+    memset(&global, 0, sizeof(GlobalInfo_t));
+    global.multi = curl_multi_init();
+
+     if (!global.multi) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "custom_speech_load curl_multi_init() failed, exiting!\n");
+      return SWITCH_STATUS_FALSE;
+    }
+
+    curl_multi_setopt(global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
+    curl_multi_setopt(global.multi, CURLMOPT_SOCKETDATA, &global);
+    curl_multi_setopt(global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
+    curl_multi_setopt(global.multi, CURLMOPT_TIMERDATA, &global);
+    curl_multi_setopt(global.multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
+
+    /* create temp folder for cache files */
+    const char* baseDir = std::getenv("JAMBONZ_TMP_CACHE_FOLDER");
+    if (!baseDir) {
+      baseDir = "/tmp/";
+    }
+    if (strcmp(baseDir, "/") == 0) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", baseDir);
+      return SWITCH_STATUS_FALSE;
+    }
+
+    fullDirPath = std::string(baseDir) + "tts-cache-files";
+
+    // Create the directory with read, write, and execute permissions for everyone
+    mode_t oldMask = umask(0);
+    int result = mkdir(fullDirPath.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
+    umask(oldMask);
+    if (result != 0) {
+      if (errno != EEXIST) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", fullDirPath.c_str());
+        fullDirPath = "";
+      }
+      else switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "folder %s already exists\n", fullDirPath.c_str());
+    }
+    else {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "created folder %s\n", fullDirPath.c_str());
+    }
+    // init mgp123
+    if (mpg123_init() != MPG123_OK) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to initiate MPG123");
+      return SWITCH_STATUS_FALSE;
+    }
+
+    /* start worker thread that handles transfers*/
+    std::thread t(threadFunc) ;
+    worker_thread.swap( t ) ;
+
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "custom_speech_loaded..\n");
+
+
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t custom_speech_unload() {
+    /* stop the ASIO IO service */
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "custom_speech_unload: stopping io service\n");
+    io_service.stop();
+
+    /* Join the worker thread */
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "custom_speech_unload: wait for worker thread to complete\n");
+    if (worker_thread.joinable()) {
+        worker_thread.join();
+    }
+
+    /* cleanup curl multi handle*/
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "custom_speech_unload: release curl multi\n");
+    curl_multi_cleanup(global.multi);
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "custom_speech_unload: completed\n");
+
+    mpg123_exit();
+
+		return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t custom_speech_open(custom_t* custom) {
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t custom_speech_feed_tts(custom_t* c, char* text, switch_speech_flag_t *flags) {
+    CURLMcode rc;
+
+    const int MAX_CHARS = 20;
+    char tempText[MAX_CHARS + 4]; // +4 for the ellipsis and null terminator
+
+    if (strlen(text) > MAX_CHARS) {
+        strncpy(tempText, text, MAX_CHARS);
+        strcpy(tempText + MAX_CHARS, "...");
+    } else {
+        strcpy(tempText, text);
+    }
+
+    /* open cache file */
+    if (c->cache_audio && fullDirPath.length() > 0) {
+      switch_uuid_t uuid;
+      char uuid_str[SWITCH_UUID_FORMATTED_LENGTH + 1];
+      char outfile[512] = "";
+      int fd;
+
+      switch_uuid_get(&uuid);
+      switch_uuid_format(uuid_str, &uuid);
+
+      switch_snprintf(outfile, sizeof(outfile), "%s%s%s.mp3", fullDirPath.c_str(), SWITCH_PATH_SEPARATOR, uuid_str);
+      c->cache_filename = strdup(outfile);
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "writing audio cache file to %s\n", c->cache_filename);
+
+      mode_t oldMask = umask(0);
+      fd = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
+      umask(oldMask);
+      if (fd == -1 ) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
+      }
+      else {
+        c->file = fdopen(fd, "wb");
+        if (!c->file) {
+          close(fd);
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
+        }
+      }
+    }
+
+    if (!c->custom_tts_url) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "custom_speech_feed_tts: no custom_tts_url provided\n");
+      return SWITCH_STATUS_FALSE;
+    }
+
+    /* format url*/
+    std::string url = c->custom_tts_url;
+
+    /* create the JSON body */
+    cJSON * jResult = cJSON_CreateObject();
+    cJSON_AddStringToObject(jResult, "text", text);
+    cJSON_AddStringToObject(jResult, "type", std::strncmp(text, "<speak", 6) == 0 ? "ssml" : "text");
+    if (c->language) {
+      cJSON_AddStringToObject(jResult, "language", c->language);
+    }
+    if (c->voice_name) {
+      cJSON_AddStringToObject(jResult, "voice", c->voice_name);
+    }
+    char *json = cJSON_PrintUnformatted(jResult);
+
+    cJSON_Delete(jResult);
+
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "custom_speech_feed_tts: [%s] [%s]\n", url.c_str(), tempText);
+
+    ConnInfo_t *conn = pool.malloc() ;
+
+    // COnfigure MPG123
+    int mhError = 0;
+    mpg123_handle *mh = mpg123_new("auto", &mhError);
+    if (!mh) {
+      const char *mhErr = mpg123_plain_strerror(mhError);
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error allocating mpg123 handle! %s\n", switch_str_nil(mhErr));
+      return SWITCH_STATUS_FALSE;
+    }
+
+    if (mpg123_open_feed(mh) != MPG123_OK) {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_open_feed!\n");
+      return SWITCH_STATUS_FALSE;
+		}
+
+    if (mpg123_format_all(mh) != MPG123_OK) {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_format_all!\n");
+      return SWITCH_STATUS_FALSE;
+		}
+
+    if (mpg123_param(mh, MPG123_FLAGS, MPG123_MONO_MIX, 0) != MPG123_OK) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error forcing single channel!\n");
+      return SWITCH_STATUS_FALSE;
+    }
+
+    CURL* easy = createEasyHandle();
+    c->conn = (void *) conn ;
+    conn->custom = c;
+    conn->easy = easy;
+    conn->mh = mh;
+    conn->global = &global;
+    conn->hdr_list = NULL ;
+    conn->file = c->file;
+    conn->body = json;
+    conn->flushed = false;
+    
+
+    c->circularBuffer = (void *) new CircularBuffer_t(8192);
+
+    if (mpg123_param(mh, MPG123_FORCE_RATE, c->rate /*Hz*/, 0) != MPG123_OK) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_param!\n");
+      return SWITCH_STATUS_FALSE;
+    }
+
+    curl_easy_setopt(easy, CURLOPT_URL, url.c_str());
+    curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, write_cb);
+    curl_easy_setopt(easy, CURLOPT_WRITEDATA, conn);
+    curl_easy_setopt(easy, CURLOPT_ERRORBUFFER, conn->error);
+    curl_easy_setopt(easy, CURLOPT_PRIVATE, conn);
+    curl_easy_setopt(easy, CURLOPT_VERBOSE, 0L);
+    curl_easy_setopt(easy, CURLOPT_NOPROGRESS, 1L);
+    curl_easy_setopt(easy, CURLOPT_HEADERFUNCTION, header_callback);
+    curl_easy_setopt(easy, CURLOPT_HEADERDATA, conn);
+    
+    /* call this function to get a socket */
+    curl_easy_setopt(easy, CURLOPT_OPENSOCKETFUNCTION, opensocket);
+
+    /* call this function to close a socket */
+    curl_easy_setopt(easy, CURLOPT_CLOSESOCKETFUNCTION, close_socket);
+
+    if (c->auth_token) {
+      std::ostringstream api_key_stream;
+      api_key_stream << "Authorization: Bearer " << c->auth_token;
+      conn->hdr_list = curl_slist_append(conn->hdr_list, api_key_stream.str().c_str());
+    }
+    conn->hdr_list = curl_slist_append(conn->hdr_list, "Accept: audio/mpeg");
+    conn->hdr_list = curl_slist_append(conn->hdr_list, "Content-Type: application/json");
+    curl_easy_setopt(easy, CURLOPT_HTTPHEADER, conn->hdr_list);
+
+    curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
+    //curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
+
+    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+
+    rc = curl_multi_add_handle(global.multi, conn->easy);
+    mcode_test("new_conn: curl_multi_add_handle", rc);
+
+    /* start a timer to measure the duration until we receive first byte of audio */
+    conn->startTime = std::chrono::high_resolution_clock::now();
+
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "custom_speech_feed_tts: called curl_multi_add_handle\n");
+
+
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t custom_speech_read_tts(custom_t* c, void *data, size_t *datalen, switch_speech_flag_t *flags) {
+    CircularBuffer_t *cBuffer = (CircularBuffer_t *) c->circularBuffer;
+    std::vector<uint16_t> pcm_data;
+
+    {
+      switch_mutex_lock(c->mutex);
+      ConnInfo_t *conn = (ConnInfo_t *) c->conn;
+      if (c->response_code > 0 && c->response_code != 200) {
+        switch_mutex_unlock(c->mutex);
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "custom_speech_read_tts, returning failure\n") ;  
+        return SWITCH_STATUS_FALSE;
+      }
+      if (conn && conn->flushed) {
+        switch_mutex_unlock(c->mutex);
+        return SWITCH_STATUS_BREAK;
+      }
+      if (cBuffer->empty()) {
+        if (c->draining) {
+          switch_mutex_unlock(c->mutex);
+          return SWITCH_STATUS_BREAK;
+        }
+        /* no audio available yet so send silence */
+        memset(data, 255, *datalen);
+        switch_mutex_unlock(c->mutex);
+        return SWITCH_STATUS_SUCCESS;
+      }
+      size_t size = std::min((*datalen/2), cBuffer->size());
+      pcm_data.insert(pcm_data.end(), cBuffer->begin(), cBuffer->begin() + size);
+      cBuffer->erase(cBuffer->begin(), cBuffer->begin() + size);
+      switch_mutex_unlock(c->mutex);
+    }
+
+    memcpy(data, pcm_data.data(), pcm_data.size() * sizeof(uint16_t));
+    *datalen = pcm_data.size() * sizeof(uint16_t);
+
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t custom_speech_flush_tts(custom_t* c) {
+    bool download_complete = c->response_code == 200;
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "custom_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;  
+    ConnInfo_t *conn = (ConnInfo_t *) c->conn;
+    CircularBuffer_t *cBuffer = (CircularBuffer_t *) c->circularBuffer;
+    delete cBuffer;
+    c->circularBuffer = nullptr ;
+
+    if (conn) {
+      conn->flushed = true;
+      if (!download_complete) {
+        if (conn->file) {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "closing audio cache file %s because download was interrupted\n", c->cache_filename);
+          if (fclose(conn->file) != 0) {
+            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "error closing audio cache file\n");
+          }
+          conn->file = nullptr ;
+        }
+
+        if (c->cache_filename) {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", c->cache_filename);
+          if (unlink(c->cache_filename) != 0) {
+            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
+              c->cache_filename, errno, strerror(errno));
+          }
+          free(c->cache_filename);
+          c->cache_filename = nullptr ;
+        }
+      }
+    }
+    if (c->session_id) {
+      switch_core_session_t* session = switch_core_session_locate(c->session_id);
+      if (session) {
+        switch_channel_t *channel = switch_core_session_get_channel(session);
+        switch_core_session_rwunlock(session);
+        if (channel) {
+          switch_event_t *event;
+          if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_STOP) == SWITCH_STATUS_SUCCESS) {
+            switch_channel_event_set_data(channel, event);
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_custom_response_code", std::to_string(c->response_code).c_str());
+            if (c->cache_filename && c->response_code == 200) {
+              switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", c->cache_filename);
+            }
+            if (c->response_code != 200 && c->err_msg) {
+              switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_error", c->err_msg);
+            }
+            switch_event_fire(&event);
+          }
+          else {
+            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
+          }
+        }
+        else {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
+        }
+      }
+    }
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+	switch_status_t custom_speech_close(custom_t* c) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "custom_speech_close\n") ;
+		return SWITCH_STATUS_SUCCESS;
+	}
+}

mod_custom_tts/custom_glue.h

@@ -0,0 +1,12 @@
+#ifndef __CUSTOM_GLUE_H__
+#define __CUSTOM_GLUE_H__
+
+switch_status_t custom_speech_load();
+switch_status_t custom_speech_open(custom_t* custom);
+switch_status_t custom_speech_feed_tts(custom_t* custom, char* text, switch_speech_flag_t *flags);
+switch_status_t custom_speech_read_tts(custom_t* custom, void *data, size_t *datalen, switch_speech_flag_t *flags);
+switch_status_t custom_speech_flush_tts(custom_t* custom);
+switch_status_t custom_speech_close(custom_t* custom);
+switch_status_t custom_speech_unload();
+
+#endif

mod_custom_tts/mod_custom_tts.c

@@ -0,0 +1,158 @@
+#include "mod_custom_tts.h"
+#include "custom_glue.h"
+
+SWITCH_MODULE_LOAD_FUNCTION(mod_custom_tts_load);
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_custom_tts_shutdown);
+SWITCH_MODULE_DEFINITION(mod_custom_tts, mod_custom_tts_load, mod_custom_tts_shutdown, NULL);
+
+static void clearCustomVendor(custom_t* c, int freeAll) {
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "clearCustomVendor\n");
+  if (c->auth_token) free(c->auth_token);
+  if (c->custom_tts_url) free(c->custom_tts_url);
+  if (c->language) free(c->language);
+  if (c->ct) free(c->ct);
+  if (c->err_msg) free(c->err_msg);
+  if (c->name_lookup_time_ms) free(c->name_lookup_time_ms);
+  if (c->connect_time_ms) free(c->connect_time_ms);
+  if (c->final_response_time_ms) free(c->final_response_time_ms);
+  if (c->cache_filename) free(c->cache_filename);
+  
+
+  c->auth_token = NULL;
+  c->custom_tts_url = NULL;
+  c->language = NULL;
+  c->ct = NULL;
+  c->err_msg = NULL;
+  c->name_lookup_time_ms = NULL;
+  c->connect_time_ms = NULL;
+  c->final_response_time_ms = NULL;
+  c->cache_filename = NULL;
+
+  if (freeAll) {
+    if (c->voice_name) free(c->voice_name);
+    if (c->session_id) free(c->session_id);
+    c->voice_name = NULL;
+    c->session_id = NULL;
+  }
+}
+
+static custom_t * createOrRetrievePrivateData(switch_speech_handle_t *sh) {
+  custom_t *c = (custom_t *) sh->private_info;  
+  if (!c) {
+    c = switch_core_alloc(sh->memory_pool, sizeof(*c));
+  	sh->private_info = c;
+    memset(c, 0, sizeof(*c));
+    switch_mutex_init(&c->mutex, SWITCH_MUTEX_NESTED, sh->memory_pool);
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "allocated custom_t\n");
+  }
+  return c;
+}
+
+switch_status_t w_speech_open(switch_speech_handle_t *sh, const char *voice_name, int rate, int channels, switch_speech_flag_t *flags)
+{
+  custom_t *c = createOrRetrievePrivateData(sh);
+  c->voice_name = strdup(voice_name);
+  c->rate = rate;
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_speech_open voice: %s, rate %d, channels %d\n", voice_name, rate, channels);
+  return custom_speech_open(c);
+}
+
+static switch_status_t w_speech_close(switch_speech_handle_t *sh, switch_speech_flag_t *flags)
+{
+  switch_status_t rc;
+  custom_t *c = createOrRetrievePrivateData(sh);
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_speech_close\n");
+
+  switch_mutex_destroy(c->mutex);
+
+  rc = custom_speech_close(c);
+  clearCustomVendor(c, 1);
+  return rc;
+}
+
+/**
+ * Freeswitch will call this function to feed us text to speak
+ */
+static switch_status_t w_speech_feed_tts(switch_speech_handle_t *sh, char *text, switch_speech_flag_t *flags)
+{
+  custom_t *c = createOrRetrievePrivateData(sh);
+  c->draining = 0;
+  c->reads = 0;
+  c->response_code = 0;
+  c->err_msg = NULL;
+
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_speech_feed_tts\n");
+
+  return custom_speech_feed_tts(c, text, flags);
+}
+
+/**
+ * Freeswitch calls periodically to get some rendered audio in L16 format. We can provide up to 8k of audio at a time.
+ */
+static switch_status_t w_speech_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, switch_speech_flag_t *flags)
+{
+  custom_t *c = createOrRetrievePrivateData(sh);
+  return custom_speech_read_tts(c, data, datalen, flags);
+}
+
+/**
+ * This is called at the end, not sure exactly what we need to do here..
+ */
+static void w_speech_flush_tts(switch_speech_handle_t *sh)
+{
+  custom_t *c = createOrRetrievePrivateData(sh);
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_speech_flush_tts\n");
+  custom_speech_flush_tts(c);
+
+  clearCustomVendor(c, 0);
+}
+
+static void w_text_param_tts(switch_speech_handle_t *sh, char *param, const char *val)
+{
+  custom_t *c = createOrRetrievePrivateData(sh);
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_text_param_tts: %s=%s\n", param, val);
+  if (0 == strcmp(param, "auth_token")) {
+    if (c->auth_token) free(c->auth_token);
+    c->auth_token = strdup(val);
+  } else if (0 == strcmp(param, "custom_tts_url")) {
+    if (c->custom_tts_url) free(c->custom_tts_url);
+    c->custom_tts_url = strdup(val);
+  } else if (0 == strcmp(param, "language")) {
+    if (c->language) free(c->language);
+    c->language = strdup(val);
+  } else if (0 == strcmp(param, "session-uuid")) {
+    if (c->session_id) free(c->session_id);
+    c->session_id = strdup(val);
+  } else if (0 == strcmp(param, "write_cache_file") && switch_true(val)) {
+    c->cache_audio = 1;
+  }
+}
+static void w_numeric_param_tts(switch_speech_handle_t *sh, char *param, int val)
+{
+}
+static void w_float_param_tts(switch_speech_handle_t *sh, char *param, double val)
+{
+}
+
+SWITCH_MODULE_LOAD_FUNCTION(mod_custom_tts_load)
+{
+  switch_speech_interface_t *speech_interface;
+
+  *module_interface = switch_loadable_module_create_module_interface(pool, modname);
+  speech_interface = switch_loadable_module_create_interface(*module_interface, SWITCH_SPEECH_INTERFACE);
+  speech_interface->interface_name = "custom";
+  speech_interface->speech_open = w_speech_open;
+  speech_interface->speech_close = w_speech_close;
+  speech_interface->speech_feed_tts = w_speech_feed_tts;
+  speech_interface->speech_read_tts = w_speech_read_tts;
+	speech_interface->speech_flush_tts = w_speech_flush_tts;
+	speech_interface->speech_text_param_tts = w_text_param_tts;
+	speech_interface->speech_numeric_param_tts = w_numeric_param_tts;
+	speech_interface->speech_float_param_tts = w_float_param_tts;
+  return custom_speech_load();
+}
+
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_custom_tts_shutdown)
+{
+  return custom_speech_unload();
+}

mod_custom_tts/mod_custom_tts.h

@@ -0,0 +1,31 @@
+#ifndef __MOD_CUSTOM_TTS_H__
+#define __MOD_CUSTOM_TTS_H__
+
+#include <switch.h>
+typedef struct custom_data {
+  char *voice_name;
+  char *auth_token;
+  char *custom_tts_url;
+  char *language;
+
+  /* result data */
+  long response_code;
+  char *ct;
+  char *name_lookup_time_ms;
+  char *connect_time_ms;
+  char *final_response_time_ms;
+  char *err_msg;
+  char *cache_filename;
+  char *session_id;
+
+  int rate;
+  int draining;
+  int reads;
+  int cache_audio;
+
+	void *conn;
+  void *circularBuffer;
+  switch_mutex_t *mutex;
+  FILE *file;
+} custom_t;
+#endif

mod_deepgram_transcribe/audio_pipe.cpp

@@ -176,7 +176,9 @@ int AudioPipe::lws_callback(struct lws *wsi,
           if (lws_is_final_fragment(wsi)) {
             if (nullptr != ap->m_recv_buf) {
               std::string msg((char *)ap->m_recv_buf, ap->m_recv_buf_ptr - ap->m_recv_buf);
-              ap->m_callback(ap->m_uuid.c_str(),  ap->m_bugname.c_str(), deepgram::AudioPipe::MESSAGE, msg.c_str(),  ap->isFinished());
+              if (!ap->m_silence_disconnect) {
+                ap->m_callback(ap->m_uuid.c_str(),  ap->m_bugname.c_str(), deepgram::AudioPipe::MESSAGE, msg.c_str(),  ap->isFinished());
+              }
               if (nullptr != ap->m_recv_buf) free(ap->m_recv_buf);
             }
             ap->m_recv_buf = ap->m_recv_buf_ptr = nullptr;
@@ -451,7 +453,7 @@ AudioPipe::AudioPipe(const char* uuid, const char* bugname, const char* host, un
   m_uuid(uuid), m_host(host), m_port(port), m_path(path), m_finished(false), m_bugname(bugname),
   m_audio_buffer_min_freespace(minFreespace), m_audio_buffer_max_len(bufLen), m_gracefulShutdown(false),
   m_audio_buffer_write_offset(LWS_PRE), m_recv_buf(nullptr), m_recv_buf_ptr(nullptr), m_useTls(useTls),
-  m_state(LWS_CLIENT_IDLE), m_wsi(nullptr), m_vhd(nullptr), m_callback(callback) {
+  m_state(LWS_CLIENT_IDLE), m_wsi(nullptr), m_vhd(nullptr), m_callback(callback), m_silence_disconnect(false) {
 
   if (apiKey) m_apiKey = apiKey;
   else m_apiKey = "";
@@ -516,6 +518,11 @@ void AudioPipe::finish() {
   bufferForSending("{\"type\": \"CloseStream\"}");
 }
 
+void AudioPipe::finish_in_silence() {
+  m_silence_disconnect = true;
+  finish();
+}
+
 void AudioPipe::waitForClose() {
   std::shared_future<void> sf(m_promise.get_future());
   sf.wait();

mod_deepgram_transcribe/audio_pipe.hpp

@@ -76,6 +76,7 @@ namespace deepgram {
 
     void close() ;
     void finish();
+    void finish_in_silence();
     void waitForClose();
     void setClosed() { m_promise.set_value(); }
     bool isFinished() { return m_finished;}
@@ -138,6 +139,7 @@ namespace deepgram {
     std::string m_bugname;
     std::promise<void> m_promise;
     bool m_useTls;
+    bool m_silence_disconnect;
   };
 
 } // namespace deepgram

mod_deepgram_transcribe/dg_transcribe_glue.cpp

@@ -22,6 +22,7 @@
 
 #define RTP_PACKETIZATION_PERIOD 20
 #define FRAME_SIZE_8000  320 /*which means each 20ms frame as 320 bytes at 8 khz (1 channel only)*/
+#define DEEPGRAM_KEEP_ALIVE_INTERVAL_SECOND 8
 
 namespace {
   static bool hasDefaultCredentials = false;
@@ -86,13 +87,17 @@ namespace {
   static const char* emptyTranscript = 
     "\"is_final\":false,\"speech_final\":false,\"channel\":{\"alternatives\":[{\"transcript\":\"\",\"confidence\":0.0,\"words\":[]}]}";
 
-  static void reaper(private_t *tech_pvt) {
+  static void reaper(private_t *tech_pvt, bool silence_disconnect) {
     std::shared_ptr<deepgram::AudioPipe> pAp;
     pAp.reset((deepgram::AudioPipe *)tech_pvt->pAudioPipe);
     tech_pvt->pAudioPipe = nullptr;
 
-    std::thread t([pAp, tech_pvt]{
-      pAp->finish();
+    std::thread t([pAp, tech_pvt, silence_disconnect]{
+      if (silence_disconnect) {
+        pAp->finish_in_silence();
+      } else {
+        pAp->finish();
+      }
       pAp->waitForClose();
       switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s (%u) got remote close\n", tech_pvt->sessionId, tech_pvt->id);
     });
@@ -167,7 +172,7 @@ namespace {
     else if (customModel) oss << "&model=" << customModel;
 
     if (var = switch_channel_get_variable(channel, "DEEPGRAM_SPEECH_MODEL_VERSION")) {
-     oss <<  "&version";
+     oss <<  "&version=";
      oss <<  var;
     }
     oss <<  "&language=";
@@ -342,60 +347,82 @@ namespace {
 
     int err;
     int useTls = true;
+    std::string host;
+    int port = 443;
+    size_t buflen = LWS_PRE + (FRAME_SIZE_8000 * desiredSampling / 8000 * channels * 1000 / RTP_PACKETIZATION_PERIOD * nAudioBufferSecs);
+
+    std::ostringstream configuration_stream;
     switch_codec_implementation_t read_impl;
     switch_channel_t *channel = switch_core_session_get_channel(session);
 
     switch_core_session_get_read_impl(session, &read_impl);
   
-    memset(tech_pvt, 0, sizeof(private_t));
-  
     std::string path;
     constructPath(session, path, desiredSampling, channels, lang, interim);
     switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "path: %s\n", path.c_str());
 
-    strncpy(tech_pvt->sessionId, switch_core_session_get_uuid(session), MAX_SESSION_ID);
-
     const char* endpoint = switch_channel_get_variable(channel, "DEEPGRAM_URI");
     if (endpoint != nullptr) {
       std::string ep(endpoint);
-
       useTls = switch_true(switch_channel_get_variable(channel, "DEEPGRAM_USE_TLS"));
 
       size_t pos = ep.find(':');
+      host = ep;
       if (pos != std::string::npos) {
-        std::string host = ep.substr(0, pos);
-        std::string port = ep.substr(pos + 1);
-        strncpy(tech_pvt->host, host.c_str(), MAX_WS_URL_LEN);
-        tech_pvt->port = ::atoi(port.c_str());
-      }
-      else {
-        strncpy(tech_pvt->host, ep.c_str(), MAX_WS_URL_LEN);
-        tech_pvt->port = 443;
+        host = ep.substr(0, pos);
+        std::string strPort = ep.substr(pos + 1);
+        port = ::atoi(strPort.c_str());
       }
       switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, 
-        "connecting to deepgram on-prem %s port %d, using tls? (%s)\n", tech_pvt->host, tech_pvt->port, useTls ? "yes" : "no");
-    }
-    else {
-      strncpy(tech_pvt->host, "api.deepgram.com", MAX_WS_URL_LEN);
-      tech_pvt->port = 443;
+        "connecting to deepgram on-prem %s port %d, using tls? (%s)\n", host.c_str(), port, useTls ? "yes" : "no");
+    } else {
+      host = "api.deepgram.com";
     }
 
-    strncpy(tech_pvt->path, path.c_str(), MAX_PATH_LEN);    
+    const char* apiKey = switch_channel_get_variable(channel, "DEEPGRAM_API_KEY");
+    if (!apiKey && defaultApiKey) {
+      apiKey = defaultApiKey;
+    } else if (!apiKey && endpoint == nullptr) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "no deepgram api key provided\n");
+      return SWITCH_STATUS_FALSE;
+    }
+
+    configuration_stream <<
+      host << ":" <<
+      port << ";" <<
+      path << ";" <<
+      buflen << ";" <<
+      read_impl.decoded_bytes_per_packet << ";" <<
+      apiKey << ";" <<
+      useTls;
+
+    if (tech_pvt->pAudioPipe) {
+      // stop sending keep alive
+      tech_pvt->is_keep_alive = 0;
+      if (0 != strcmp(tech_pvt->configuration, configuration_stream.str().c_str())) {
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "fork_data_init: stop existing deepgram connection, old configuration %s, new configuration %s\n",
+          tech_pvt->configuration, configuration_stream.str().c_str());
+        reaper(tech_pvt, true);
+      } else {
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "fork_data_init: enable existing deepgram connection\n");
+        return SWITCH_STATUS_SUCCESS;
+      }
+    } else {
+      memset(tech_pvt, 0, sizeof(private_t));
+    }
+
+    strncpy(tech_pvt->sessionId, switch_core_session_get_uuid(session), MAX_SESSION_ID);
+    strncpy(tech_pvt->host, host.c_str(), MAX_WS_URL_LEN);
+    tech_pvt->port = port;
+    strncpy(tech_pvt->path, path.c_str(), MAX_PATH_LEN);   
+    strncpy(tech_pvt->configuration, configuration_stream.str().c_str(), MAX_PATH_LEN) ;
     tech_pvt->sampling = desiredSampling;
     tech_pvt->responseHandler = responseHandler;
     tech_pvt->channels = channels;
     tech_pvt->id = ++idxCallCount;
     tech_pvt->buffer_overrun_notified = 0;
+    tech_pvt->is_keep_alive = 0;
     strncpy(tech_pvt->bugname, bugname, MAX_BUG_LEN);
-    
-    size_t buflen = LWS_PRE + (FRAME_SIZE_8000 * desiredSampling / 8000 * channels * 1000 / RTP_PACKETIZATION_PERIOD * nAudioBufferSecs);
-
-    const char* apiKey = switch_channel_get_variable(channel, "DEEPGRAM_API_KEY");
-    if (!apiKey && defaultApiKey) apiKey = defaultApiKey;
-    else if (!apiKey && endpoint == nullptr) {
-      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "no deepgram api key provided\n");
-      return SWITCH_STATUS_FALSE;
-    }
 
     deepgram::AudioPipe* ap = new deepgram::AudioPipe(tech_pvt->sessionId, bugname, tech_pvt->host, tech_pvt->port, tech_pvt->path, 
       buflen, read_impl.decoded_bytes_per_packet, apiKey, useTls, eventCallback);
@@ -406,18 +433,26 @@ namespace {
 
     tech_pvt->pAudioPipe = static_cast<void *>(ap);
 
-    switch_mutex_init(&tech_pvt->mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session));
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connecting now\n");
+    ap->connect();
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connection in progress\n");
 
-    if (desiredSampling != sampling) {
-      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) resampling from %u to %u\n", tech_pvt->id, sampling, desiredSampling);
-      tech_pvt->resampler = speex_resampler_init(channels, sampling, desiredSampling, SWITCH_RESAMPLE_QUALITY, &err);
-      if (0 != err) {
-        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error initializing resampler: %s.\n", speex_resampler_strerror(err));
-        return SWITCH_STATUS_FALSE;
-      }
+    if (!tech_pvt->mutex) {
+      switch_mutex_init(&tech_pvt->mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session));
     }
-    else {
-      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) no resampling needed for this call\n", tech_pvt->id);
+    
+    if (!tech_pvt->resampler) {
+      if (desiredSampling != sampling) {
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) resampling from %u to %u\n", tech_pvt->id, sampling, desiredSampling);
+        tech_pvt->resampler = speex_resampler_init(channels, sampling, desiredSampling, SWITCH_RESAMPLE_QUALITY, &err);
+        if (0 != err) {
+          switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error initializing resampler: %s.\n", speex_resampler_strerror(err));
+          return SWITCH_STATUS_FALSE;
+        }
+      }
+      else {
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) no resampling needed for this call\n", tech_pvt->id);
+      }
     }
 
     switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) fork_data_init\n", tech_pvt->id);
@@ -474,11 +509,22 @@ extern "C" {
   switch_status_t dg_transcribe_session_init(switch_core_session_t *session, 
     responseHandler_t responseHandler, uint32_t samples_per_second, uint32_t channels, 
     char* lang, int interim, char* bugname, void **ppUserData)
-  {    	
+  {
     int err;
+    switch_channel_t *channel = switch_core_session_get_channel(session);
+    switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, bugname);
+    private_t* tech_pvt;
+    if (bug) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "reuse existing kep alive deepgram connection\n");
+      tech_pvt = (private_t*) switch_core_media_bug_get_user_data(bug);
+    } else {
+      tech_pvt = (private_t *) switch_core_session_alloc(session, sizeof(private_t));
+      tech_pvt->pAudioPipe = NULL;
+      tech_pvt->is_keep_alive = 0;
+      tech_pvt->mutex = NULL;
+      tech_pvt->resampler = NULL;
+    }
 
-    // allocate per-session data structure
-    private_t* tech_pvt = (private_t *) switch_core_session_alloc(session, sizeof(private_t));
     if (!tech_pvt) {
       switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "error allocating memory!\n");
       return SWITCH_STATUS_FALSE;
@@ -491,16 +537,13 @@ extern "C" {
 
     *ppUserData = tech_pvt;
 
-    deepgram::AudioPipe *pAudioPipe = static_cast<deepgram::AudioPipe *>(tech_pvt->pAudioPipe);
-    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connecting now\n");
-    pAudioPipe->connect();
-    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connection in progress\n");
     return SWITCH_STATUS_SUCCESS;
   }
 
 	switch_status_t dg_transcribe_session_stop(switch_core_session_t *session,int channelIsClosing, char* bugname) {
     switch_channel_t *channel = switch_core_session_get_channel(session);
     switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, bugname);
+    const bool use_single_connection = switch_true(std::getenv("DEEPGRAM_SPEECH_USE_SINGLE_CONNECTION"));
     if (!bug) {
       switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "dg_transcribe_session_stop: no bug - websocket conection already closed\n");
       return SWITCH_STATUS_FALSE;
@@ -511,6 +554,12 @@ extern "C" {
     switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) dg_transcribe_session_stop\n", id);
 
     if (!tech_pvt) return SWITCH_STATUS_FALSE;
+    if (use_single_connection && !channelIsClosing) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "dg_transcribe_session_stop: call is running, use_single_connection is true, keep alive is activated\n", id);
+      tech_pvt->is_keep_alive = 1;
+      tech_pvt->frame_count = 0;
+      return SWITCH_STATUS_SUCCESS;
+    }
       
     // close connection and get final responses
     switch_mutex_lock(tech_pvt->mutex);
@@ -518,7 +567,7 @@ extern "C" {
     if (!channelIsClosing) switch_core_media_bug_remove(session, &bug);
 
     deepgram::AudioPipe *pAudioPipe = static_cast<deepgram::AudioPipe *>(tech_pvt->pAudioPipe);
-    if (pAudioPipe) reaper(tech_pvt);
+    if (pAudioPipe) reaper(tech_pvt, false);
     destroy_tech_pvt(tech_pvt);
     switch_mutex_unlock(tech_pvt->mutex);
     switch_mutex_destroy(tech_pvt->mutex);
@@ -532,9 +581,31 @@ extern "C" {
     size_t inuse = 0;
     bool dirty = false;
     char *p = (char *) "{\"msg\": \"buffer overrun\"}";
+    char *keep_alive = (char *) "{\"type\": \"KeepAlive\"}";
 
     if (!tech_pvt) return SWITCH_TRUE;
     
+
+    // Keep sending keep alive if there is no transcribe activity
+    if (tech_pvt->is_keep_alive && tech_pvt->pAudioPipe) {
+      deepgram::AudioPipe *pAudioPipe = static_cast<deepgram::AudioPipe *>(tech_pvt->pAudioPipe);
+      if (++tech_pvt->frame_count * 20 /*ms*/ / 1000 >= DEEPGRAM_KEEP_ALIVE_INTERVAL_SECOND) {
+        tech_pvt->frame_count = 0;
+        pAudioPipe->bufferForSending(keep_alive);
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "dg_transcribe_frame: sending %s to deepgram\n", keep_alive);
+      }
+      // remove media bug buffered data
+      while (true) {
+        unsigned char data[SWITCH_RECOMMENDED_BUFFER_SIZE] = {0};
+        switch_frame_t frame = { 0 };
+        frame.data = data;
+        frame.buflen = SWITCH_RECOMMENDED_BUFFER_SIZE;
+        switch_status_t rv = switch_core_media_bug_read(bug, &frame, SWITCH_TRUE);
+        if (rv != SWITCH_STATUS_SUCCESS) break;
+      }
+      return SWITCH_TRUE;
+    }
+    
     if (switch_mutex_trylock(tech_pvt->mutex) == SWITCH_STATUS_SUCCESS) {
       if (!tech_pvt->pAudioPipe) {
         switch_mutex_unlock(tech_pvt->mutex);
@@ -545,7 +616,6 @@ extern "C" {
         switch_mutex_unlock(tech_pvt->mutex);
         return SWITCH_TRUE;
       }
-
       pAudioPipe->lockAudioBuffer();
       size_t available = pAudioPipe->binarySpaceAvailable();
       if (NULL == tech_pvt->resampler) {
@@ -587,7 +657,6 @@ extern "C" {
           if (frame.datalen) {
             spx_uint32_t out_len = available >> 1;  // space for samples which are 2 bytes
             spx_uint32_t in_len = frame.samples;
-
             speex_resampler_process_interleaved_int(tech_pvt->resampler, 
               (const spx_int16_t *) frame.data, 
               (spx_uint32_t *) &in_len, 

mod_deepgram_transcribe/mod_deepgram_transcribe.c

@@ -5,6 +5,7 @@
  */
 #include "mod_deepgram_transcribe.h"
 #include "dg_transcribe_glue.h"
+#include <stdio.h>
 
 /* Prototypes */
 SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_deepgram_transcribe_shutdown);
@@ -72,8 +73,10 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
 	switch_codec_implementation_t read_impl = { 0 };
 	void *pUserData;
 	uint32_t samples_per_second;
+	int use_single_connection = switch_true(getenv("DEEPGRAM_SPEECH_USE_SINGLE_CONNECTION"));
+	bug = switch_channel_get_private(channel, bugname);
 
-	if (switch_channel_get_private(channel, bugname)) {
+	if (bug && !use_single_connection) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing bug from previous transcribe\n");
 		do_stop(session, bugname);
 	}
@@ -90,11 +93,13 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing dg speech session.\n");
 		return SWITCH_STATUS_FALSE;
 	}
-	if ((status = switch_core_media_bug_add(session, "dg_transcribe", NULL, capture_callback, pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
-		return status;
+	if (!bug || !use_single_connection) {
+		if ((status = switch_core_media_bug_add(session, "dg_transcribe", NULL, capture_callback, pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
+			return status;
+		}
+		switch_channel_set_private(channel, bugname, bug);
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "added media bug for dg transcribe\n");
 	}
-  switch_channel_set_private(channel, bugname, bug);
-	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "added media bug for dg transcribe\n");
 
 	return SWITCH_STATUS_SUCCESS;
 }

mod_deepgram_transcribe/mod_deepgram_transcribe.h

@@ -39,6 +39,9 @@ struct private_data {
   unsigned int id;
   int buffer_overrun_notified:1;
   int is_finished:1;
+  int is_keep_alive;
+  int frame_count;
+  char configuration[MAX_PATH_LEN];
 };
 
 typedef struct private_data private_t;

mod_deepgram_tts/deepgram_glue.cpp

@@ -65,6 +65,27 @@ std::string secondsToMillisecondsString(double seconds) {
     return std::to_string(milliseconds_long);
 }
 
+static std::string getEnvVar(const std::string& varName) {
+    const char* val = std::getenv(varName.c_str());
+    return val ? std::string(val) : "";
+}
+
+static long getConnectionTimeout() {
+    std::string connectTimeoutStr = getEnvVar("DEEPGRAM_TTS_CURL_CONNECT_TIMEOUT");
+
+    if (connectTimeoutStr.empty()) {
+        connectTimeoutStr = getEnvVar("TTS_CURL_CONNECT_TIMEOUT");
+    }
+
+    long connectTimeout = 10L;
+
+    if (!connectTimeoutStr.empty()) {
+        connectTimeout = std::stol(connectTimeoutStr);
+    }
+
+    return connectTimeout;
+}
+
 static CURL* createEasyHandle(void) {
   CURL* easy = curl_easy_init();
   if(!easy) {
@@ -77,7 +98,7 @@ static CURL* createEasyHandle(void) {
 
   // set connect timeout to 3 seconds and total timeout to 109 seconds
   curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
-  curl_easy_setopt(easy, CURLOPT_TIMEOUT, 10L);
+  curl_easy_setopt(easy, CURLOPT_TIMEOUT, getConnectionTimeout());
 
   return easy ;    
 }
@@ -500,6 +521,7 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
 
           switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
           switch_event_fire(&event);
+          d->playback_start_sent = 1;
         }
         else {
           switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
@@ -741,7 +763,7 @@ extern "C" {
     std::string url;
     std::ostringstream url_stream;
     // always use sample_rate=8000 for support jambonz caching system.
-    url_stream << "https://api.deepgram.com/v1/speak?model=" << d->voice_name << "&encoding=linear16&sample_rate=8000";
+    url_stream << (d->endpoint != nullptr ? d->endpoint : "https://api.deepgram.com") << "/v1/speak?model=" << d->voice_name << "&encoding=linear16&sample_rate=8000";
     url = url_stream.str();
 
     /* create the JSON body */
@@ -805,7 +827,10 @@ extern "C" {
     curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
     //curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
 
-    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+    // libcurl adding random byte to the response body that creates white noise to audio file
+    // https://github.com/curl/curl/issues/10525
+    const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
+    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
 
     rc = curl_multi_add_handle(global.multi, conn->easy);
     mcode_test("new_conn: curl_multi_add_handle", rc);
@@ -901,18 +926,18 @@ extern "C" {
           }
           conn->file = nullptr ;
         }
-
-        if (d->cache_filename) {
-          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", d->cache_filename);
-          if (unlink(d->cache_filename) != 0) {
-            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
-              d->cache_filename, errno, strerror(errno));
-          }
-          free(d->cache_filename);
-          d->cache_filename = nullptr ;
-        }
       }
     }
+    // If playback_start has not been sent, delete the file
+    if (d->cache_filename && !d->playback_start_sent) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", d->cache_filename);
+      if (unlink(d->cache_filename) != 0) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
+          d->cache_filename, errno, strerror(errno));
+      }
+      free(d->cache_filename);
+      d->cache_filename = nullptr ;
+    }
     if (d->session_id) {
       switch_core_session_t* session = switch_core_session_locate(d->session_id);
       if (session) {

mod_deepgram_tts/mod_deepgram_tts.c

@@ -19,10 +19,12 @@ static void cleardeepgram(deepgram_t* d, int freeAll) {
   if (d->connect_time_ms) free(d->connect_time_ms);
   if (d->final_response_time_ms) free(d->final_response_time_ms);
   if (d->cache_filename) free(d->cache_filename);
+  if (d->endpoint) free(d->endpoint);
   
 
   d->api_key = NULL;
   d->request_id = NULL;
+  d->endpoint = NULL;
 
   d->reported_model_name = NULL;
   d->reported_model_uuid = NULL;
@@ -84,6 +86,9 @@ static switch_status_t d_speech_feed_tts(switch_speech_handle_t *sh, char *text,
   deepgram_t *d = createOrRetrievePrivateData(sh);
   d->draining = 0;
   d->reads = 0;
+  d->response_code = 0;
+  d->err_msg = NULL;
+  d->playback_start_sent = 0;
 
   switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_feed_tts\n");
 
@@ -118,6 +123,9 @@ static void d_text_param_tts(switch_speech_handle_t *sh, char *param, const char
   if (0 == strcmp(param, "api_key")) {
     if (d->api_key) free(d->api_key);
     d->api_key = strdup(val);
+  } else if (0 == strcmp(param, "endpoint")) {
+    if (d->endpoint) free(d->endpoint);
+    d->endpoint = strdup(val);
   } else if (0 == strcmp(param, "voice")) {
     if (d->voice_name) free(d->voice_name);
     d->voice_name = strdup(val);

mod_deepgram_tts/mod_deepgram_tts.h

@@ -7,6 +7,7 @@
 typedef struct deepgram_data {
   char *voice_name;
   char *api_key;
+  char *endpoint;
 
   /* result data */
   long response_code;
@@ -31,6 +32,7 @@ typedef struct deepgram_data {
   int draining;
   int reads;
   int cache_audio;
+  int playback_start_sent;
 
 	void *conn;
   void *circularBuffer;

mod_dub/ap_http.cpp

@@ -136,7 +136,10 @@ void AudioProducerHttp::start(std::function<void(bool, const std::string&)> call
   /* call this function to close a socket */
   curl_easy_setopt(_easy, CURLOPT_CLOSESOCKETFUNCTION, &AudioProducerHttp::close_socket);
 
-  curl_easy_setopt(_easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+  // libcurl adding random byte to the response body that creates white noise to audio file
+  // https://github.com/curl/curl/issues/10525
+  const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
+  curl_easy_setopt(_easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
 
   /* keep the speed down so we don't have to buffer large amounts*/
   curl_easy_setopt(_easy, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)31415);

mod_dub/mod_dub.c

@@ -159,14 +159,16 @@ static switch_status_t dub_silence_track(switch_core_session_t *session, char* t
   switch_channel_t *channel = switch_core_session_get_channel(session);
   switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, MY_BUG_NAME);
   switch_status_t status = SWITCH_STATUS_FALSE;
-  
+
+  // DH: I found it is much simpler to implement silence as a sequence of remove and add operations  
   if (bug) {
     struct cap_cb *cb =(struct cap_cb *) switch_core_media_bug_get_user_data(bug);
-    status = silence_dub_track(cb, trackName);
+    status = remove_dub_track(cb, trackName);
     if (status != SWITCH_STATUS_SUCCESS) {
-      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_silence_track: error silencing track %s\n", trackName);
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_silence_track: error finding track %s\n", trackName);
       return SWITCH_STATUS_FALSE;
     }
+    status = dub_add_track(session, trackName);
   }
   else {
     switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_silence_track: bug not found\n");
@@ -269,7 +271,7 @@ SWITCH_STANDARD_API(dub_function)
         }
         else {
           char* text = argv[3];
-          int gain = argc > 4 ? atoi(argv[4]) : 0;
+          int gain = argc > 5 ? atoi(argv[5]) : 0;
           switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "sayOnTrack %s gain %d\n", text, gain);
           status = dub_say_on_track(session, track, text, gain);
         }

mod_dub/tts_vendor_parser.cpp

@@ -4,6 +4,60 @@
 #include <switch_json.h>
 #include <map>
 
+
+switch_status_t custom_vendor_parse_text(const std::map<std::string, std::string>& params, const std::string& text, 
+  std::string& url, std::string& body, std::vector<std::string>& headers) {
+  std::string auth_token;
+  std::string voice_name;
+  std::string custom_tts_url;
+  std::string language;
+
+  for (const auto& pair : params) {
+    if (pair.first == "auth_token") {
+      auth_token = pair.second;
+    } else if (pair.first == "voice") {
+      voice_name = pair.second;
+    } else if (pair.first == "custom_tts_url") {
+      custom_tts_url = pair.second;
+    } else if (pair.first == "language") {
+      language = pair.second;
+    }
+  }
+
+  if (custom_tts_url.empty()) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "custom_vendor_parse_text: no custom_tts_url provided\n");
+    return SWITCH_STATUS_FALSE;
+  }
+
+  url = custom_tts_url;
+
+  /* create the JSON body */
+  cJSON * jResult = cJSON_CreateObject();
+  cJSON_AddStringToObject(jResult, "text", text.c_str());
+  cJSON_AddStringToObject(jResult, "type", text.substr(0, 6) == "<speak" ? "ssml" : "text");
+  cJSON_AddNumberToObject(jResult, "samplingRate", 8000);
+  if (!voice_name.empty()) {
+    cJSON_AddStringToObject(jResult, "voice", voice_name.c_str());
+  }
+  if (!language.empty()) {
+    cJSON_AddStringToObject(jResult, "language", language.c_str());
+  }
+  char* _body = cJSON_PrintUnformatted(jResult);
+  body = _body;
+
+  cJSON_Delete(jResult);
+  free(_body);
+
+  // Create headers
+  if (!auth_token.empty()) {
+    headers.push_back("Authorization: Bearer " + auth_token);
+  }
+  headers.push_back("Accept: audio/mp3");
+  headers.push_back("Content-Type: application/json");
+
+  return SWITCH_STATUS_SUCCESS;
+}
+
 switch_status_t rimelabs_parse_text(const std::map<std::string, std::string>& params, const std::string& text, 
   std::string& url, std::string& body, std::vector<std::string>& headers) {
   std::string api_key;
@@ -324,7 +378,10 @@ switch_status_t playht_parse_text(const std::map<std::string, std::string>& para
     cJSON_AddStringToObject(jResult, "quality", quality.c_str());
   }
   if (!speed.empty()) {
-    cJSON_AddNumberToObject(jResult, "speed", atoi(speed.c_str()));
+      double val = strtod(speed.c_str(), NULL);
+      if (val != 0.0) {
+        cJSON_AddNumberToObject(jResult, "speed", val);
+      }
   }
   if (!seed.empty()) {
     cJSON_AddNumberToObject(jResult, "seed", atoi(seed.c_str()));
@@ -476,7 +533,9 @@ switch_status_t tts_vendor_parse_text(const std::string& say, std::string& url,
     return playht_parse_text(params, text, url, body, headers);
   } else if (params["vendor"] == "rimelabs") {
     return rimelabs_parse_text(params, text, url, body, headers);
-  } else {
+  } else if (params["vendor"] == "custom") {
+    return custom_vendor_parse_text(params, text, url, body, headers);
+  }  else {
     switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "tts_vendor_parse_text: There is no available parser for vendor %s\n", params["vendor"]);
     return SWITCH_STATUS_FALSE;
   }

mod_elevenlabs_tts/README.md

@@ -1,11 +1,11 @@
 # mod_google_tts
 
-A Freeswitch module that allows Google Text-to-Speech API to be used as a tts provider.
+A Freeswitch module that allows Eleven Labs' Text-to-Speech API to be used as a tts provider.
 
 ## API
 
 ### Commands
-This freeswitch module does not add any new commands, per se.  Rather, it integrates into the Freeswitch TTS interface such that it is invoked when an application uses the mod_dptools `speak` command with a tts engine of `google_tts` and a voice equal to the language code associated to one of the [supported Wavenet voices](https://cloud.google.com/text-to-speech/docs/voices)
+This freeswitch module does not add any new commands, per se.  Rather, it integrates into the Freeswitch TTS interface such that it is invoked when an application uses the mod_dptools `speak` command with a tts engine of `elevenlabs` and a voice equal to the language code associated to one of the [supported Eleven Labs voices](https://elevenlabs.io/docs/api-reference/query-library)
 
 ### Events
 None.
@@ -13,11 +13,19 @@ None.
 ## Usage
 When using [drachtio-fsrmf](https://www.npmjs.com/package/drachtio-fsmrf), you can access this functionality via the speak method on the 'endpoint' object.
 ```js
-ep.speak({
-    ttsEngine: 'google_tts',
-    voice: 'en-GB-Wavenet-A',
-    text: 'This aggression will not stand'
-  });
+var text = "Hello World";
+await endpoint.speak({
+    "ttsEngine": 'elevenlabs',
+    "voice": "W9OIfHh5DtdYiZUcFiql",
+    "text": `{use_speaker_boost=1,optimize_streaming_latency=4,style=0.5,stability=0.5,similarity_boost=0.75,api_key=XXYYZZ,model_id=eleven_turbo_v2}${text}`,
+});
 ```
-## Examples
-[google_tts.js](../../examples/google_tts.js)
+## Options
+
+Documentation on these options can be found in Eleven Labs API docs: [Voice Settings](https://elevenlabs.io/docs/speech-synthesis/voice-settings)
+
+- use_speaker_boost
+- optimize_streaming_latency
+- style
+- stability
+- similarity_boost

mod_elevenlabs_tts/elevenlabs_glue.cpp

@@ -168,6 +168,27 @@ int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo_t *g) {
   return 0;
 }
 
+static std::string getEnvVar(const std::string& varName) {
+    const char* val = std::getenv(varName.c_str());
+    return val ? std::string(val) : "";
+}
+
+static long getConnectionTimeout() {
+    std::string connectTimeoutStr = getEnvVar("ELEVENLABS_TTS_CURL_CONNECT_TIMEOUT");
+
+    if (connectTimeoutStr.empty()) {
+        connectTimeoutStr = getEnvVar("TTS_CURL_CONNECT_TIMEOUT");
+    }
+
+    long connectTimeout = 10L;
+
+    if (!connectTimeoutStr.empty()) {
+        connectTimeout = std::stol(connectTimeoutStr);
+    }
+
+    return connectTimeout;
+}
+
 static CURL* createEasyHandle(void) {
   CURL* easy = curl_easy_init();
   if(!easy) {
@@ -180,7 +201,7 @@ static CURL* createEasyHandle(void) {
 
   // set connect timeout to 3 seconds and total timeout to 109 seconds
   curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
-  curl_easy_setopt(easy, CURLOPT_TIMEOUT, 10L);
+  curl_easy_setopt(easy, CURLOPT_TIMEOUT, getConnectionTimeout());
 
   return easy ;    
 }
@@ -451,9 +472,6 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
   }
   {
     switch_mutex_lock(el->mutex);
-    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: received data, response %ld\n", 
-      el->response_code);
-
     if (el->response_code > 0 && el->response_code != 200) {
       std::string body((char *) ptr, bytes_received);
       switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: received body %s\n", body.c_str());
@@ -535,6 +553,7 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
 
           switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
           switch_event_fire(&event);
+          el->playback_start_sent = 1;
         }
         else {
           switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
@@ -811,6 +830,12 @@ extern "C" {
     cJSON * jResult = cJSON_CreateObject();
     cJSON_AddStringToObject(jResult, "model_id", el->model_id);
     cJSON_AddStringToObject(jResult, "text", text);
+    if (el->previous_text) {
+      cJSON_AddStringToObject(jResult, "previous_text", el->previous_text);
+    }
+    if (el->next_text) {
+      cJSON_AddStringToObject(jResult, "next_text", el->next_text);
+    }
     if (el->similarity_boost || el->style || el->use_speaker_boost || el->stability) {
       cJSON * jVoiceSettings = cJSON_CreateObject();
       cJSON_AddItemToObject(jResult, "voice_settings", jVoiceSettings);
@@ -886,7 +911,10 @@ extern "C" {
     curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
     //curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
 
-    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+    // libcurl adding random byte to the response body that creates white noise to audio file
+    // https://github.com/curl/curl/issues/10525
+    const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
+    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
 
     rc = curl_multi_add_handle(global.multi, conn->easy);
     mcode_test("new_conn: curl_multi_add_handle", rc);
@@ -987,18 +1015,18 @@ extern "C" {
           }
           conn->file = nullptr ;
         }
-
-        if (el->cache_filename) {
-          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", el->cache_filename);
-          if (unlink(el->cache_filename) != 0) {
-            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
-              el->cache_filename, errno, strerror(errno));
-          }
-          free(el->cache_filename);
-          el->cache_filename = nullptr ;
-        }
       }
     }
+    // If playback_start has not been sent, delete the file
+    if (el->cache_filename && !el->playback_start_sent) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", el->cache_filename);
+      if (unlink(el->cache_filename) != 0) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
+          el->cache_filename, errno, strerror(errno));
+      }
+      free(el->cache_filename);
+      el->cache_filename = nullptr ;
+    }
     if (el->session_id) {
       switch_core_session_t* session = switch_core_session_locate(el->session_id);
       if (session) {

mod_elevenlabs_tts/mod_elevenlabs_tts.c

@@ -19,6 +19,8 @@ static void clearElevenlabs(elevenlabs_t* el, int freeAll) {
   if (el->style) free(el->style);
   if (el->use_speaker_boost) free(el->use_speaker_boost);
   if (el->optimize_streaming_latency) free(el->optimize_streaming_latency);
+  if (el->previous_text) free(el->previous_text);
+  if (el->next_text) free(el->next_text);
   if (el->ct) free(el->ct);
   if (el->reported_latency) free(el->reported_latency);
   if (el->request_id) free(el->request_id);
@@ -36,6 +38,8 @@ static void clearElevenlabs(elevenlabs_t* el, int freeAll) {
   el->style = NULL;
   el->use_speaker_boost = NULL;
   el->optimize_streaming_latency = NULL;
+  el->previous_text = NULL;
+  el->next_text = NULL;
   el->ct = NULL;
   el->reported_latency = NULL;
   el->request_id = NULL;
@@ -106,6 +110,9 @@ static switch_status_t ell_speech_feed_tts(switch_speech_handle_t *sh, char *tex
   elevenlabs_t *el = createOrRetrievePrivateData(sh);
   el->draining = 0;
   el->reads = 0;
+  el->response_code = 0;
+  el->err_msg = NULL;
+  el->playback_start_sent = 0;
   
   switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "ell_speech_feed_tts\n");
   
@@ -171,6 +178,14 @@ static void ell_text_param_tts(switch_speech_handle_t *sh, char *param, const ch
     if (el->optimize_streaming_latency) free(el->optimize_streaming_latency);
     el->optimize_streaming_latency = strdup(val);
   }
+  else if (0 == strcmp(param, "next_text")) {
+    if (el->next_text) free(el->next_text);
+    el->next_text = strdup(val);
+  }
+  else if (0 == strcmp(param, "previous_text")) {
+    if (el->previous_text) free(el->previous_text);
+    el->previous_text = strdup(val);
+  }
   else if (0 == strcmp(param, "write_cache_file") && switch_true(val)) {
     el->cache_audio = 1;
   }

mod_elevenlabs_tts/mod_elevenlabs_tts.h

@@ -17,6 +17,8 @@ struct elevenlabs_data {
   char* style;
   char* use_speaker_boost;
   char* optimize_streaming_latency;
+  char* previous_text;
+  char* next_text;
 
   /* result data */
   long response_code;
@@ -39,6 +41,7 @@ struct elevenlabs_data {
   int draining;
   int reads;
   int cache_audio;
+  int playback_start_sent;
   SpeexResamplerState *resampler;
 };
 

mod_google_transcribe/google_glue_v1.cpp

@@ -233,6 +233,10 @@ static void *SWITCH_THREAD_FUNC grpc_read_thread(switch_thread_t *thread, void *
     auto speech_event_type = response.speech_event_type();
     if (response.has_error()) {
       Status status = response.error();
+      //error 11 is handled in finished session, avoid sending jambonz_transcribe::error event for this here.
+      if (11 == status.code()) {
+        continue;
+      }
       switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "grpc_read_thread: error %s (%d)\n", status.message().c_str(), status.code()) ;
       cJSON* json = cJSON_CreateObject();
       cJSON_AddStringToObject(json, "type", "error");

mod_jambonz_transcribe/audio_pipe.cpp

@@ -229,16 +229,6 @@ int AudioPipe::lws_callback(struct lws *wsi,
 }
 
 
-// static members
-static const lws_retry_bo_t retry = {
-    nullptr,   // retry_ms_table
-    0,         // retry_ms_table_count
-    0,         // conceal_count
-    UINT16_MAX,         // secs_since_valid_ping
-    UINT16_MAX,        // secs_since_valid_hangup
-    0          // jitter_percent
-};
-
 struct lws_context *AudioPipe::context = nullptr;
 std::thread AudioPipe::serviceThread;
 std::mutex AudioPipe::mutex_connects;
@@ -376,6 +366,28 @@ bool AudioPipe::lws_service_thread() {
     { NULL, NULL, 0, 0 }
   };
 
+  uint16_t secs_sinceq_valid_ping = UINT16_MAX;
+  uint16_t secs_since_valid_hangup = UINT16_MAX;
+
+  char* wsVar = std::getenv("WS_PING_INTERVAL");
+  if (wsVar != nullptr) {
+    secs_sinceq_valid_ping = std::atoi(wsVar);
+  }
+
+  wsVar = std::getenv("WS_NO_PONG_HANGUP_INTERVAL");
+  if (wsVar != nullptr) {
+    secs_since_valid_hangup = std::atoi(wsVar);
+  }
+
+  const lws_retry_bo_t retry = {
+    nullptr,                  // retry_ms_table
+    0,                        // retry_ms_table_count
+    0,                        // conceal_count
+    secs_sinceq_valid_ping,   // secs_sinceq_valid_ping
+    secs_since_valid_hangup,  // secs_since_valid_hangup
+    0                         // jitter_percent
+  };
+
   memset(&info, 0, sizeof info); 
   info.port = CONTEXT_PORT_NO_LISTEN; 
   info.options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT;

mod_jambonz_transcribe/jb_transcribe_glue.cpp

@@ -145,7 +145,7 @@ namespace {
     cJSON_AddStringToObject(json, "format", "raw");
     cJSON_AddStringToObject(json, "encoding", "LINEAR16");
     cJSON_AddBoolToObject(json, "interimResults", tech_pvt->interim);
-    cJSON_AddNumberToObject(json, "sampleRateHz", 8000);
+    cJSON_AddNumberToObject(json, "sampleRateHz", tech_pvt->sampling);
     if (var = switch_channel_get_variable(channel, "JAMBONZ_STT_OPTIONS")) {
       cJSON* jOptions = cJSON_Parse(var);
       if (jOptions)  {
@@ -353,7 +353,7 @@ extern "C" {
   }
 	
   switch_status_t jb_transcribe_session_init(switch_core_session_t *session, 
-    responseHandler_t responseHandler, uint32_t samples_per_second, uint32_t channels, 
+    responseHandler_t responseHandler, uint32_t samples_per_second, int desiredSampling, uint32_t channels, 
     char* lang, int interim, char* bugname, void **ppUserData)
   {    	
     int err;
@@ -365,7 +365,7 @@ extern "C" {
       return SWITCH_STATUS_FALSE;
     }
 
-    if (SWITCH_STATUS_SUCCESS != fork_data_init(tech_pvt, session, samples_per_second, 8000, channels, lang, interim, bugname, responseHandler)) {
+    if (SWITCH_STATUS_SUCCESS != fork_data_init(tech_pvt, session, samples_per_second, desiredSampling, channels, lang, interim, bugname, responseHandler)) {
       destroy_tech_pvt(tech_pvt);
       return SWITCH_STATUS_FALSE;
     }
@@ -403,6 +403,7 @@ extern "C" {
         }
         jambonz::AudioPipe *pAudioPipe = static_cast<jambonz::AudioPipe *>(tech_pvt->pAudioPipe);
         if (pAudioPipe) reaper(tech_pvt);
+        switch_mutex_unlock(tech_pvt->mutex);
         destroy_tech_pvt(tech_pvt);
         switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) jb_transcribe_session_stop, bug removed\n", id);
         return SWITCH_STATUS_SUCCESS;
@@ -410,6 +411,7 @@ extern "C" {
       else {
         switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "jb_transcribe_session_stop: race condition, previous close completed\n");
       }
+      switch_mutex_unlock(tech_pvt->mutex);
     }
     return SWITCH_STATUS_FALSE;
   }

mod_jambonz_transcribe/jb_transcribe_glue.h

@@ -6,7 +6,7 @@ int parse_ws_uri(switch_channel_t *channel, const char* szServerUri, char* host,
 switch_status_t jb_transcribe_init();
 switch_status_t jb_transcribe_cleanup();
 switch_status_t jb_transcribe_session_init(switch_core_session_t *session, responseHandler_t responseHandler, 
-		uint32_t samples_per_second, uint32_t channels, char* lang, int interim, char* bugname, void **ppUserData);
+		uint32_t samples_per_second, int desiredSampling, uint32_t channels, char* lang, int interim, char* bugname, void **ppUserData);
 switch_status_t jb_transcribe_session_stop(switch_core_session_t *session, int channelIsClosing, char* bugname);
 switch_bool_t jb_transcribe_frame(switch_core_session_t *session, switch_media_bug_t *bug);
 

mod_jambonz_transcribe/mod_jambonz_transcribe.c

@@ -73,6 +73,8 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
 	switch_codec_implementation_t read_impl = { 0 };
 	void *pUserData;
 	uint32_t samples_per_second;
+	uint32_t desiredSampling = 8000;
+	const char* var;
 
   if (!switch_channel_get_variable(channel, "JAMBONZ_STT_URL")) {
     switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Missing JAMBONZ_STT_URL channel var\n");
@@ -91,8 +93,11 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
 	}
 
 	samples_per_second = !strcasecmp(read_impl.iananame, "g722") ? read_impl.actual_samples_per_second : read_impl.samples_per_second;
-
-	if (SWITCH_STATUS_FALSE == jb_transcribe_session_init(session, responseHandler, samples_per_second, flags & SMBF_STEREO ? 2 : 1, lang, interim, bugname, &pUserData)) {
+	var = switch_channel_get_variable(channel, "JAMBONZ_STT_SAMPLING");
+	if (var != NULL) {
+		desiredSampling = atoi(var);
+	}
+	if (SWITCH_STATUS_FALSE == jb_transcribe_session_init(session, responseHandler, samples_per_second, desiredSampling, flags & SMBF_STEREO ? 2 : 1, lang, interim, bugname, &pUserData)) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing jb speech session.\n");
 		return SWITCH_STATUS_FALSE;
 	}
@@ -100,7 +105,7 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
 		return status;
 	}
   switch_channel_set_private(channel, bugname, bug);
-	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "added media bug for jb transcribe\n");
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "added media bug for jb transcribe: %s\n", bugname);
 
 	return SWITCH_STATUS_SUCCESS;
 }
@@ -146,7 +151,7 @@ SWITCH_STANDARD_API(jb_transcribe_function)
 		if ((lsession = switch_core_session_locate(argv[0]))) {
 			if (!strcasecmp(argv[1], "stop")) {
 				char *bugname = argc > 2 ? argv[2] : MY_BUG_NAME;
-    		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "stop transcribing\n");
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "stop transcribing %s\n", bugname);
 				status = do_stop(lsession, bugname);
 			} else if (!strcasecmp(argv[1], "start")) {
         char* lang = argv[2];

mod_playht_tts/mod_playht_tts.c

@@ -14,6 +14,10 @@ static void clearPlayht(playht_t* p, int freeAll) {
   if (p->seed) free(p->seed);
   if (p->temperature) free(p->temperature);
   if (p->voice_engine) free(p->voice_engine);
+  if (p->synthesize_url) free(p->synthesize_url);
+  if (p->language) free(p->language);
+  if (p->top_p) free(p->top_p);
+  if (p->repetition_penalty) free(p->repetition_penalty);
   if (p->emotion) free(p->emotion);
   if (p->voice_guidance) free(p->voice_guidance);
   if (p->style_guidance) free(p->style_guidance);
@@ -36,6 +40,10 @@ static void clearPlayht(playht_t* p, int freeAll) {
   p->seed = NULL;
   p->temperature = NULL;
   p->voice_engine = NULL;
+  p->synthesize_url = NULL;
+  p->language = NULL;
+  p->top_p = NULL;
+  p->repetition_penalty = NULL;
   p->emotion = NULL;
   p->voice_guidance = NULL;
   p->style_guidance = NULL;
@@ -99,6 +107,9 @@ static switch_status_t p_speech_feed_tts(switch_speech_handle_t *sh, char *text,
   playht_t *p = createOrRetrievePrivateData(sh);
   p->draining = 0;
   p->reads = 0;
+  p->response_code = 0;
+  p->err_msg = NULL;
+  p->playback_start_sent = 0;
 
   switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "p_speech_feed_tts\n");
 
@@ -151,6 +162,18 @@ static void p_text_param_tts(switch_speech_handle_t *sh, char *param, const char
   } else if (0 == strcmp(param, "voice_engine")) {
     if (p->voice_engine) free(p->voice_engine);
     p->voice_engine = strdup(val);
+  } else if (0 == strcmp(param, "synthesize_url")) {
+    if (p->synthesize_url) free(p->synthesize_url);
+    p->synthesize_url = strdup(val);
+  } else if (0 == strcmp(param, "language")) {
+    if (p->language) free(p->language);
+    p->language = strdup(val);
+  } else if (0 == strcmp(param, "top_p")) {
+    if (p->top_p) free(p->top_p);
+    p->top_p = strdup(val);
+  } else if (0 == strcmp(param, "repetition_penalty")) {
+    if (p->repetition_penalty) free(p->repetition_penalty);
+    p->repetition_penalty = strdup(val);
   } else if (0 == strcmp(param, "emotion")) {
     if (p->emotion) free(p->emotion);
     p->emotion = strdup(val);

mod_playht_tts/mod_playht_tts.h

@@ -11,10 +11,14 @@ typedef struct playht_data {
   char *seed;
   char *temperature;
   char *voice_engine;
+  char *synthesize_url;
+  char *language;
   char *emotion;
   char *voice_guidance;
   char *style_guidance;
   char *text_guidance;
+  char *top_p;
+  char *repetition_penalty;
 
   /* result data */
   long response_code;
@@ -31,6 +35,7 @@ typedef struct playht_data {
   int draining;
   int reads;
   int cache_audio;
+  int playback_start_sent;
 
 	void *conn;
   void *circularBuffer;

mod_playht_tts/playht_glue.cpp

@@ -64,6 +64,27 @@ std::string secondsToMillisecondsString(double seconds) {
     return std::to_string(milliseconds_long);
 }
 
+static std::string getEnvVar(const std::string& varName) {
+    const char* val = std::getenv(varName.c_str());
+    return val ? std::string(val) : "";
+}
+
+static long getConnectionTimeout() {
+    std::string connectTimeoutStr = getEnvVar("PLAYHT_TTS_CURL_CONNECT_TIMEOUT");
+
+    if (connectTimeoutStr.empty()) {
+        connectTimeoutStr = getEnvVar("TTS_CURL_CONNECT_TIMEOUT");
+    }
+
+    long connectTimeout = 20L;
+
+    if (!connectTimeoutStr.empty()) {
+        connectTimeout = std::stol(connectTimeoutStr);
+    }
+
+    return connectTimeout;
+}
+
 static CURL* createEasyHandle(void) {
   CURL* easy = curl_easy_init();
   if(!easy) {
@@ -76,9 +97,10 @@ static CURL* createEasyHandle(void) {
 
   // set connect timeout to 3 seconds and total timeout to 109 seconds
   curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
-  curl_easy_setopt(easy, CURLOPT_TIMEOUT, 10L);
+  //For long text, PlayHT took more than 20 seconds to complete the download.
+  curl_easy_setopt(easy, CURLOPT_TIMEOUT, getConnectionTimeout());
 
-  return easy ;    
+  return easy ;
 }
 
 static void cleanupConn(ConnInfo_t *conn) {
@@ -451,12 +473,12 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
       return 0;
     }
 
-    /* cache file will stay in the mp3 format for size (smaller) and simplicity */
-    if (conn->file) fwrite(data, sizeof(uint8_t), bytes_received, conn->file);
-
     pcm_data = convert_mp3_to_linear(conn, data, bytes_received);
     size_t bytesResampled = pcm_data.size() * sizeof(uint16_t);
 
+    /* cache same data to avoid streaming and cached audio quality is different*/
+    if (conn->file) fwrite(pcm_data.data(), sizeof(uint8_t), bytesResampled, conn->file);
+
     // Resize the buffer if necessary
     if (cBuffer->capacity() - cBuffer->size() < (bytesResampled / sizeof(uint16_t))) {
       //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "write_cb growing buffer\n"); 
@@ -510,6 +532,7 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
 
           switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
           switch_event_fire(&event);
+          p->playback_start_sent = 1;
         }
         else {
           switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
@@ -757,7 +780,7 @@ extern "C" {
     }
 
     /* format url*/
-    std::string url = "https://api.play.ht/api/v2/tts/stream";
+    std::string url = p->synthesize_url;
 
     /* create the JSON body */
     cJSON * jResult = cJSON_CreateObject();
@@ -772,10 +795,10 @@ extern "C" {
       cJSON_AddStringToObject(jResult, "quality", p->quality);
     }
     if (p->speed) {
-      cJSON_AddNumberToObject(jResult, "speed", atoi(p->speed));
-    }
-    if (p->seed) {
-      cJSON_AddNumberToObject(jResult, "seed", atoi(p->seed));
+      double val = strtod(p->speed, NULL);
+      if (val != 0.0) {
+        cJSON_AddNumberToObject(jResult, "speed", val);
+      }
     }
     if (p->temperature) {
       cJSON_AddNumberToObject(jResult, "temperature", std::strtof(p->temperature, nullptr));
@@ -792,6 +815,15 @@ extern "C" {
     if (p->text_guidance) {
       cJSON_AddNumberToObject(jResult, "text_guidance", atoi(p->text_guidance));
     }
+    if (strcmp(p->voice_engine, "Play3.0") == 0 && p->language) {
+      cJSON_AddStringToObject(jResult, "language", p->language);
+    }
+    if (p->repetition_penalty) {
+      cJSON_AddNumberToObject(jResult, "repetition_penalty", std::strtof(p->repetition_penalty, nullptr));
+    }
+    if (p->top_p) {
+      cJSON_AddNumberToObject(jResult, "top_p", std::strtof(p->top_p, nullptr));
+    }
     char *json = cJSON_PrintUnformatted(jResult);
 
     cJSON_Delete(jResult);
@@ -863,9 +895,12 @@ extern "C" {
 
     /* call this function to close a socket */
     curl_easy_setopt(easy, CURLOPT_CLOSESOCKETFUNCTION, close_socket);
+    // Play3.0 voice engine doesn't need authorization
+    if (strcmp(p->voice_engine, "Play3.0") != 0) {
+      conn->hdr_list = curl_slist_append(conn->hdr_list, api_key_stream.str().c_str());
+      conn->hdr_list = curl_slist_append(conn->hdr_list, user_id_stream.str().c_str());
+    }
 
-    conn->hdr_list = curl_slist_append(conn->hdr_list, api_key_stream.str().c_str());
-    conn->hdr_list = curl_slist_append(conn->hdr_list, user_id_stream.str().c_str());
     conn->hdr_list = curl_slist_append(conn->hdr_list, "Accept: audio/mpeg");
     conn->hdr_list = curl_slist_append(conn->hdr_list, "Content-Type: application/json");
     curl_easy_setopt(easy, CURLOPT_HTTPHEADER, conn->hdr_list);
@@ -873,7 +908,10 @@ extern "C" {
     curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
     //curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
 
-    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+    // libcurl adding random byte to the response body that creates white noise to audio file
+    // https://github.com/curl/curl/issues/10525
+    const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
+    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
 
     rc = curl_multi_add_handle(global.multi, conn->easy);
     mcode_test("new_conn: curl_multi_add_handle", rc);
@@ -927,11 +965,14 @@ extern "C" {
 
   switch_status_t playht_speech_flush_tts(playht_t* p) {
     bool download_complete = p->response_code == 200;
-    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "playht_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;  
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "playht_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;
     ConnInfo_t *conn = (ConnInfo_t *) p->conn;
     CircularBuffer_t *cBuffer = (CircularBuffer_t *) p->circularBuffer;
+    // In multi threads, only delete the circular buffer when write and read buffer action finished using it.
+    switch_mutex_lock(p->mutex);
     delete cBuffer;
     p->circularBuffer = nullptr ;
+    switch_mutex_unlock(p->mutex);
 
     if (conn) {
       conn->flushed = true;
@@ -943,18 +984,18 @@ extern "C" {
           }
           conn->file = nullptr ;
         }
-
-        if (p->cache_filename) {
-          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", p->cache_filename);
-          if (unlink(p->cache_filename) != 0) {
-            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
-              p->cache_filename, errno, strerror(errno));
-          }
-          free(p->cache_filename);
-          p->cache_filename = nullptr ;
-        }
       }
     }
+    // if playback event has not been sent, delete the file.
+    if (p->cache_filename && !p->playback_start_sent) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", p->cache_filename);
+      if (unlink(p->cache_filename) != 0) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
+          p->cache_filename, errno, strerror(errno));
+      }
+      free(p->cache_filename);
+      p->cache_filename = nullptr ;
+    }
     if (p->session_id) {
       switch_core_session_t* session = switch_core_session_locate(p->session_id);
       if (session) {

mod_rimelabs_tts/mod_rimelabs_tts.c

@@ -82,6 +82,9 @@ static switch_status_t d_speech_feed_tts(switch_speech_handle_t *sh, char *text,
   rimelabs_t *d = createOrRetrievePrivateData(sh);
   d->draining = 0;
   d->reads = 0;
+  d->response_code = 0;
+  d->err_msg = NULL;
+  d->playback_start_sent = 0;
 
   switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_feed_tts\n");
 

mod_rimelabs_tts/mod_rimelabs_tts.h

@@ -25,6 +25,7 @@ typedef struct rimelabs_data {
   int draining;
   int reads;
   int cache_audio;
+  int playback_start_sent;
 
 	void *conn;
   void *circularBuffer;

mod_rimelabs_tts/rimelabs_glue.cpp

@@ -66,6 +66,27 @@ std::string secondsToMillisecondsString(double seconds) {
     return std::to_string(milliseconds_long);
 }
 
+static std::string getEnvVar(const std::string& varName) {
+    const char* val = std::getenv(varName.c_str());
+    return val ? std::string(val) : "";
+}
+
+static long getConnectionTimeout() {
+    std::string connectTimeoutStr = getEnvVar("RIMELABS_TTS_CURL_CONNECT_TIMEOUT");
+
+    if (connectTimeoutStr.empty()) {
+        connectTimeoutStr = getEnvVar("TTS_CURL_CONNECT_TIMEOUT");
+    }
+
+    long connectTimeout = 10L;
+
+    if (!connectTimeoutStr.empty()) {
+        connectTimeout = std::stol(connectTimeoutStr);
+    }
+
+    return connectTimeout;
+}
+
 static CURL* createEasyHandle(void) {
   CURL* easy = curl_easy_init();
   if(!easy) {
@@ -78,7 +99,7 @@ static CURL* createEasyHandle(void) {
 
   // set connect timeout to 3 seconds and total timeout to 109 seconds
   curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
-  curl_easy_setopt(easy, CURLOPT_TIMEOUT, 10L);
+  curl_easy_setopt(easy, CURLOPT_TIMEOUT, getConnectionTimeout());
 
   return easy ;    
 }
@@ -493,6 +514,7 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
 
           switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
           switch_event_fire(&event);
+          d->playback_start_sent = 1;
         }
         else {
           switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
@@ -804,7 +826,10 @@ extern "C" {
     curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
     //curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
 
-    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+    // libcurl adding random byte to the response body that creates white noise to audio file
+    // https://github.com/curl/curl/issues/10525
+    const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
+    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
 
     rc = curl_multi_add_handle(global.multi, conn->easy);
     mcode_test("new_conn: curl_multi_add_handle", rc);
@@ -900,18 +925,18 @@ extern "C" {
           }
           conn->file = nullptr ;
         }
-
-        if (d->cache_filename) {
-          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", d->cache_filename);
-          if (unlink(d->cache_filename) != 0) {
-            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
-              d->cache_filename, errno, strerror(errno));
-          }
-          free(d->cache_filename);
-          d->cache_filename = nullptr ;
-        }
       }
     }
+    // if playback_start event has not been sent, delete the file
+    if (d->cache_filename && !d->playback_start_sent) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", d->cache_filename);
+      if (unlink(d->cache_filename) != 0) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
+          d->cache_filename, errno, strerror(errno));
+      }
+      free(d->cache_filename);
+      d->cache_filename = nullptr ;
+    }
     if (d->session_id) {
       switch_core_session_t* session = switch_core_session_locate(d->session_id);
       if (session) {

mod_vad_detect/LICENSE

@@ -0,0 +1,8 @@
+Copyright 2023, Drachtio Communications Services, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

mod_vad_detect/Makefile.am

@@ -0,0 +1,9 @@
+include $(top_srcdir)/build/modmake.rulesam
+MODNAME=mod_vad_detect
+
+mod_LTLIBRARIES = mod_vad_detect.la
+mod_vad_detect_la_SOURCES  = mod_vad_detect.c 
+mod_vad_detect_la_CFLAGS   = $(AM_CFLAGS)
+mod_vad_detect_la_CXXFLAGS = $(AM_CXXFLAGS) -std=c++11
+mod_vad_detect_la_LIBADD   = $(switch_builddir)/libfreeswitch.la
+mod_vad_detect_la_LDFLAGS  = -avoid-version -module -no-undefined

mod_vad_detect/README.md

@@ -0,0 +1,41 @@
+# mod_vad_detect
+
+mod_vad_detect is a Freeswitch module designed to detect the start and end points of speech in a conversation.
+
+## API
+
+### Commands
+This Freeswitch module provides the following API commands:
+
+```
+uuid_vad_detect start [one-shot|continuous] mode silence-ms voice-ms [bugname]
+```
+This command attaches a media bug to a channel and starts Voice Activity Detection (VAD).
+- `uuid`: Unique identifier of the Freeswitch channel.
+- `one-shot`: Detects the start of speech, sends an event titled `vad_detect:start_talking`, and ceases listening.
+- `continuous`: Continuously listens and reports all events, including `vad_detect:start_talking` and `vad_detect:stop_talking`.
+- `mode`:
+    - -1 ("disable fvad, use native")
+    - 0 ("quality")
+    - 1 ("low bitrate")
+    - 2 ("aggressive")
+    - 3  ("very aggressive")
+- `silence-ms`: number of milliseconds of silence that must come to transition from talking to stop talking
+- `voice-ms`: number of milliseconds of voice that must come to transition to start talking
+
+```
+uuid_vad_detect stop [bugname]
+```
+This command halts VAD detection on the specified channel.
+
+### Channel Variables
+
+### Events
+- `vad_detect:start_talking`: Indicates the detection of the start of speech.
+- `vad_detect:stop_talking`: Indicates the detection of the end of speech.
+
+## Usage
+When utilizing [drachtio-fsrmf](https://www.npmjs.com/package/drachtio-fsmrf), you can employ this API command via the api method found on the 'endpoint' object. Here is an example:
+```js
+ep.api('uuid_vad_detect', `${ep.uuid} start one-shot 2 150 250 vad_detect`);
+```

mod_vad_detect/mod_vad_detect.c

@@ -0,0 +1,243 @@
+#include "mod_vad_detect.h"
+
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_vad_detect_shutdown);
+SWITCH_MODULE_LOAD_FUNCTION(mod_vad_detect_load);
+SWITCH_MODULE_DEFINITION(mod_vad_detect, mod_vad_detect_load, mod_vad_detect_shutdown, NULL);
+
+static void responseHandler(switch_core_session_t* session, switch_vad_state_t state, const char* bugname) {
+  switch_event_t *event;
+  switch_channel_t *channel = switch_core_session_get_channel(session);
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "responseHandler event %s, detect %s.\n", VAD_EVENT_DETECTION,
+    state == SWITCH_VAD_STATE_START_TALKING ? "start_talking" : "stop_talking");
+  switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, VAD_EVENT_DETECTION);
+  switch_channel_event_set_data(channel, event);
+  switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "detected-event", state == SWITCH_VAD_STATE_START_TALKING ? "start_talking" : "stop_talking");
+  if (bugname) switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "media-bugname", bugname);
+  switch_event_fire(&event);
+}
+
+static void cleanVadDetect(private_t* u) {
+  if (u) {
+    if (u->vad) {
+      switch_vad_destroy(&u->vad);
+      u->vad = NULL;
+    }
+    if (u->bugname) free(u->bugname);
+    if (u->strategy) free(u->strategy);
+    if (u->sessionId) free(u->sessionId);
+  }
+}
+
+static switch_status_t do_stop(switch_core_session_t *session, char* bugname) {
+  switch_status_t status = SWITCH_STATUS_SUCCESS;
+
+  switch_channel_t *channel = switch_core_session_get_channel(session);
+  switch_media_bug_t *bug = switch_channel_get_private(channel, bugname);
+  if (bug) {
+    switch_channel_set_private(channel, bugname, NULL);
+    switch_core_media_bug_remove(session, &bug);
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "do_stop: stopped vad detection.\n");
+  }
+
+  return status;
+}
+
+static void *SWITCH_THREAD_FUNC stop_thread(switch_thread_t *thread, void *obj) {
+  private_t* u = (private_t*) obj;
+  switch_core_session_t* session = switch_core_session_locate(u->sessionId);
+  do_stop(session, u->bugname);
+  switch_core_session_rwunlock(session);
+  return NULL;
+}
+
+static switch_bool_t capture_callback(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type)
+{
+  switch_core_session_t *session = switch_core_media_bug_get_session(bug);
+  switch_memory_pool_t *pool = switch_core_session_get_pool(session);
+  private_t* userData = (private_t*) user_data;
+
+  switch (type) {
+  case SWITCH_ABC_TYPE_INIT:
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Got SWITCH_ABC_TYPE_INIT.\n");
+    break;
+
+  case SWITCH_ABC_TYPE_CLOSE:
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Got SWITCH_ABC_TYPE_CLOSE.\n");
+    cleanVadDetect(userData);
+    break;
+  
+  case SWITCH_ABC_TYPE_READ:
+    {
+      uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
+      switch_threadattr_t *thd_attr = NULL;
+      switch_frame_t frame;
+      memset(&frame, 0, sizeof(frame));
+      frame.data = data;
+      frame.buflen = SWITCH_RECOMMENDED_BUFFER_SIZE;
+      if (userData->stopping) {
+        return SWITCH_TRUE;
+      }
+      while (switch_core_media_bug_read(bug, &frame, SWITCH_TRUE) == SWITCH_STATUS_SUCCESS && !switch_test_flag((&frame), SFF_CNG)) {
+        if (frame.datalen) {
+          switch_vad_state_t state = switch_vad_process(userData->vad, (int16_t*) frame.data, frame.samples);
+          switch (state)
+          {
+            case SWITCH_VAD_STATE_START_TALKING:
+            case SWITCH_VAD_STATE_STOP_TALKING:
+              responseHandler(session, state, userData->bugname);
+              if (!strcasecmp(userData->strategy, "one-shot")) {
+                userData->stopping = 1;
+                // create the stop thread
+                switch_threadattr_create(&thd_attr, pool);
+                switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE);
+                switch_thread_create(&userData->thread, thd_attr, stop_thread, userData, pool);
+              }
+            break;
+
+            case SWITCH_VAD_STATE_TALKING:
+            case SWITCH_VAD_STATE_NONE:
+            default:
+              break;
+          }
+        }
+      }
+    }
+    break;
+
+  case SWITCH_ABC_TYPE_WRITE:
+  default:
+    break;
+  }
+
+  return SWITCH_TRUE;
+}
+
+static switch_status_t start_capture(switch_core_session_t *session, switch_media_bug_flag_t flags, 
+  char* action, int mode, uint32_t silence_ms, uint32_t voice_ms, char* bugname)
+{
+  switch_channel_t *channel = switch_core_session_get_channel(session);
+  switch_media_bug_t *bug;
+  switch_status_t status = SWITCH_STATUS_SUCCESS; 
+  private_t* userData = (private_t *) switch_core_session_alloc(session, sizeof(*userData));
+  switch_codec_implementation_t read_impl = { 0 };
+  uint32_t samples_per_second;
+
+  if (switch_channel_get_private(channel, bugname)) {
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "removing bug from previous vad detection\n");
+    do_stop(session, bugname);
+  }
+  
+
+  if (switch_channel_pre_answer(channel) != SWITCH_STATUS_SUCCESS) {
+    return SWITCH_STATUS_FALSE;
+  }
+  switch_core_session_get_read_impl(session, &read_impl);
+  samples_per_second = !strcasecmp(read_impl.iananame, "g722") ? read_impl.actual_samples_per_second : read_impl.samples_per_second;
+
+  userData->stopping = 0;
+  userData->vad = switch_vad_init(samples_per_second, 1);
+  if (userData->vad) {
+    userData->bugname = strdup(bugname);
+    userData->strategy = strdup(action);
+    userData->sessionId = strdup(switch_core_session_get_uuid(session));
+    switch_vad_set_mode(userData->vad, mode);
+    switch_vad_set_param(userData->vad, "silence_ms", silence_ms);
+    switch_vad_set_param(userData->vad, "voice_ms", voice_ms);
+    // switch_vad_set_param(userData->vad, "debug", 1);
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "configured vad mode %d, silence_ms %d, voice_ms %d\n", 
+      mode, silence_ms, voice_ms);
+    if ((status = switch_core_media_bug_add(session, bugname, NULL, capture_callback, userData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Failed to initiate vad resource\n");
+      return status;
+    }
+    switch_channel_set_private(channel, bugname, bug);
+  }
+  switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Successfully initiated vad resource\n");
+  return SWITCH_STATUS_SUCCESS;
+}
+
+#define VAD_API_SYNTAX "<uuid> [start|stop] [one-shot|continuous] mode silence-ms voice-ms [bugname]"
+SWITCH_STANDARD_API(vad_detect_function) {
+  char *mycmd = NULL, *argv[7] = { 0 };
+  int argc = 0;
+  switch_status_t status = SWITCH_STATUS_FALSE;
+  switch_media_bug_flag_t flags = SMBF_READ_STREAM /* | SMBF_WRITE_STREAM | SMBF_READ_PING */;
+
+  if (!zstr(cmd) && (mycmd = strdup(cmd))) {
+    argc = switch_separate_string(mycmd, ' ', argv, (sizeof(argv) / sizeof(argv[0])));
+  }
+
+  if (zstr(cmd) || 
+      (!strcasecmp(argv[1], "stop") && argc < 2) ||
+      (!strcasecmp(argv[1], "start") && argc < 3) ||
+      zstr(argv[0])) {
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error with command %s %s %s.\n", cmd, argv[0], argv[1]);
+    stream->write_function(stream, "-USAGE: %s\n", VAD_API_SYNTAX);
+    goto done;
+  } else {
+    switch_core_session_t *lsession = NULL;
+
+    if ((lsession = switch_core_session_locate(argv[0]))) {
+      if (!strcasecmp(argv[1], "stop")) {
+        char *bugname = argc > 2 ? argv[2] : MY_BUG_NAME;
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "stop vad detection %s\n", bugname);
+        status = do_stop(lsession, bugname);
+      } else if (!strcasecmp(argv[1], "start")) {
+        char* action = argv[2];
+        int mode = atoi(argv[3]);
+        uint32_t silence_ms = atoi(argv[4]);
+        uint32_t voice_ms = atoi(argv[5]);
+        char *bugname = argc > 6 ? argv[6] : MY_BUG_NAME;
+
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "start vad detect action: %s mode: %d silence_ms: %d voice_ms: %d bugname: %s.\n",
+          action, mode, silence_ms, voice_ms, bugname);
+        status = start_capture(lsession, flags, action, mode, silence_ms, voice_ms, bugname);
+      }
+      switch_core_session_rwunlock(lsession);
+    }
+  }
+
+  if (status == SWITCH_STATUS_SUCCESS) {
+    stream->write_function(stream, "+OK Success\n");
+  } else {
+    stream->write_function(stream, "-ERR Operation Failed\n");
+  }
+
+  done:
+
+  switch_safe_free(mycmd);
+  return SWITCH_STATUS_SUCCESS;
+}
+SWITCH_MODULE_LOAD_FUNCTION(mod_vad_detect_load)
+{
+  switch_api_interface_t *api_interface;
+
+  /* create/register custom event message type */
+  if (switch_event_reserve_subclass(VAD_EVENT_DETECTION) != SWITCH_STATUS_SUCCESS) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't register subclass %s!\n", VAD_EVENT_DETECTION);
+    return SWITCH_STATUS_TERM;
+  }
+
+  /* connect my internal structure to the blank pointer passed to me */
+  *module_interface = switch_loadable_module_create_module_interface(pool, modname);
+
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "VAD dectetion API loading..\n");
+
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "VAD detection API successfully loaded\n");
+
+  SWITCH_ADD_API(api_interface, "uuid_vad_detect", "VAD detection API", vad_detect_function, VAD_API_SYNTAX);
+  switch_console_set_complete("add uuid_vad_detect start [one-shot|continuous] mode silence-ms voice-ms [bugname]");
+  switch_console_set_complete("add uuid_vad_detect stop [bugname]");
+
+  /* indicate that the module should continue to be loaded */
+  return SWITCH_STATUS_SUCCESS;
+}
+
+/*
+  Called when the system shuts down
+  Macro expands to: switch_status_t mod_vad_detect_shutdown() */
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_vad_detect_shutdown)
+{
+  switch_event_free_subclass(VAD_EVENT_DETECTION);
+  return SWITCH_STATUS_SUCCESS;
+}

mod_vad_detect/mod_vad_detect.h

@@ -0,0 +1,22 @@
+#ifndef __MOD_VAD_DETECT_H__
+#define __MOD_VAD_DETECT_H__
+
+#include <switch.h>
+
+#define MY_BUG_NAME "vad_detect"
+#define VAD_EVENT_DETECTION "vad_detect:detection"
+
+typedef struct private_data
+{
+  char *bugname;
+  char *strategy;
+  char *sessionId;
+
+  int stopping;
+
+  switch_vad_t *vad;
+  switch_thread_t* thread;
+
+} private_t;
+
+#endif

mod_verbio_transcribe/LICENSE

@@ -0,0 +1,8 @@
+Copyright 2023, Drachtio Communications Services, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

mod_verbio_transcribe/Makefile.am

@@ -0,0 +1,10 @@
+include $(top_srcdir)/build/modmake.rulesam
+MODNAME=mod_verbio_transcribe
+
+mod_LTLIBRARIES = mod_verbio_transcribe.la
+mod_verbio_transcribe_la_SOURCES  = mod_verbio_transcribe.c verbio_glue.cpp
+mod_verbio_transcribe_la_CFLAGS   = $(AM_CFLAGS)
+mod_verbio_transcribe_la_CXXFLAGS = -I $(top_srcdir)/libs/verbio-asr-grpc-api/stubs $(AM_CXXFLAGS) -std=c++17
+
+mod_verbio_transcribe_la_LIBADD   = $(switch_builddir)/libfreeswitch.la
+mod_verbio_transcribe_la_LDFLAGS  = -avoid-version -module -no-undefined -shared -lstdc++ -lboost_system -lboost_thread

mod_verbio_transcribe/README.md

@@ -0,0 +1,3 @@
+# mod_verbio_transcribe
+
+A Freeswitch module that generates real-time transcriptions on a Freeswitch channel by using the Verbio Microsoft streaming transcription API

mod_verbio_transcribe/mod_verbio_transcribe.c

@@ -0,0 +1,201 @@
+/* 
+ *
+ * mod_verbio_transcribe.c -- Freeswitch module for using verbio streaming transcribe api
+ *
+ */
+#include "mod_verbio_transcribe.h"
+#include "verbio_glue.h"
+
+/* Prototypes */
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_verbio_transcribe_shutdown);
+SWITCH_MODULE_LOAD_FUNCTION(mod_verbio_transcribe_load);
+
+SWITCH_MODULE_DEFINITION(mod_verbio_transcribe, mod_verbio_transcribe_load, mod_verbio_transcribe_shutdown, NULL);
+
+static switch_status_t do_stop(switch_core_session_t *session, char* bugname);
+
+static void responseHandler(switch_core_session_t* session, const char* eventName, const char * json, const char* bugname, int finished) {
+  switch_event_t *event;
+  switch_channel_t *channel = switch_core_session_get_channel(session);
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "responseHandler event %s, body %s.\n", eventName, json);
+  switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, eventName);
+  switch_channel_event_set_data(channel, event);
+  switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "transcription-vendor", "verbio");
+  switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "transcription-session-finished", finished ? "true" : "false");
+  if (finished) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "responseHandler returning event %s, from finished recognition session\n", eventName);
+  }
+  if (json) switch_event_add_body(event, "%s", json);
+  if (bugname) switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "media-bugname", bugname);
+  switch_event_fire(&event);
+}
+
+static switch_bool_t capture_callback(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type)
+{
+  switch_core_session_t *session = switch_core_media_bug_get_session(bug);
+
+  switch (type) {
+  case SWITCH_ABC_TYPE_INIT:
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got SWITCH_ABC_TYPE_INIT.\n");
+    break;
+
+  case SWITCH_ABC_TYPE_CLOSE:
+    {
+      struct cap_cb* cb = (struct cap_cb*) switch_core_media_bug_get_user_data(bug);
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Got SWITCH_ABC_TYPE_CLOSE.\n");
+
+      verbio_speech_session_cleanup(session, 1, cb->bugname);
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Finished SWITCH_ABC_TYPE_CLOSE.\n");
+    }
+    break;
+  
+  case SWITCH_ABC_TYPE_READ:
+    return verbio_speech_frame(bug, user_data);
+    break;
+
+  case SWITCH_ABC_TYPE_WRITE:
+  default:
+    break;
+  }
+
+  return SWITCH_TRUE;
+}
+
+static switch_status_t start_capture(switch_core_session_t *session, switch_media_bug_flag_t flags, char* lang, int interim, char* bugname)
+{
+  switch_channel_t *channel = switch_core_session_get_channel(session);
+  switch_media_bug_t *bug;
+  switch_status_t status;
+  void *pUserData;
+
+  if (switch_channel_get_private(channel, bugname)) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Verbio removing bug from previous transcribe\n");
+    do_stop(session, bugname);
+  }
+
+  if (switch_channel_pre_answer(channel) != SWITCH_STATUS_SUCCESS) {
+    return SWITCH_STATUS_FALSE;
+  }
+
+  if (SWITCH_STATUS_FALSE == verbio_speech_session_init(session, responseHandler, 
+    flags & SMBF_STEREO ? 2 : 1/*channels*/,lang, interim, bugname, &pUserData)) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing verbio speech session.\n");
+    return SWITCH_STATUS_FALSE;
+  }
+  if ((status = switch_core_media_bug_add(session, bugname, NULL, capture_callback, pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
+    return status;
+  }
+  switch_channel_set_private(channel, bugname, bug);
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "added media bug for verbio transcribe\n");
+
+  return SWITCH_STATUS_SUCCESS;
+}
+
+static switch_status_t do_stop(switch_core_session_t *session, char* bugname)
+{
+  switch_status_t status = SWITCH_STATUS_SUCCESS;
+
+  switch_channel_t *channel = switch_core_session_get_channel(session);
+  switch_media_bug_t *bug = switch_channel_get_private(channel, bugname);
+
+  if (bug) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "do_stop: Received user command command to stop transcribe.\n");
+    status = verbio_speech_session_cleanup(session, 0, bugname);
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "do_stop: stopped transcribe.\n");
+  }
+
+  return status;
+}
+
+#define TRANSCRIBE_API_SYNTAX "<uuid> [start|stop] lang-code [interim] [stereo|mono] [bugname]"
+SWITCH_STANDARD_API(verbio_transcribe_function)
+{
+  char *mycmd = NULL, *argv[6] = { 0 };
+  int argc = 0;
+  switch_status_t status = SWITCH_STATUS_FALSE;
+  switch_media_bug_flag_t flags = SMBF_READ_STREAM /* | SMBF_WRITE_STREAM | SMBF_READ_PING */;
+
+  if (!zstr(cmd) && (mycmd = strdup(cmd))) {
+    argc = switch_separate_string(mycmd, ' ', argv, (sizeof(argv) / sizeof(argv[0])));
+  }
+
+
+  if (zstr(cmd) || zstr(argv[0])) {
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error with command %s %s %s.\n", cmd, argv[0], argv[1]);
+    stream->write_function(stream, "-USAGE: %s\n", TRANSCRIBE_API_SYNTAX);
+    goto done;
+  } else {
+    switch_core_session_t *lsession = NULL;
+
+    if ((lsession = switch_core_session_locate(argv[0]))) {
+      if (!strcasecmp(argv[1], "stop")) {
+        char *bugname = argc > 2 ? argv[2] : MY_BUG_NAME;
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "Verbio stop transcribing %s\n", bugname);
+        status = do_stop(lsession, bugname);
+      } else if (!strcasecmp(argv[1], "start")) {
+        char* lang = argv[2];
+        int interim = argc > 3 && !strcmp(argv[3], "interim");
+				char *bugname = argc > 5 ? argv[5] : MY_BUG_NAME;
+				if (argc > 4 && !strcmp(argv[4], "stereo")) {
+          flags |= SMBF_WRITE_STREAM ;
+          flags |= SMBF_STEREO;
+				}
+    		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "start transcribing %s %s %s\n", 
+          lang, interim ? "interim": "complete", bugname);
+        status = start_capture(lsession, flags, lang, interim, bugname);
+      }
+      switch_core_session_rwunlock(lsession);
+    }
+  }
+
+  if (status == SWITCH_STATUS_SUCCESS) {
+    stream->write_function(stream, "+OK Success\n");
+  } else {
+    stream->write_function(stream, "-ERR Operation Failed\n");
+  }
+
+  done:
+
+  switch_safe_free(mycmd);
+  return SWITCH_STATUS_SUCCESS;
+}
+
+
+SWITCH_MODULE_LOAD_FUNCTION(mod_verbio_transcribe_load)
+{
+  switch_api_interface_t *api_interface;
+
+  /* create/register custom event message type */
+  if (switch_event_reserve_subclass(TRANSCRIBE_EVENT_RESULTS) != SWITCH_STATUS_SUCCESS) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't register subclass %s!\n", TRANSCRIBE_EVENT_RESULTS);
+    return SWITCH_STATUS_TERM;
+  }
+
+  /* connect my internal structure to the blank pointer passed to me */
+  *module_interface = switch_loadable_module_create_module_interface(pool, modname);
+
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "verbio Speech Transcription API loading..\n");
+
+  if (SWITCH_STATUS_FALSE == verbio_speech_init()) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Failed initializing verbio speech interface\n");
+  }
+
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "verbio Speech Transcription API successfully loaded\n");
+
+  SWITCH_ADD_API(api_interface, "uuid_verbio_transcribe", "verbio Speech Transcription API", verbio_transcribe_function, TRANSCRIBE_API_SYNTAX);
+  switch_console_set_complete("add uuid_verbio_transcribe start lang-code [interim|final] [stereo|mono] [bugname]");
+  switch_console_set_complete("add uuid_verbio_transcribe stop ");
+
+  /* indicate that the module should continue to be loaded */
+  return SWITCH_STATUS_SUCCESS;
+}
+
+/*
+  Called when the system shuts down
+  Macro expands to: switch_status_t mod_verbio_transcribe_shutdown() */
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_verbio_transcribe_shutdown)
+{
+  verbio_speech_cleanup();
+  switch_event_free_subclass(TRANSCRIBE_EVENT_RESULTS);
+  return SWITCH_STATUS_SUCCESS;
+}

mod_verbio_transcribe/mod_verbio_transcribe.h

@@ -0,0 +1,48 @@
+#ifndef __MOD_VERBIO_TRANSCRIBE_H__
+#define __MOD_VERBIO_TRANSCRIBE_H__
+
+#include <switch.h>
+#include <speex/speex_resampler.h>
+
+#include <unistd.h>
+
+#define MY_BUG_NAME "verbio_transcribe"
+#define MAX_ENGINE_VERSION_LEN (2)
+#define MAX_BUG_LEN (64)
+#define MAX_SESSION_ID (256)
+#define LONG_TEXT_LEN (1024)
+#define MAX_LANGUAGE_LEN (6)
+#define TRANSCRIBE_EVENT_RESULTS                     "verbio_transcribe::transcription"
+#define TRANSCRIBE_EVENT_ERROR                       "jambonz_transcribe::error"
+
+/* per-channel data */
+typedef void (*responseHandler_t)(switch_core_session_t* session, const char* event, const char * json, const char* bugname, int finished);
+
+struct cap_cb {
+  switch_mutex_t *mutex;
+  char sessionId[MAX_SESSION_ID+1];
+  char bugname[MAX_BUG_LEN+1];
+  char access_token[LONG_TEXT_LEN + 1];
+  char language[MAX_LANGUAGE_LEN + 1];
+  char inline_grammar[LONG_TEXT_LEN + 1];
+  char grammar_uri[LONG_TEXT_LEN + 1];
+  char label[MAX_SESSION_ID+1];
+  uint32_t engine_version;
+  uint32_t topic;
+  uint32_t enable_formatting; 
+  uint32_t enable_diarization;
+  uint32_t channels;
+  uint32_t interim;
+  uint32_t recognition_timeout;
+  uint32_t speech_complete_timeout;
+  uint32_t speech_incomplete_timeout;
+  uint32_t finished;
+  
+
+  SpeexResamplerState *resampler;
+  void* streamer;
+  responseHandler_t responseHandler;
+  switch_thread_t* thread;
+};
+
+#endif

mod_verbio_transcribe/simple_buffer.h

@@ -0,0 +1,51 @@
+/**
+ * (very) simple and limited circular buffer, 
+ * supporting only the use case of doing all of the adds
+ * and then subsquently retrieves.
+ * 
+ */
+class SimpleBuffer {
+  public:
+    SimpleBuffer(uint32_t chunkSize, uint32_t numChunks) : numItems(0),
+    m_numChunks(numChunks), m_chunkSize(chunkSize) {
+      m_pData = new char[chunkSize * numChunks];
+      m_pNextWrite = m_pData;
+    }
+    ~SimpleBuffer() {
+      delete [] m_pData;
+    }
+
+    void add(void *data, uint32_t datalen) {
+      if (datalen % m_chunkSize != 0) return;
+      int numChunks = datalen / m_chunkSize;
+      for (int i = 0; i < numChunks; i++) {
+        memcpy(m_pNextWrite, data, m_chunkSize);
+        data = static_cast<char*>(data) + m_chunkSize;
+        if (numItems < m_numChunks) numItems++;
+
+        uint32_t offset = (m_pNextWrite - m_pData) / m_chunkSize;
+        if (offset >= m_numChunks - 1) m_pNextWrite = m_pData;
+        else m_pNextWrite += m_chunkSize;
+      }
+    }
+
+    char* getNextChunk() {
+      if (numItems--) {
+        char *p = m_pNextWrite;
+        uint32_t offset = (m_pNextWrite - m_pData) / m_chunkSize;
+        if (offset >= m_numChunks - 1) m_pNextWrite = m_pData;
+        else m_pNextWrite += m_chunkSize;
+        return p;
+      }
+      return nullptr;
+    }
+
+    uint32_t getNumItems() { return numItems;}
+
+  private:
+    char *m_pData;
+    uint32_t numItems;
+    uint32_t m_chunkSize;
+    uint32_t m_numChunks;
+    char* m_pNextWrite;
+};

mod_verbio_transcribe/verbio_glue.cpp

@@ -0,0 +1,455 @@
+#include <cstdlib>
+#include <algorithm>
+#include <future>
+
+#include <switch.h>
+#include <switch_json.h>
+#include <grpc++/grpc++.h>
+#include <google/protobuf/util/json_util.h>
+
+#include "speechcenter/recognizer/v1/recognition.grpc.pb.h"
+
+namespace verbio_asr = speechcenter::recognizer::v1;
+
+#include "mod_verbio_transcribe.h"
+#include "simple_buffer.h"
+
+#define CHUNKSIZE (320)
+
+namespace {
+  int case_insensitive_match(std::string s1, std::string s2) {
+   std::transform(s1.begin(), s1.end(), s1.begin(), ::tolower);
+   std::transform(s2.begin(), s2.end(), s2.begin(), ::tolower);
+   if(s1.compare(s2) == 0)
+      return 1; //The strings are same
+   return 0; //not matched
+  }
+}
+
+class GStreamer {
+public:
+  GStreamer(cap_cb *cb) : 
+    m_writesDone(false), 
+    m_connected(false), 
+    m_interim(cb->interim),
+    m_audioBuffer(CHUNKSIZE, 15) {
+
+    strncpy(m_sessionId, cb->sessionId, 256);
+    auto channelCreds = grpc::SslCredentials(grpc::SslCredentialsOptions());
+    m_channel = grpc::CreateChannel(
+                    "us.speechcenter.verbio.com",
+                    grpc::CompositeChannelCredentials(
+                    grpc::SslCredentials(grpc::SslCredentialsOptions()),
+                    grpc::AccessTokenCredentials(cb->access_token)));
+
+    if (!m_channel) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "GStreamer %p failed creating grpc channel\n", this);  
+      throw std::runtime_error(std::string("Error creating grpc channel"));
+    }
+
+    m_stub = std::move(verbio_asr::Recognizer::NewStub(m_channel));
+
+    auto* config = m_request.mutable_config();
+    // RecognitionParameters
+    auto* params = config->mutable_parameters();
+    params->set_language(cb->language);
+    auto* pcm = params->mutable_pcm();
+    pcm->set_sample_rate_hz(8000);
+    params->set_audio_channels_number(cb->channels);
+    params->set_enable_formatting(cb->enable_formatting);
+    auto* resource = config->mutable_resource();
+    resource->set_topic(static_cast<verbio_asr::RecognitionResource_Topic>(cb->topic));
+    if (!zstr(cb->inline_grammar) || !zstr(cb->grammar_uri)) {
+      auto* grammar = resource->mutable_grammar();
+      if (cb->inline_grammar) {
+        grammar->set_inline_grammar(cb->inline_grammar);
+      } else if (cb->grammar_uri) {
+        grammar->set_grammar_uri(cb->grammar_uri);
+      }
+    }
+
+    config->set_version(static_cast<verbio_asr::RecognitionConfig_AsrVersion>(cb->engine_version));
+    if (cb->label) {
+      config->add_label(cb->label);
+    }
+    if (cb->recognition_timeout || cb->speech_complete_timeout || cb->speech_incomplete_timeout) {
+      auto* timer = config->mutable_configuration();
+      timer->set_start_input_timers(true);
+      if (cb->recognition_timeout) {
+        timer->set_recognition_timeout(cb->recognition_timeout);
+      }
+      if (cb->speech_complete_timeout) {
+        timer->set_speech_complete_timeout(cb->speech_complete_timeout);
+      }
+      if (cb->speech_incomplete_timeout) {
+        timer->set_speech_incomplete_timeout(cb->speech_incomplete_timeout);
+      }
+    }
+  }
+
+  ~GStreamer() {
+  }
+
+  void connect() {
+    assert(!m_connected);
+    // Begin a stream.
+
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p creating streamer\n", this);  
+    m_streamer = m_stub->StreamingRecognize(&m_context);
+    m_connected = true;
+
+    // read thread is waiting on this
+    m_promise.set_value();
+
+    // Write the first request, containing the config only.
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p sending initial message\n", this);  
+    bool ok = m_streamer->Write(m_request);
+    m_request.clear_config();
+
+    // send any buffered audio
+    int nFrames = m_audioBuffer.getNumItems();
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p got stream ready, %d buffered frames\n", this, nFrames);  
+    if (nFrames) {
+      char *p;
+      do {
+        p = m_audioBuffer.getNextChunk();
+        if (p) {
+          write(p, CHUNKSIZE);
+        }
+      } while (p);
+    }
+  }
+
+  bool write(void* data, uint32_t datalen) {
+    if (!m_connected) {
+      if (datalen % CHUNKSIZE == 0) {
+        m_audioBuffer.add(data, datalen);
+      }
+      return true;
+    }
+    m_request.clear_audio();
+    m_request.set_audio(data, datalen); 
+    bool ok = m_streamer->Write(m_request);
+    return ok;
+  }
+
+  uint32_t nextMessageSize(void) {
+    uint32_t size = 0;
+    m_streamer->NextMessageSize(&size);
+    return size;
+  }
+
+  bool read(verbio_asr::RecognitionStreamingResponse* response) {
+    return m_streamer->Read(response);
+  }
+
+  grpc::Status finish() {
+    return m_streamer->Finish();
+  }
+
+  void writesDone() {
+    // grpc crashes if we call this twice on a stream
+    if (!m_connected) {
+      cancelConnect();
+    }
+    else if (!m_writesDone) {
+      m_streamer->WritesDone();
+      m_writesDone = true;
+    }
+  }
+
+  bool waitForConnect() {
+    std::shared_future<void> sf(m_promise.get_future());
+    sf.wait();
+    return m_connected;
+  }
+
+  void cancelConnect() {
+    assert(!m_connected);
+    m_promise.set_value();
+  } 
+
+  bool isConnected() {
+    return m_connected;
+  }
+
+private:
+  grpc::ClientContext m_context;
+  std::shared_ptr<grpc::Channel> m_channel;
+  std::unique_ptr<verbio_asr::Recognizer::Stub> m_stub;
+  verbio_asr::RecognitionStreamingRequest m_request;
+  std::unique_ptr< grpc::ClientReaderWriterInterface<verbio_asr::RecognitionStreamingRequest, verbio_asr::RecognitionStreamingResponse> > m_streamer;
+  bool m_writesDone;
+  bool m_connected;
+  bool m_interim;
+  std::string m_language;
+  std::promise<void> m_promise;
+  SimpleBuffer m_audioBuffer;
+  char m_sessionId[256];
+};
+
+static void *SWITCH_THREAD_FUNC grpc_read_thread(switch_thread_t *thread, void *obj) {
+  struct cap_cb *cb = (struct cap_cb *) obj;
+  GStreamer* streamer = (GStreamer *) cb->streamer;
+
+  bool connected = streamer->waitForConnect();
+  if (!connected) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "verbio transcribe grpc read thread exiting since we didnt connect\n") ;
+    return nullptr;
+  }
+
+  // Read responses.
+  verbio_asr::RecognitionStreamingResponse response;
+  while (streamer->read(&response)) {  // Returns false when no more to read.
+    if (response.has_error()) {
+      // handle error
+      const auto& error = response.error();
+      auto reason = error.reason();
+      cJSON* json = cJSON_CreateObject();
+      cJSON_AddStringToObject(json, "type", "error");
+      cJSON_AddStringToObject(json, "error", reason.c_str());
+      char* json_string = cJSON_PrintUnformatted(json);
+
+      switch_core_session_t* session = switch_core_session_locate(cb->sessionId);
+      if (!session) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "grpc_read_thread: session %s is gone!\n", cb->sessionId) ;
+        return nullptr;
+      }
+      cb->responseHandler(session, TRANSCRIBE_EVENT_ERROR, json_string, cb->bugname, cb->finished);
+      switch_core_session_rwunlock(session);
+      // clean
+      free(json_string);
+      cJSON_Delete(json);
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer recognition error %s\n", reason.c_str());
+      break;
+    } else if (!response.has_result()) {
+      // there is no available results yet.
+      continue;
+    } else {
+      const auto& result = response.result();
+      if (response.result().alternatives_size() > 0) {
+        const auto& alternative = response.result().alternatives(0);
+        if (alternative.words_size() == 0) {
+            continue;
+        }
+      }
+      std::string json_string;
+      google::protobuf::util::JsonPrintOptions options;
+      options.always_print_primitive_fields = true;
+      options.preserve_proto_field_names = true;
+      absl::Status status = google::protobuf::util::MessageToJsonString(result, &json_string, options);
+
+      if (!status.ok()) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Cannot parse verbio result, error: %s", status.ToString()) ;
+        
+      } else {
+        switch_core_session_t* session = switch_core_session_locate(cb->sessionId);
+        if (!session) {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "grpc_read_thread: session %s is gone!\n", cb->sessionId) ;
+          return nullptr;
+        }
+        cb->responseHandler(session, TRANSCRIBE_EVENT_RESULTS, json_string.c_str(), cb->bugname, cb->finished);
+        switch_core_session_rwunlock(session);
+      }
+    }
+  }
+  return nullptr;
+}
+
+extern "C" {
+
+  switch_status_t verbio_speech_init() {
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t verbio_speech_cleanup() {
+    return SWITCH_STATUS_SUCCESS;
+  }
+  switch_status_t verbio_speech_session_init(switch_core_session_t *session, responseHandler_t responseHandler, 
+    uint32_t channels, char* lang, int interim, char* bugname, void **ppUserData) {
+
+    switch_channel_t *channel = switch_core_session_get_channel(session);
+    switch_memory_pool_t *pool = switch_core_session_get_pool(session);
+    auto read_codec = switch_core_session_get_read_codec(session);
+    uint32_t sampleRate = read_codec->implementation->actual_samples_per_second;
+    struct cap_cb *cb;
+    int err;
+
+    cb =(struct cap_cb *) switch_core_session_alloc(session, sizeof(*cb));
+    strncpy(cb->sessionId, switch_core_session_get_uuid(session), MAX_SESSION_ID);
+    strncpy(cb->bugname, bugname, MAX_BUG_LEN);
+    cb->channels = channels;
+    cb->interim = interim;
+    cb->finished = 0;
+
+    // Read Verbio configuration from channel variables
+    const char* var;
+    if (var = switch_channel_get_variable(channel, "VERBIO_ACCESS_TOKEN")) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Using channel vars for verbio authentication\n");
+      strncpy(cb->access_token, var, LONG_TEXT_LEN);
+    }
+    else if (std::getenv("VERBIO_ACCESS_TOKEN")) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "Using env vars for verbio authentication\n");
+      strncpy(cb->access_token, std::getenv("VERBIO_ACCESS_TOKEN"), LONG_TEXT_LEN);
+    }
+    else {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "No channel vars or env vars for verbio authentication. Stop initiating Verbio connection\n");
+      return SWITCH_STATUS_FALSE;
+    }
+    cb->enable_formatting = switch_true(switch_channel_get_variable(channel, "VERBIO_ENABLE_FORMATTING"));
+    cb->enable_diarization = switch_true(switch_channel_get_variable(channel, "VERBIO_ENABLE_DIARIZATION"));
+    strncpy(cb->language, lang, MAX_LANGUAGE_LEN);
+    if (var = switch_channel_get_variable(channel, "VERBIO_ENGINE_VERSION")) {
+      cb->engine_version = atoi(var);
+    }
+    if (var = switch_channel_get_variable(channel, "VERBIO_TOPIC")) {
+      cb->topic = atoi(var);
+    } else {
+      cb->topic = 0;
+    }
+    if (var = switch_channel_get_variable(channel, "VERBIO_INLINE_GRAMMAR")) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "xhoaluu1 %s\n", var);
+      strncpy(cb->inline_grammar, var, LONG_TEXT_LEN);
+    }
+    if (var = switch_channel_get_variable(channel, "VERBIO_GRAMMAR_URI")) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "xhoaluu2 %s\n", var);
+      strncpy(cb->grammar_uri, var, LONG_TEXT_LEN);
+    }
+    if (var = switch_channel_get_variable(channel, "VERBIO_LABEL")) {
+      strncpy(cb->label, var, MAX_SESSION_ID);
+    }
+    if (var = switch_channel_get_variable(channel, "VERBIO_RECOGNITION_TIMEOUT")) {
+      cb->recognition_timeout = atoi(var);
+    }
+    if (var = switch_channel_get_variable(channel, "VERBIO_SPEECH_COMPLETE_TIMEOUT")) {
+      cb->speech_complete_timeout = atoi(var);
+    }
+    if (var = switch_channel_get_variable(channel, "VERBIO_SPEECH_INCOMPLETE_TIMEOUT")) {
+      cb->speech_incomplete_timeout = atoi(var);
+    }
+
+    if (switch_mutex_init(&cb->mutex, SWITCH_MUTEX_NESTED, pool) != SWITCH_STATUS_SUCCESS) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error initializing mutex\n");
+      return SWITCH_STATUS_FALSE;
+    }
+
+    if (sampleRate != 8000) {
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "verbio_speech_session_init:  initializing resampler\n");
+        cb->resampler = speex_resampler_init(channels, sampleRate, 8000, SWITCH_RESAMPLE_QUALITY, &err);
+      if (0 != err) {
+          switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing resampler: %s.\n",
+          switch_channel_get_name(channel), speex_resampler_strerror(err));
+        return SWITCH_STATUS_FALSE;
+      }
+    } else {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "%s: no resampling needed for this call\n", switch_channel_get_name(channel));
+    }
+    cb->responseHandler = responseHandler;
+
+    GStreamer *streamer = NULL;
+    try {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "verbio_speech_session_init:  allocating streamer\n");
+      streamer = new GStreamer(cb);
+      cb->streamer = streamer;
+    } catch (std::exception& e) {
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing gstreamer: %s.\n", 
+        switch_channel_get_name(channel), e.what());
+      return SWITCH_STATUS_FALSE;
+    }
+
+    streamer->connect();
+
+    // create the read thread
+    switch_threadattr_t *thd_attr = NULL;
+    switch_threadattr_create(&thd_attr, pool);
+    switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE);
+    switch_thread_create(&cb->thread, thd_attr, grpc_read_thread, cb, pool);
+
+    *ppUserData = cb;
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t verbio_speech_session_cleanup(switch_core_session_t *session, int channelIsClosing, char* bugname) {
+    switch_channel_t *channel = switch_core_session_get_channel(session);
+    switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, bugname);
+    if (bug) {
+      struct cap_cb *cb = (struct cap_cb *) switch_core_media_bug_get_user_data(bug);
+      switch_mutex_lock(cb->mutex);
+
+      switch_channel_set_private(channel, cb->bugname, NULL);
+
+      // close connection and get final responses
+      GStreamer* streamer = (GStreamer *) cb->streamer;
+
+      if (streamer) {
+        streamer->writesDone();
+        cb->finished = 1;
+
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "verbio_speech_session_cleanup: GStreamer (%p) waiting for read thread to complete\n", (void*)streamer);
+        switch_status_t st;
+        switch_thread_join(&st, cb->thread);
+        switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "verbio_speech_session_cleanup:  GStreamer (%p) read thread completed\n", (void*)streamer);
+
+        delete streamer;
+        cb->streamer = NULL;
+      }
+
+      if (cb->resampler) {
+        speex_resampler_destroy(cb->resampler);
+      }
+
+      if (!channelIsClosing) {
+        switch_core_media_bug_remove(session, &bug);
+      }
+
+      switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "verbio_speech_session_cleanup: Closed stream\n");
+
+      switch_mutex_unlock(cb->mutex);
+      switch_mutex_destroy(cb->mutex);
+      cb->mutex = nullptr;
+
+      return SWITCH_STATUS_SUCCESS;
+    }
+
+    switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s Bug is not attached.\n", switch_channel_get_name(channel));
+    return SWITCH_STATUS_FALSE;
+  }
+
+  switch_bool_t verbio_speech_frame(switch_media_bug_t *bug, void* user_data) {
+      switch_core_session_t *session = switch_core_media_bug_get_session(bug);
+      struct cap_cb *cb = (struct cap_cb *) user_data;
+      if (cb->streamer) {
+        GStreamer* streamer = (GStreamer *) cb->streamer;
+        uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
+        switch_frame_t frame = {};
+        frame.data = data;
+        frame.buflen = SWITCH_RECOMMENDED_BUFFER_SIZE;
+
+        if (switch_mutex_trylock(cb->mutex) == SWITCH_STATUS_SUCCESS) {
+          while (switch_core_media_bug_read(bug, &frame, SWITCH_TRUE) == SWITCH_STATUS_SUCCESS && !switch_test_flag((&frame), SFF_CNG)) {
+            if (frame.datalen) {
+
+              if (cb->resampler) {
+                spx_int16_t out[SWITCH_RECOMMENDED_BUFFER_SIZE];
+                spx_uint32_t out_len = SWITCH_RECOMMENDED_BUFFER_SIZE;
+                spx_uint32_t in_len = frame.samples;
+                size_t written;
+
+                speex_resampler_process_interleaved_int(cb->resampler,
+                  (const spx_int16_t *) frame.data,
+                  (spx_uint32_t *) &in_len,
+                  &out[0],
+                  &out_len);
+                streamer->write( &out[0], sizeof(spx_int16_t) * out_len);
+              }
+              else {
+                streamer->write( frame.data, sizeof(spx_int16_t) * frame.samples);
+              }
+            }
+          }
+          switch_mutex_unlock(cb->mutex);
+        }
+      }
+      return SWITCH_TRUE;
+    }
+}

mod_verbio_transcribe/verbio_glue.h

@@ -0,0 +1,11 @@
+#ifndef __VERBIO_GLUE_H__
+#define __VERBIO_GLUE_H__
+
+switch_status_t verbio_speech_init();
+switch_status_t verbio_speech_cleanup();
+switch_status_t verbio_speech_session_init(switch_core_session_t *session, responseHandler_t responseHandler, 
+		uint32_t channels, char* lang, int interim, char* bugname, void **ppUserData);
+switch_status_t verbio_speech_session_cleanup(switch_core_session_t *session, int channelIsClosing, char* bugname);
+switch_bool_t verbio_speech_frame(switch_media_bug_t *bug, void* user_data);
+
+#endif

mod_verbio_tts/LICENSE

@@ -0,0 +1,8 @@
+Copyright 2023, Drachtio Communications Services, LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

mod_verbio_tts/Makefile.am

@@ -0,0 +1,10 @@
+include $(top_srcdir)/build/modmake.rulesam
+MODNAME=mod_verbio_tts
+
+mod_LTLIBRARIES = mod_verbio_tts.la
+mod_verbio_tts_la_SOURCES  = mod_verbio_tts.c verbio_glue.cpp
+mod_verbio_tts_la_CFLAGS   = $(AM_CFLAGS)
+mod_verbio_tts_la_CXXFLAGS = $(AM_CXXFLAGS) -std=c++17
+
+mod_verbio_tts_la_LIBADD   = $(switch_builddir)/libfreeswitch.la
+mod_verbio_tts_la_LDFLAGS  = -avoid-version -module -no-undefined -shared -lstdc++ -lboost_system -lboost_thread

mod_verbio_tts/README.md

@@ -0,0 +1,3 @@
+# mod_verbio_tts
+
+A Freeswitch module that allows speak text to speech audio from Verbio stream.

mod_verbio_tts/mod_verbio_tts.c

@@ -0,0 +1,153 @@
+#include "mod_verbio_tts.h"
+#include "verbio_glue.h"
+
+SWITCH_MODULE_LOAD_FUNCTION(mod_verbio_tts_load);
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_verbio_tts_shutdown);
+SWITCH_MODULE_DEFINITION(mod_verbio_tts, mod_verbio_tts_load, mod_verbio_tts_shutdown, NULL);
+
+static void clearverbio(verbio_t* v, int freeAll) {
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "clearverbio\n");
+  if (v->access_token) free(v->access_token);
+
+  if (v->ct) free(v->ct);
+  if (v->err_msg) free(v->err_msg);
+  if (v->name_lookup_time_ms) free(v->name_lookup_time_ms);
+  if (v->connect_time_ms) free(v->connect_time_ms);
+  if (v->final_response_time_ms) free(v->final_response_time_ms);
+  if (v->cache_filename) free(v->cache_filename);
+  
+
+  v->access_token = NULL;
+  v->ct = NULL;
+  v->err_msg = NULL;
+  v->name_lookup_time_ms = NULL;
+  v->connect_time_ms = NULL;
+  v->final_response_time_ms = NULL;
+  v->cache_filename = NULL;
+
+  if (freeAll) {
+    if (v->voice_name) free(v->voice_name);
+    if (v->session_id) free(v->session_id);
+    v->voice_name = NULL;
+    v->session_id = NULL;
+  }
+}
+
+static verbio_t * createOrRetrievePrivateData(switch_speech_handle_t *sh) {
+  verbio_t *v = (verbio_t *) sh->private_info;  
+  if (!v) {
+    v = switch_core_alloc(sh->memory_pool, sizeof(*v));
+  	sh->private_info = v;
+    memset(v, 0, sizeof(*v));
+    switch_mutex_init(&v->mutex, SWITCH_MUTEX_NESTED, sh->memory_pool);
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "allocated verbio_t\n");
+  }
+  return v;
+}
+
+switch_status_t v_speech_open(switch_speech_handle_t *sh, const char *voice_name, int rate, int channels, switch_speech_flag_t *flags)
+{
+  verbio_t *v = createOrRetrievePrivateData(sh);
+  v->voice_name = strdup(voice_name);
+  v->rate = rate;
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "v_speech_open voice: %s, rate %d, channels %d\n", voice_name, rate, channels);
+  return verbio_speech_open(v);
+}
+
+static switch_status_t v_speech_close(switch_speech_handle_t *sh, switch_speech_flag_t *flags)
+{
+  switch_status_t rc;
+  verbio_t *v = createOrRetrievePrivateData(sh);
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "v_speech_close\n");
+
+  switch_mutex_destroy(v->mutex);
+
+  rc = verbio_speech_close(v);
+  clearverbio(v, 1);
+  return rc;
+}
+
+/**
+ * Freeswitch will call this function to feed us text to speak
+ */
+static switch_status_t v_speech_feed_tts(switch_speech_handle_t *sh, char *text, switch_speech_flag_t *flags)
+{
+  verbio_t *v = createOrRetrievePrivateData(sh);
+  v->draining = 0;
+  v->reads = 0;
+  v->response_code = 0;
+  v->err_msg = NULL;
+
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "v_speech_feed_tts\n");
+
+  return verbio_speech_feed_tts(v, text, flags);
+}
+
+/**
+ * Freeswitch calls periodically to get some rendered audio in L16 format. We can provide up to 8k of audio at a time.
+ */
+static switch_status_t v_speech_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, switch_speech_flag_t *flags)
+{
+  verbio_t *v = createOrRetrievePrivateData(sh);
+  return verbio_speech_read_tts(v, data, datalen, flags);
+}
+
+/**
+ * This is called at the end, not sure exactly what we need to do here..
+ */
+static void v_speech_flush_tts(switch_speech_handle_t *sh)
+{
+  verbio_t *v = createOrRetrievePrivateData(sh);
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "v_speech_flush_tts\n");
+  verbio_speech_flush_tts(v);
+
+  clearverbio(v, 0);
+}
+
+static void v_text_param_tts(switch_speech_handle_t *sh, char *param, const char *val)
+{
+  verbio_t *v = createOrRetrievePrivateData(sh);
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "v_text_param_tts: %s=%s\n", param, val);
+  if (0 == strcmp(param, "access_token")) {
+    if (v->access_token) free(v->access_token);
+    v->access_token = strdup(val);
+  } else if (0 == strcmp(param, "voice")) {
+    if (v->voice_name) free(v->voice_name);
+    v->voice_name = strdup(val);
+  } else if (0 == strcmp(param, "session-uuid")) {
+    if (v->session_id) free(v->session_id);
+    v->session_id = strdup(val);
+  } else if (0 == strcmp(param, "write_cache_file") && switch_true(val)) {
+    v->cache_audio = 1;
+  }
+}
+
+static void v_numeric_param_tts(switch_speech_handle_t *sh, char *param, int val)
+{
+}
+static void v_float_param_tts(switch_speech_handle_t *sh, char *param, double val)
+{
+}
+
+SWITCH_MODULE_LOAD_FUNCTION(mod_verbio_tts_load)
+{
+  switch_speech_interface_t *speech_interface;
+
+  *module_interface = switch_loadable_module_create_module_interface(pool, modname);
+  speech_interface = switch_loadable_module_create_interface(*module_interface, SWITCH_SPEECH_INTERFACE);
+  speech_interface->interface_name = "verbio";
+  speech_interface->speech_open = v_speech_open;
+  speech_interface->speech_close = v_speech_close;
+  speech_interface->speech_feed_tts = v_speech_feed_tts;
+  speech_interface->speech_read_tts = v_speech_read_tts;
+	speech_interface->speech_flush_tts = v_speech_flush_tts;
+	speech_interface->speech_text_param_tts = v_text_param_tts;
+	speech_interface->speech_numeric_param_tts = v_numeric_param_tts;
+	speech_interface->speech_float_param_tts = v_float_param_tts;
+  return verbio_speech_load();
+}
+
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_verbio_tts_shutdown)
+{
+  return verbio_speech_unload();
+}

mod_verbio_tts/mod_verbio_tts.h

@@ -0,0 +1,33 @@
+#ifndef __MOD_VERBIO_TTS_H__
+#define __MOD_VERBIO_TTS_H__
+
+#include <switch.h>
+#include <speex/speex_resampler.h>
+
+typedef struct verbio_data {
+  char *voice_name;
+  char *access_token;
+
+  /* result data */
+  long response_code;
+  char *ct;
+  char *name_lookup_time_ms;
+  char *connect_time_ms;
+  char *final_response_time_ms;
+  char *err_msg;
+  char *cache_filename;
+  char *session_id;
+
+  int rate;
+  int draining;
+  int reads;
+  int cache_audio;
+
+	void *conn;
+  void *circularBuffer;
+  switch_mutex_t *mutex;
+  FILE *file;
+  SpeexResamplerState *resampler;
+} verbio_t;
+
+#endif

mod_verbio_tts/verbio_glue.cpp

@@ -0,0 +1,958 @@
+#include "mod_verbio_tts.h"
+#include <switch.h>
+#include <switch_json.h>
+#include <curl/curl.h>
+#include <cstdlib>
+
+#include <boost/circular_buffer.hpp>
+#include <boost/thread.hpp>
+#include <boost/asio.hpp>
+#include <boost/asio/ssl.hpp>
+#include <boost/pool/object_pool.hpp>
+#include <boost/bind/bind.hpp>
+#include <boost/tokenizer.hpp>
+#include <boost/foreach.hpp>
+#include <boost/asio.hpp>
+#include <boost/assign/list_of.hpp>
+#include <boost/algorithm/string.hpp>
+
+#include <speex/speex_resampler.h>
+
+#define BUFFER_GROW_SIZE (80000)
+
+typedef boost::circular_buffer<uint16_t> CircularBuffer_t;
+/* Global information, common to all connections */
+typedef struct
+{
+  CURLM *multi;
+  int still_running;
+} GlobalInfo_t;
+static GlobalInfo_t global;
+
+/* Information associated with a specific easy handle */
+typedef struct
+{
+  CURL *easy;
+  verbio_t* verbio;
+  char* body;
+  struct curl_slist *hdr_list;
+  GlobalInfo_t *global;
+  char error[CURL_ERROR_SIZE];
+  FILE* file;
+  std::chrono::time_point<std::chrono::high_resolution_clock> startTime;
+  bool flushed;
+
+  bool has_last_byte;
+  uint8_t last_byte;
+} ConnInfo_t;
+
+
+static boost::object_pool<ConnInfo_t> pool ;
+static std::map<curl_socket_t, boost::asio::ip::tcp::socket *> socket_map;
+static boost::asio::io_service io_service;
+static boost::asio::deadline_timer timer(io_service);
+static std::string fullDirPath;
+static std::thread worker_thread;
+
+std::string secondsToMillisecondsString(double seconds) {
+    // Convert to milliseconds
+    double milliseconds = seconds * 1000.0;
+
+    // Truncate to remove fractional part
+    long milliseconds_long = static_cast<long>(milliseconds);
+
+    // Convert to string
+    return std::to_string(milliseconds_long);
+}
+
+static std::string getEnvVar(const std::string& varName) {
+    const char* val = std::getenv(varName.c_str());
+    return val ? std::string(val) : "";
+}
+
+static long getConnectionTimeout() {
+    std::string connectTimeoutStr = getEnvVar("VERBIO_TTS_CURL_CONNECT_TIMEOUT");
+
+    if (connectTimeoutStr.empty()) {
+        connectTimeoutStr = getEnvVar("TTS_CURL_CONNECT_TIMEOUT");
+    }
+
+    long connectTimeout = 10L;
+
+    if (!connectTimeoutStr.empty()) {
+        connectTimeout = std::stol(connectTimeoutStr);
+    }
+
+    return connectTimeout;
+}
+
+static CURL* createEasyHandle(void) {
+  CURL* easy = curl_easy_init();
+  if(!easy) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "curl_easy_init() failed!\n");
+    return nullptr ;
+  }  
+
+  curl_easy_setopt(easy, CURLOPT_FOLLOWLOCATION, 1L);
+  curl_easy_setopt(easy, CURLOPT_USERAGENT, "jambonz/0.8.5");
+
+  // set connect timeout to 3 seconds and total timeout to 109 seconds
+  curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
+  curl_easy_setopt(easy, CURLOPT_TIMEOUT, getConnectionTimeout());
+
+  return easy ;
+}
+
+static void cleanupConn(ConnInfo_t *conn) {
+  auto v = conn->verbio;
+
+  if( conn->hdr_list ) {
+    curl_slist_free_all(conn->hdr_list);
+    conn->hdr_list = nullptr ;
+  }
+  curl_easy_cleanup(conn->easy);
+
+  if (conn->file) {
+    if (fclose(conn->file) != 0) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error closing audio cache file\n");
+    }
+    conn->file = nullptr ;
+  }
+
+  v->conn = nullptr ;
+  v->draining = 1;
+
+  memset(conn, 0, sizeof(ConnInfo_t));
+  pool.destroy(conn) ;
+}
+
+/* Check for completed transfers, and remove their easy handles */
+void check_multi_info(GlobalInfo_t *g) {
+  CURLMsg *msg;
+  int msgs_left;
+  ConnInfo_t *conn;
+  CURL *easy;
+  CURLcode res;
+  
+  while((msg = curl_multi_info_read(g->multi, &msgs_left))) {
+    if(msg->msg == CURLMSG_DONE) {
+      long response_code;
+      double namelookup=0, connect=0, total=0 ;
+      char *ct = NULL ;
+
+      easy = msg->easy_handle;
+      res = msg->data.result;
+      curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
+      curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &response_code);
+      curl_easy_getinfo(easy, CURLINFO_CONTENT_TYPE, &ct);
+
+      curl_easy_getinfo(easy, CURLINFO_NAMELOOKUP_TIME, &namelookup);
+      curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connect);
+      curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &total);
+
+      auto v = conn->verbio;
+      v->response_code = response_code;
+      if (ct) v->ct = strdup(ct);
+
+      std::string name_lookup_ms = secondsToMillisecondsString(namelookup);
+      std::string connect_ms = secondsToMillisecondsString(connect);
+      std::string final_response_time_ms = secondsToMillisecondsString(total);
+
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, 
+        "mod_verbio_tts: response: %ld, content-type %s,"
+        "dns(ms): %"  CURL_FORMAT_CURL_OFF_T ".%06ld, "
+        "connect(ms): %"  CURL_FORMAT_CURL_OFF_T ".%06ld, "
+        "total(ms): %"  CURL_FORMAT_CURL_OFF_T ".%06ld\n",
+        response_code, ct,
+        (long)(namelookup), (long)(fmod(namelookup, 1.0) * 1000000),
+        (long)(connect), (long)(fmod(connect, 1.0) * 1000000),
+        (long)(total), (long)(fmod(total, 1.0) * 1000000));
+
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "name lookup time: %s\n", name_lookup_ms.c_str());
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "connect time: %s\n", connect_ms.c_str());
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "final response time: %s\n", final_response_time_ms.c_str());
+
+      v->name_lookup_time_ms = strdup(name_lookup_ms.c_str());
+      v->connect_time_ms = strdup(connect_ms.c_str());
+      v->final_response_time_ms = strdup(final_response_time_ms.c_str());
+
+      curl_multi_remove_handle(g->multi, easy);
+      cleanupConn(conn);
+    }
+  }
+}
+
+int mcode_test(const char *where, CURLMcode code) {
+  if(CURLM_OK != code) {
+    const char *s;
+    switch(code) {
+    case CURLM_CALL_MULTI_PERFORM:
+      s = "CURLM_CALL_MULTI_PERFORM";
+      break;
+    case CURLM_BAD_HANDLE:
+      s = "CURLM_BAD_HANDLE";
+      break;
+    case CURLM_BAD_EASY_HANDLE:
+      s = "CURLM_BAD_EASY_HANDLE";
+      break;
+    case CURLM_OUT_OF_MEMORY:
+      s = "CURLM_OUT_OF_MEMORY";
+      break;
+    case CURLM_INTERNAL_ERROR:
+      s = "CURLM_INTERNAL_ERROR";
+      break;
+    case CURLM_UNKNOWN_OPTION:
+      s = "CURLM_UNKNOWN_OPTION";
+      break;
+    case CURLM_LAST:
+      s = "CURLM_LAST";
+      break;
+    default:
+      s = "CURLM_unknown";
+      break;
+    case CURLM_BAD_SOCKET:
+      s = "CURLM_BAD_SOCKET";
+      break;
+    }
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mcode_test ERROR: %s returns %s:%d\n", where, s, code);
+
+    return -1;
+  }
+  return 0 ;
+}
+
+static void remsock(int *f, GlobalInfo_t *g) {
+  if(f) {
+    free(f);
+    f = NULL;
+  }
+}
+
+/* Called by asio when there is an action on a socket */
+static void event_cb(GlobalInfo_t *g, curl_socket_t s, int action, const boost::system::error_code & error, int *fdp) {
+  int f = *fdp;
+
+  //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X has action %d\n", s, action) ; 
+
+  // Socket already POOL REMOVED.
+  if (f == CURL_POLL_REMOVE) {
+    //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X removed\n", s); 
+    remsock(fdp, g);
+    return;
+  }
+
+  if(socket_map.find(s) == socket_map.end()) {
+    //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb: socket  %#X already closed\n, s");
+    return;
+  }
+
+  /* make sure the event matches what are wanted */
+  if(f == action || f == CURL_POLL_INOUT) {
+    if(error) {
+      action = CURL_CSELECT_ERR;
+    }
+    CURLMcode rc = curl_multi_socket_action(g->multi, s, action, &g->still_running);
+
+    mcode_test("event_cb: curl_multi_socket_action", rc);
+    check_multi_info(g);
+
+    if(g->still_running <= 0) {
+      timer.cancel();
+    }
+
+    /* keep on watching.
+      * the socket may have been closed and/or fdp may have been changed
+      * in curl_multi_socket_action(), so check them both */
+    if(!error && socket_map.find(s) != socket_map.end() &&
+        (f == action || f == CURL_POLL_INOUT)) {
+      boost::asio::ip::tcp::socket *tcp_socket = socket_map.find(s)->second;
+
+      if(action == CURL_POLL_IN) {
+        tcp_socket->async_read_some(boost::asio::null_buffers(),
+                                    boost::bind(&event_cb, g, s,
+                                                action, boost::placeholders::_1, fdp));
+      }
+      if(action == CURL_POLL_OUT) {
+        tcp_socket->async_write_some(boost::asio::null_buffers(),
+                                      boost::bind(&event_cb, g, s,
+                                                  action, boost::placeholders::_1, fdp));
+      } 
+    }
+  }
+}
+
+/* socket functions */
+static void setsock(int *fdp, curl_socket_t s, CURL *e, int act, int oldact, GlobalInfo_t *g) {
+  std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(s);
+
+  if(it == socket_map.end()) {
+    //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setsock: socket  %#X not found\n, s");
+    return;
+  }
+
+  boost::asio::ip::tcp::socket * tcp_socket = it->second;
+
+  *fdp = act;
+
+  if(act == CURL_POLL_IN) {
+    if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
+      tcp_socket->async_read_some(boost::asio::null_buffers(),
+                                  boost::bind(&event_cb, g, s,
+                                  CURL_POLL_IN, boost::placeholders::_1, fdp));
+    }
+  }
+  else if(act == CURL_POLL_OUT) {
+    if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
+      tcp_socket->async_write_some(boost::asio::null_buffers(),
+                                    boost::bind(&event_cb, g, s,
+                                    CURL_POLL_OUT, boost::placeholders::_1, fdp));
+    }
+  }
+  else if(act == CURL_POLL_INOUT) {
+    if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
+      tcp_socket->async_read_some(boost::asio::null_buffers(),
+                                  boost::bind(&event_cb, g, s,
+                                  CURL_POLL_IN, boost::placeholders::_1, fdp));
+    }
+    if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
+      tcp_socket->async_write_some(boost::asio::null_buffers(),
+                                    boost::bind(&event_cb, g, s,
+                                    CURL_POLL_OUT, boost::placeholders::_1, fdp));
+    }
+  }
+}
+
+static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo_t *g) {
+  /* fdp is used to store current action */
+  int *fdp = (int *) calloc(sizeof(int), 1);
+
+  setsock(fdp, s, easy, action, 0, g);
+  curl_multi_assign(g->multi, s, fdp);
+}
+
+static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) {
+  GlobalInfo_t *g = &global;
+
+  int *actionp = (int *) sockp;
+  static const char *whatstr[] = { "none", "IN", "OUT", "INOUT", "REMOVE"};
+
+  if(what == CURL_POLL_REMOVE) {
+    *actionp = what;
+  }
+  else {
+    if(!actionp) {
+      addsock(s, e, what, g);
+    }
+    else {
+      setsock(actionp, s, e, what, *actionp, g);
+    }
+  }
+  return 0;  
+}
+
+static void threadFunc() {      
+  /* to make sure the event loop doesn't terminate when there is no work to do */
+  io_service.reset() ;
+  boost::asio::io_service::work work(io_service);
+  
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_verbio_tts threadFunc - starting\n");
+
+  for(;;) {
+      
+    try {
+      io_service.run() ;
+      break ;
+    }
+    catch( std::exception& e) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mod_verbio_tts threadFunc - Error: %s\n", e.what());
+    }
+  }
+  switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_verbio_tts threadFunc - ending\n");
+}
+
+
+/* Called by asio when our timeout expires */
+static void timer_cb(const boost::system::error_code & error, GlobalInfo_t *g)
+{
+  //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "timer_cb\n");
+
+  if(!error) {
+    CURLMcode rc = curl_multi_socket_action(g->multi, CURL_SOCKET_TIMEOUT, 0, &g->still_running);
+    mcode_test("timer_cb: curl_multi_socket_action", rc);
+    check_multi_info(g);
+  }
+}
+
+int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo_t *g) {
+
+  /* cancel running timer */
+  timer.cancel();
+
+  if(timeout_ms >= 0) {
+    // from libcurl 7.88.1-10+deb12u4 does not allow call curl_multi_socket_action or curl_multi_perform in curl_multi callback directly
+    timer.expires_from_now(boost::posix_time::millisec(timeout_ms ? timeout_ms : 1));
+    timer.async_wait(boost::bind(&timer_cb, boost::placeholders::_1, g));
+  }
+
+  return 0;
+}
+
+/* CURLOPT_WRITEFUNCTION */
+static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
+  bool fireEvent = false;
+  uint8_t *data = (uint8_t *) ptr;
+  size_t bytes_received = size * nmemb;
+  size_t total_bytes_to_process;
+  auto v = conn->verbio;
+  CircularBuffer_t *cBuffer = (CircularBuffer_t *) v->circularBuffer;
+  
+  if (conn->flushed || cBuffer == nullptr) {
+    /* this will abort the transfer */
+    return 0;
+  }
+  // Buffer to hold combined data if there is unprocessed byte from the last call.
+  std::unique_ptr<uint8_t[]> combinedData;
+
+  if (conn->has_last_byte) {
+    conn->has_last_byte = false;  // We'll handle the last_byte now, so toggle the flag off
+
+    // Allocate memory for the new data array
+    combinedData.reset(new uint8_t[bytes_received + 1]);
+
+    // Prepend the last byte from previous call
+    combinedData[0] = conn->last_byte;
+
+    // Copy the new data following the prepended byte
+    memcpy(combinedData.get() + 1, data, bytes_received);
+
+    // Point our data pointer to the new array
+    data = combinedData.get();
+
+    total_bytes_to_process = bytes_received + 1;
+  } else {
+    total_bytes_to_process = bytes_received;
+  }
+
+  // If we now have an odd total, save the last byte for next time
+  if ((total_bytes_to_process % sizeof(int16_t)) != 0) {
+    conn->last_byte = data[total_bytes_to_process - 1];
+    conn->has_last_byte = true;
+    total_bytes_to_process--;
+  }
+
+  int16_t* inputData = reinterpret_cast<int16_t*>(data);
+  if (0 == v->reads++) {
+    fireEvent = true;
+    // Verbio return PCM linear16 WAV file which contains 44 bytes headers, remove that.
+    inputData += 22;
+    total_bytes_to_process -= 44;
+  }
+  size_t numSamples = total_bytes_to_process / sizeof(int16_t);
+  {
+    switch_mutex_lock(v->mutex);
+
+    if (v->response_code > 0 && v->response_code != 200) {
+      std::string body((char *) ptr, bytes_received);
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: received body %s\n", body.c_str());
+      v->err_msg = strdup(body.c_str());
+      switch_mutex_unlock(v->mutex);
+      return 0;
+    }
+
+    /* cache file will stay in the mp3 format for size (smaller) and simplicity */
+    if (conn->file) fwrite(inputData, sizeof(int16_t), numSamples, conn->file);
+
+    // Resize the buffer if necessary
+    if (cBuffer->capacity() - cBuffer->size() < numSamples) {
+      //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "write_cb growing buffer\n"); 
+
+      //TODO: if buffer exceeds some max size, return CURL_WRITEFUNC_ERROR to abort the transfer
+      cBuffer->set_capacity(cBuffer->size() + std::max(numSamples, (size_t)BUFFER_GROW_SIZE));
+    }
+    
+    /* Push the data into the buffer */
+    cBuffer->insert(cBuffer->end(), inputData, inputData + numSamples);
+
+    switch_mutex_unlock(v->mutex);
+  }
+  if (fireEvent && v->session_id) {
+    auto endTime = std::chrono::high_resolution_clock::now();
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(endTime - conn->startTime);
+    auto time_to_first_byte_ms = std::to_string(duration.count());
+    switch_core_session_t* session = switch_core_session_locate(v->session_id);
+    if (session) {
+      switch_channel_t *channel = switch_core_session_get_channel(session);
+      switch_core_session_rwunlock(session);
+      if (channel) {
+        switch_event_t *event;
+        if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_START) == SWITCH_STATUS_SUCCESS) {
+          switch_channel_event_set_data(channel, event);
+
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: firing playback-started\n");
+
+          switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
+          if (v->name_lookup_time_ms) {
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_verbio_name_lookup_time_ms", v->name_lookup_time_ms);
+          }
+          if (v->connect_time_ms) {
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_verbio_connect_time_ms", v->connect_time_ms);
+          }
+          if (v->final_response_time_ms) {
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_verbio_final_response_time_ms", v->final_response_time_ms);
+          }
+          if (v->voice_name) {
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_verbio_voice_name", v->voice_name);
+          }
+          if (v->cache_filename) {
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", v->cache_filename);
+          }
+
+          switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
+          switch_event_fire(&event);
+        }
+        else {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
+        }
+      }
+      else {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
+      }
+    }
+    else {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: session %s not found\n", v->session_id);
+    }
+  }
+  return size*nmemb;
+}
+
+static bool parseHeader(const std::string& str, std::string& header, std::string& value) {
+    std::vector<std::string> parts;
+    boost::split(parts, str, boost::is_any_of(":"), boost::token_compress_on);
+
+    if (parts.size() != 2)
+        return false;
+
+    header = boost::trim_copy(parts[0]);
+    value = boost::trim_copy(parts[1]);
+    return true;
+}
+
+static int extract_response_code(const std::string& input) {
+  std::size_t space_pos = input.find(' ');
+  if (space_pos == std::string::npos) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid HTTP response format %s\n", input.c_str());
+    return 0;
+  }
+
+  std::size_t code_start_pos = space_pos + 1;
+  std::size_t code_end_pos = input.find(' ', code_start_pos);
+  if (code_end_pos == std::string::npos) {
+    code_end_pos = input.length();
+  }
+
+  std::string code_str = input.substr(code_start_pos, code_end_pos - code_start_pos);
+  int response_code = std::stoi(code_str);
+  return response_code;
+}
+
+static size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo_t *conn) {
+  size_t bytes_received = size * nitems;
+  const std::string prefix = "HTTP/";
+  verbio_t* v = conn->verbio;
+  std::string header, value;
+  std::string input(buffer, bytes_received);
+  if (parseHeader(input, header, value)) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s with value %s\n", header.c_str(), value.c_str());
+  }
+  else {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: %s\n", input.c_str());
+    if (input.rfind(prefix, 0) == 0) {
+      try {
+        v->response_code = extract_response_code(input);
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: parsed response code: %ld\n", v->response_code);
+      } catch (const std::invalid_argument& e) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "header_callback: invalid response code %s\n", input.substr(prefix.length()).c_str());
+      }
+    }
+  }
+  return bytes_received;
+}
+
+/* CURLOPT_OPENSOCKETFUNCTION */
+static curl_socket_t opensocket(void *clientp, curlsocktype purpose, struct curl_sockaddr *address) {
+  curl_socket_t sockfd = CURL_SOCKET_BAD;
+
+  //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "opensocket: %d\n", purpose);
+  /* restrict to IPv4 */
+  if(purpose == CURLSOCKTYPE_IPCXN && address->family == AF_INET) {
+    /* create a tcp socket object */
+    boost::asio::ip::tcp::socket *tcp_socket = new boost::asio::ip::tcp::socket(io_service);
+
+    /* open it and get the native handle*/
+    boost::system::error_code ec;
+    tcp_socket->open(boost::asio::ip::tcp::v4(), ec);
+
+    if(ec) {
+      /* An error occurred */
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't open socket [%ld][%s]\n", ec, ec.message().c_str());
+    }
+    else {
+      sockfd = tcp_socket->native_handle();
+
+      /* save it for monitoring */
+      socket_map.insert(std::pair<curl_socket_t, boost::asio::ip::tcp::socket *>(sockfd, tcp_socket));
+    }
+  }
+  return sockfd;
+}
+
+/* CURLOPT_CLOSESOCKETFUNCTION */
+static int close_socket(void *clientp, curl_socket_t item) {
+  //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "close_socket : %#X\n", item);
+
+  std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(item);
+  if(it != socket_map.end()) {
+    delete it->second;
+    socket_map.erase(it);
+  }
+  return 0;
+}
+
+
+extern "C" {
+  switch_status_t verbio_speech_load() {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "verbio_speech_loading..\n");
+    memset(&global, 0, sizeof(GlobalInfo_t));
+    global.multi = curl_multi_init();
+
+     if (!global.multi) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "verbio_speech_load curl_multi_init() failed, exiting!\n");
+      return SWITCH_STATUS_FALSE;
+    }
+
+    curl_multi_setopt(global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
+    curl_multi_setopt(global.multi, CURLMOPT_SOCKETDATA, &global);
+    curl_multi_setopt(global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
+    curl_multi_setopt(global.multi, CURLMOPT_TIMERDATA, &global);
+    curl_multi_setopt(global.multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
+
+    /* create temp folder for cache files */
+    const char* baseDir = std::getenv("JAMBONZ_TMP_CACHE_FOLDER");
+    if (!baseDir) {
+      baseDir = "/tmp/";
+    }
+    if (strcmp(baseDir, "/") == 0) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", baseDir);
+      return SWITCH_STATUS_FALSE;
+    }
+
+    fullDirPath = std::string(baseDir) + "tts-cache-files";
+
+    // Create the directory with read, write, and execute permissions for everyone
+    mode_t oldMask = umask(0);
+    int result = mkdir(fullDirPath.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
+    umask(oldMask);
+    if (result != 0) {
+      if (errno != EEXIST) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", fullDirPath.c_str());
+        fullDirPath = "";
+      }
+      else switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "folder %s already exists\n", fullDirPath.c_str());
+    }
+    else {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "created folder %s\n", fullDirPath.c_str());
+    }
+
+    /* start worker thread that handles transfers*/
+    std::thread t(threadFunc) ;
+    worker_thread.swap( t ) ;
+
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "verbio_speech_loaded..\n");
+
+
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t verbio_speech_unload() {
+    /* stop the ASIO IO service */
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "verbio_speech_unload: stopping io service\n");
+    io_service.stop();
+
+    /* Join the worker thread */
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "verbio_speech_unload: wait for worker thread to complete\n");
+    if (worker_thread.joinable()) {
+        worker_thread.join();
+    }
+
+    /* cleanup curl multi handle*/
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "verbio_speech_unload: release curl multi\n");
+    curl_multi_cleanup(global.multi);
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "verbio_speech_unload: completed\n");
+
+		return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t verbio_speech_open(verbio_t* verbio) {
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t verbio_speech_feed_tts(verbio_t* v, char* text, switch_speech_flag_t *flags) {
+    CURLMcode rc;
+
+    const int MAX_CHARS = 20;
+    char tempText[MAX_CHARS + 4]; // +4 for the ellipsis and null terminator
+
+    if (strlen(text) > MAX_CHARS) {
+        strncpy(tempText, text, MAX_CHARS);
+        strcpy(tempText + MAX_CHARS, "...");
+    } else {
+        strcpy(tempText, text);
+    }
+
+    /* open cache file */
+    if (v->cache_audio && fullDirPath.length() > 0) {
+      switch_uuid_t uuid;
+      char uuid_str[SWITCH_UUID_FORMATTED_LENGTH + 1];
+      char outfile[512] = "";
+      int fd;
+
+      switch_uuid_get(&uuid);
+      switch_uuid_format(uuid_str, &uuid);
+
+      switch_snprintf(outfile, sizeof(outfile), "%s%s%s.r8", fullDirPath.c_str(), SWITCH_PATH_SEPARATOR, uuid_str);
+      v->cache_filename = strdup(outfile);
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "writing audio cache file to %s\n", v->cache_filename);
+
+      mode_t oldMask = umask(0);
+      fd = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
+      umask(oldMask);
+      if (fd == -1 ) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
+      }
+      else {
+        v->file = fdopen(fd, "wb");
+        if (!v->file) {
+          close(fd);
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
+        }
+      }
+    }
+
+    if (!v->access_token) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "verbio_speech_feed_tts: no access_token provided\n");
+      return SWITCH_STATUS_FALSE;
+    }
+    /* format url*/
+    std::string url = "https://us.rest.speechcenter.verbio.com/api/v1/synthesize";
+
+    /* create the JSON body */
+    cJSON * jResult = cJSON_CreateObject();
+    cJSON_AddStringToObject(jResult, "voice_id", v->voice_name);
+    cJSON_AddStringToObject(jResult, "output_sample_rate", "8k");
+    cJSON_AddStringToObject(jResult, "output_encoding", "pcm16");
+    cJSON_AddStringToObject(jResult, "text", text);
+    char *json = cJSON_PrintUnformatted(jResult);
+
+    cJSON_Delete(jResult);
+
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "verbio_speech_feed_tts: [%s] [%s]\n", url.c_str(), tempText);
+
+    ConnInfo_t *conn = pool.malloc() ;
+
+    CURL* easy = createEasyHandle();
+    v->conn = (void *) conn ;
+    conn->verbio = v;
+    conn->easy = easy;
+    conn->global = &global;
+    conn->hdr_list = NULL ;
+    conn->file = v->file;
+    conn->body = json;
+    conn->flushed = false;
+    conn->has_last_byte = false;
+    conn->last_byte = 0;
+
+
+    v->circularBuffer = (void *) new CircularBuffer_t(BUFFER_GROW_SIZE);
+    // Always use verbio at rate 8000 for helping cache audio from jambonz.
+    if (v->rate != 8000) {
+      int err;
+      v->resampler = speex_resampler_init(1, 8000, v->rate, SWITCH_RESAMPLE_QUALITY, &err);
+      if (0 != err) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing resampler: %s.\n", speex_resampler_strerror(err));
+        return SWITCH_STATUS_FALSE;
+      }
+    }
+
+    std::ostringstream api_key_stream;
+    api_key_stream << "Authorization: Bearer " << v->access_token;
+
+    curl_easy_setopt(easy, CURLOPT_URL, url.c_str());
+    curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, write_cb);
+    curl_easy_setopt(easy, CURLOPT_WRITEDATA, conn);
+    curl_easy_setopt(easy, CURLOPT_ERRORBUFFER, conn->error);
+    curl_easy_setopt(easy, CURLOPT_PRIVATE, conn);
+    curl_easy_setopt(easy, CURLOPT_VERBOSE, 0L);
+    curl_easy_setopt(easy, CURLOPT_NOPROGRESS, 1L);
+    curl_easy_setopt(easy, CURLOPT_HEADERFUNCTION, header_callback);
+    curl_easy_setopt(easy, CURLOPT_HEADERDATA, conn);
+    
+    /* call this function to get a socket */
+    curl_easy_setopt(easy, CURLOPT_OPENSOCKETFUNCTION, opensocket);
+
+    /* call this function to close a socket */
+    curl_easy_setopt(easy, CURLOPT_CLOSESOCKETFUNCTION, close_socket);
+
+    conn->hdr_list = curl_slist_append(conn->hdr_list, api_key_stream.str().c_str());
+    conn->hdr_list = curl_slist_append(conn->hdr_list, "Content-Type: application/json");
+    curl_easy_setopt(easy, CURLOPT_HTTPHEADER, conn->hdr_list);
+
+    curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
+    //curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
+
+    // libcurl adding random byte to the response body that creates white noise to audio file
+    // https://github.com/curl/curl/issues/10525
+    const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
+    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
+
+    rc = curl_multi_add_handle(global.multi, conn->easy);
+    mcode_test("new_conn: curl_multi_add_handle", rc);
+
+    /* start a timer to measure the duration until we receive first byte of audio */
+    conn->startTime = std::chrono::high_resolution_clock::now();
+
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "verbio_speech_feed_tts: called curl_multi_add_handle\n");
+
+
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t verbio_speech_read_tts(verbio_t* v, void *data, size_t *datalen, switch_speech_flag_t *flags) {
+    CircularBuffer_t *cBuffer = (CircularBuffer_t *) v->circularBuffer;
+    std::vector<uint16_t> pcm_data;
+
+    {
+      switch_mutex_lock(v->mutex);
+      ConnInfo_t *conn = (ConnInfo_t *) v->conn;
+      if (v->response_code > 0 && v->response_code != 200) {
+        switch_mutex_unlock(v->mutex);
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "verbio_speech_read_tts, returning failure\n") ;  
+        return SWITCH_STATUS_FALSE;
+      }
+      if (conn && conn->flushed) {
+        switch_mutex_unlock(v->mutex);
+        return SWITCH_STATUS_BREAK;
+      }
+      if (cBuffer->empty()) {
+        if (v->draining) {
+          switch_mutex_unlock(v->mutex);
+          return SWITCH_STATUS_BREAK;
+        }
+        /* no audio available yet so send silence */
+        memset(data, 255, *datalen);
+        switch_mutex_unlock(v->mutex);
+        return SWITCH_STATUS_SUCCESS;
+      }
+      size_t size = std::min((*datalen/(2 * v->rate / 8000)), cBuffer->size());
+      pcm_data.insert(pcm_data.end(), cBuffer->begin(), cBuffer->begin() + size);
+      cBuffer->erase(cBuffer->begin(), cBuffer->begin() + size);
+      switch_mutex_unlock(v->mutex);
+    }
+
+    size_t data_size = pcm_data.size();
+
+    if (v->resampler) {
+      std::vector<int16_t> in(pcm_data.begin(), pcm_data.end());
+
+      std::vector<int16_t> out((*datalen));
+      spx_uint32_t in_len = data_size;
+      spx_uint32_t out_len = out.size();
+      speex_resampler_process_interleaved_int(v->resampler, in.data(), &in_len, out.data(), &out_len);
+
+      if (out_len > out.size()) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Resampler output exceeded maximum buffer size!\n");
+        return SWITCH_STATUS_FALSE;
+      }
+
+      memcpy(data, out.data(), out_len * sizeof(int16_t));
+      *datalen = out_len * sizeof(int16_t);
+    } else {
+      memcpy(data, pcm_data.data(), pcm_data.size() * sizeof(uint16_t));
+      *datalen = pcm_data.size() * sizeof(uint16_t);
+    }
+
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+  switch_status_t verbio_speech_flush_tts(verbio_t* v) {
+    bool download_complete = v->response_code == 200;
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "verbio_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;  
+
+    ConnInfo_t *conn = (ConnInfo_t *) v->conn;
+    CircularBuffer_t *cBuffer = (CircularBuffer_t *) v->circularBuffer;
+    delete cBuffer;
+    v->circularBuffer = nullptr ;
+
+    // destroy resampler
+    if (v->resampler) {
+      speex_resampler_destroy(v->resampler);
+      v->resampler = NULL;
+    }
+
+    if (conn) {
+      conn->flushed = true;
+      if (!download_complete) {
+        if (conn->file) {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "closing audio cache file %s because download was interrupted\n", v->cache_filename);
+          if (fclose(conn->file) != 0) {
+            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "error closing audio cache file\n");
+          }
+          conn->file = nullptr ;
+        }
+
+        if (v->cache_filename) {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", v->cache_filename);
+          if (unlink(v->cache_filename) != 0) {
+            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
+              v->cache_filename, errno, strerror(errno));
+          }
+          free(v->cache_filename);
+          v->cache_filename = nullptr ;
+        }
+      }
+    }
+    if (v->session_id) {
+      switch_core_session_t* session = switch_core_session_locate(v->session_id);
+      if (session) {
+        switch_channel_t *channel = switch_core_session_get_channel(session);
+        switch_core_session_rwunlock(session);
+        if (channel) {
+          switch_event_t *event;
+          if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_STOP) == SWITCH_STATUS_SUCCESS) {
+            switch_channel_event_set_data(channel, event);
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
+            switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_verbio_response_code", std::to_string(v->response_code).c_str());
+            if (v->cache_filename && v->response_code == 200) {
+              switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", v->cache_filename);
+            }
+            if (v->response_code != 200 && v->err_msg) {
+              switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_error", v->err_msg);
+            }
+            switch_event_fire(&event);
+          }
+          else {
+            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
+          }
+        }
+        else {
+          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
+        }
+      }
+    }
+    return SWITCH_STATUS_SUCCESS;
+  }
+
+	switch_status_t verbio_speech_close(verbio_t* w) {
+    switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "verbio_speech_close\n") ;
+		return SWITCH_STATUS_SUCCESS;
+	}
+}

mod_verbio_tts/verbio_glue.h

@@ -0,0 +1,12 @@
+#ifndef __VERBIO_GLUE_H__
+#define __VERBIO_GLUE_H__
+
+switch_status_t verbio_speech_load();
+switch_status_t verbio_speech_open(verbio_t* verbio);
+switch_status_t verbio_speech_feed_tts(verbio_t* verbio, char* text, switch_speech_flag_t *flags);
+switch_status_t verbio_speech_read_tts(verbio_t* verbio, void *data, size_t *datalen, switch_speech_flag_t *flags);
+switch_status_t verbio_speech_flush_tts(verbio_t* verbio);
+switch_status_t verbio_speech_close(verbio_t* verbio);
+switch_status_t verbio_speech_unload();
+
+#endif

mod_whisper_tts/mod_whisper_tts.c

@@ -90,6 +90,9 @@ static switch_status_t w_speech_feed_tts(switch_speech_handle_t *sh, char *text,
   whisper_t *w = createOrRetrievePrivateData(sh);
   w->draining = 0;
   w->reads = 0;
+  w->response_code = 0;
+  w->err_msg = NULL;
+  w->playback_start_sent = 0;
 
   switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_speech_feed_tts\n");
 

mod_whisper_tts/mod_whisper_tts.h

@@ -35,6 +35,7 @@ typedef struct whisper_data {
   int draining;
   int reads;
   int cache_audio;
+  int playback_start_sent;
 
 	void *conn;
   void *circularBuffer;

mod_whisper_tts/whisper_glue.cpp

@@ -64,6 +64,27 @@ std::string secondsToMillisecondsString(double seconds) {
     return std::to_string(milliseconds_long);
 }
 
+static std::string getEnvVar(const std::string& varName) {
+    const char* val = std::getenv(varName.c_str());
+    return val ? std::string(val) : "";
+}
+
+static long getConnectionTimeout() {
+    std::string connectTimeoutStr = getEnvVar("WHISPER_TTS_CURL_CONNECT_TIMEOUT");
+
+    if (connectTimeoutStr.empty()) {
+        connectTimeoutStr = getEnvVar("TTS_CURL_CONNECT_TIMEOUT");
+    }
+
+    long connectTimeout = 10L;
+
+    if (!connectTimeoutStr.empty()) {
+        connectTimeout = std::stol(connectTimeoutStr);
+    }
+
+    return connectTimeout;
+}
+
 static CURL* createEasyHandle(void) {
   CURL* easy = curl_easy_init();
   if(!easy) {
@@ -76,7 +97,7 @@ static CURL* createEasyHandle(void) {
 
   // set connect timeout to 3 seconds and total timeout to 109 seconds
   curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
-  curl_easy_setopt(easy, CURLOPT_TIMEOUT, 10L);
+  curl_easy_setopt(easy, CURLOPT_TIMEOUT, getConnectionTimeout());
 
   return easy ;    
 }
@@ -528,6 +549,7 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
 
           switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
           switch_event_fire(&event);
+          w->playback_start_sent = 1;
         }
         else {
           switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
@@ -837,17 +859,9 @@ extern "C" {
 
     w->circularBuffer = (void *) new CircularBuffer_t(8192);
 
-    if (w->session_id) {
-      int err;
-      switch_codec_implementation_t read_impl;
-      switch_core_session_t *psession = switch_core_session_locate(w->session_id);
-      switch_core_session_get_read_impl(psession, &read_impl);
-      switch_core_session_rwunlock(psession);
-      uint32_t samples_per_second = !strcasecmp(read_impl.iananame, "g722") ? read_impl.actual_samples_per_second : read_impl.samples_per_second;
-      if (mpg123_param(mh, MPG123_FORCE_RATE, samples_per_second /*Hz*/, 0) != MPG123_OK) {
-        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_param!\n");
-        return SWITCH_STATUS_FALSE;
-      }
+    if (mpg123_param(mh, MPG123_FORCE_RATE, w->rate /*Hz*/, 0) != MPG123_OK) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_param!\n");
+      return SWITCH_STATUS_FALSE;
     }
 
     std::ostringstream api_key_stream;
@@ -876,7 +890,10 @@ extern "C" {
     curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
     //curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
 
-    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+    // libcurl adding random byte to the response body that creates white noise to audio file
+    // https://github.com/curl/curl/issues/10525
+    const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
+    curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
 
     rc = curl_multi_add_handle(global.multi, conn->easy);
     mcode_test("new_conn: curl_multi_add_handle", rc);
@@ -946,18 +963,18 @@ extern "C" {
           }
           conn->file = nullptr ;
         }
-
-        if (w->cache_filename) {
-          switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", w->cache_filename);
-          if (unlink(w->cache_filename) != 0) {
-            switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
-              w->cache_filename, errno, strerror(errno));
-          }
-          free(w->cache_filename);
-          w->cache_filename = nullptr ;
-        }
       }
     }
+    // if playback_start event has not been sent, delete the file
+    if (w->cache_filename && !w->playback_start_sent) {
+      switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", w->cache_filename);
+      if (unlink(w->cache_filename) != 0) {
+        switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n", 
+          w->cache_filename, errno, strerror(errno));
+      }
+      free(w->cache_filename);
+      w->cache_filename = nullptr ;
+    }
     if (w->session_id) {
       switch_core_session_t* session = switch_core_session_locate(w->session_id);
       if (session) {