Compare commits

..

6 Commits

Author SHA1 Message Date
Dave Horton e431c5d159 changes to mod_azure_tts for event handling, resolve session locking issue in mod_whisper (#37) 2024-04-07 11:39:09 -04:00
Dave Horton a96fb2b4b2 Fixes/mod azure tts (#36)
* fixes for unlocking mutex and minimizing time under lock

* call .get() on future returned from speechSynthesizer->SpeakTextAsync

* mod_azure_tts: various fixes, including dangling session lock and reducing latency

* mod_deepgram_transcribe: add support for transcribing filler words
2024-04-06 12:26:56 -04:00
Hoan Luu Huu 94361f1d20 mod_dub support sayOnTrack Deepgram (#35)
* mod_dub support sayOnTrack Deepgram

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* mod_dub sayOnTrack support azure

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* wip

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* wip

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* wip

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* wip

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* wip

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* support whisper

* wip

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

---------

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>
2024-04-05 07:16:31 -04:00
Hoan Luu Huu d05cfb8ef0 remove deepgram tts linear16 wav 44 bytes header (#34)
* remove deepgram tts linear16 wav 44 bytes header

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* wip

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

---------

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>
2024-04-05 07:15:02 -04:00
Hoan Luu Huu 8e7f4a3fab Feat/mod deepgram tts (#33)
* deepgram tts mode

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* wip

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* wip

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* fix crash audio sound

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* wip

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* wip

* run with opus

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>

* dg-request-id check

---------

Signed-off-by: Hoan HL <quan.luuhoang8@gmail.com>
2024-04-04 15:54:22 -04:00
Dave Horton fa4930d53e Fix/docker build (#32)
* explicit flags for boost

* remove pragma statements

* remove some logging in mod_azure_tts

* pragma back in

* compile error fixes

* mod_azure_tts: set Playback-File-Type to tts_stream
2024-04-03 13:49:05 -04:00
24 changed files with 1444 additions and 113 deletions
+1 -1
View File
@@ -15,4 +15,4 @@ endif
endif
mod_audio_fork_la_LIBADD = $(switch_builddir)/libfreeswitch.la
mod_audio_fork_la_LDFLAGS = -avoid-version -module -no-undefined -shared `pkg-config --libs libwebsockets` -lstdc++
mod_audio_fork_la_LDFLAGS = -avoid-version -module -no-undefined -shared `pkg-config --libs libwebsockets` -lstdc++ -lboost_system -lboost_thread
+2
View File
@@ -1,6 +1,7 @@
#include "vector_math.h"
#include <assert.h>
#include <string.h>
#include <cstdlib>
#define GRANULAR_VOLUME_MAX (50)
#define SMAX 32767
@@ -153,6 +154,7 @@ typedef union {
} vector_data_t;
#else
#pragma message("Building without vector math support")
void vector_add(int16_t* a, int16_t* b, size_t len) {
for (size_t i = 0; i < len; i++) {
a[i] += b[i];
+1 -1
View File
@@ -7,4 +7,4 @@ mod_azure_tts_la_CFLAGS = $(AM_CFLAGS)
mod_azure_tts_la_CXXFLAGS = $(AM_CXXFLAGS) -std=c++14 -I/usr/local/include/MicrosoftSpeechSDK/cxx_api -I/usr/local/include/MicrosoftSpeechSDK/c_api
mod_azure_tts_la_LIBADD = $(switch_builddir)/libfreeswitch.la
mod_azure_tts_la_LDFLAGS = -avoid-version -module -no-undefined -L/usr/local/lib/MicrosoftSpeechSDK/x64 -lMicrosoft.CognitiveServices.Speech.core -shared `pkg-config --libs boost` -lstdc++
mod_azure_tts_la_LDFLAGS = -avoid-version -module -no-undefined -L/usr/local/lib/MicrosoftSpeechSDK/x64 -lMicrosoft.CognitiveServices.Speech.core -shared -lstdc++ -lboost_system -lboost_thread
+80 -75
View File
@@ -7,6 +7,7 @@
#include <cstdlib>
#include <string>
#include <chrono>
#include <thread>
#define BUFFER_SIZE 8129
@@ -16,6 +17,34 @@ using namespace Microsoft::CognitiveServices::Speech;
static std::string fullDirPath;
static void start_synthesis(std::shared_ptr<SpeechSynthesizer> speechSynthesizer, const char* text, azure_t* a) {
try {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "start_synthesis calling \n");
auto result = std::strncmp(text, "<speak", 6) == 0 ?
speechSynthesizer->SpeakSsmlAsync(text).get() :
speechSynthesizer->SpeakTextAsync(text).get();
if (result->Reason == ResultReason::SynthesizingAudioCompleted) {
a->response_code = 200;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "start_synthesis completed id %s, audio data - bytes: %ld, duration: %ldms\n",
result->ResultId.c_str(), result->GetAudioLength(), result->AudioDuration.count());
} else if (result->Reason == ResultReason::Canceled) {
auto cancellation = SpeechSynthesisCancellationDetails::FromResult(result);
a->response_code = static_cast<long int>(cancellation->ErrorCode);
a->err_msg = strdup(cancellation->ErrorDetails.c_str());
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error synthesizing text %d with error string: %s.\n",
static_cast<int>(cancellation->ErrorCode), cancellation->ErrorDetails.c_str());
} else {
a->response_code = 500;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error synthsize text %s (%d).\n", text, static_cast<int>(result->Reason));
}
} catch (const std::exception& e) {
a->response_code = 500;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mod_azure_tts: Exception in start_synthesis %s\n", e.what());
}
a->draining = 1;
}
extern "C" {
switch_status_t azure_speech_load() {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "azure_speech_loading..\n");
@@ -109,8 +138,12 @@ extern "C" {
if (a->session_id) {
int err;
switch_codec_implementation_t read_impl;
/* lock and unlock session */
switch_core_session_t *psession = switch_core_session_locate(a->session_id);
switch_core_session_get_read_impl(psession, &read_impl);
switch_core_session_rwunlock(psession);
uint32_t samples_per_second = !strcasecmp(read_impl.iananame, "g722") ? read_impl.actual_samples_per_second : read_impl.samples_per_second;
a->samples_rate = samples_per_second;
if (samples_per_second != 8000 /*Hz*/) {
@@ -150,43 +183,32 @@ extern "C" {
speechSynthesizer->SynthesisStarted += [a](const SpeechSynthesisEventArgs& e) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts SynthesisStarted\n");
a->response_code = 200;
};
speechSynthesizer->Synthesizing += [a](const SpeechSynthesisEventArgs& e) {
if (a->flushed) return;
bool fireEvent = false;
CircularBuffer_t *cBuffer = (CircularBuffer_t *) a->circularBuffer;
std::vector<uint16_t> pcm_data;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Synthesizing: received data\n");
if (a->flushed) {
return;
auto audioData = e.Result->GetAudioData();
if (a->file) {
fwrite(audioData->data(), 1, audioData->size(), a->file);
}
{
switch_mutex_lock(a->mutex);
auto audioData = e.Result->GetAudioData();
for (size_t i = 0; i < audioData->size(); i += sizeof(int16_t)) {
int16_t value = static_cast<int16_t>((*audioData)[i]) | (static_cast<int16_t>((*audioData)[i + 1]) << 8);
pcm_data.push_back(value);
}
/* and write to the file */
size_t bytesResampled = pcm_data.size() * sizeof(uint16_t);
if (a->file) fwrite(pcm_data.data(), sizeof(uint16_t), pcm_data.size(), a->file);
/**
* this sort of reinterpretation can be dangerous as a general rule, but in this case we know that the data
* is 16-bit PCM, so it's safe to do this and its much faster than copying the data byte by byte
*/
const uint16_t* begin = reinterpret_cast<const uint16_t*>(audioData->data());
const uint16_t* end = reinterpret_cast<const uint16_t*>(audioData->data() + audioData->size());
// Resize the buffer if necessary
if (cBuffer->capacity() - cBuffer->size() < (bytesResampled / sizeof(uint16_t))) {
//TODO: if buffer exceeds some max size, return CURL_WRITEFUNC_ERROR to abort the transfer
cBuffer->set_capacity(cBuffer->size() + std::max((bytesResampled / sizeof(uint16_t)), (size_t)BUFFER_SIZE));
}
/* Push the data into the buffer */
cBuffer->insert(cBuffer->end(), pcm_data.data(), pcm_data.data() + pcm_data.size());
switch_mutex_unlock(a->mutex);
/* lock as briefly as possible */
switch_mutex_lock(a->mutex);
if (cBuffer->capacity() - cBuffer->size() < audioData->size()) {
cBuffer->set_capacity(cBuffer->size() + std::max( audioData->size(), (size_t)BUFFER_SIZE));
}
cBuffer->insert(cBuffer->end(), begin, end);
switch_mutex_unlock(a->mutex);
if (0 == a->reads++) {
fireEvent = true;
@@ -200,10 +222,12 @@ extern "C" {
switch_core_session_t* session = switch_core_session_locate(a->session_id);
if (session) {
switch_channel_t *channel = switch_core_session_get_channel(session);
switch_core_session_rwunlock(session);
if (channel) {
switch_event_t *event;
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_START) == SWITCH_STATUS_SUCCESS) {
switch_channel_event_set_data(channel, event);
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
if (a->cache_filename) {
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", a->cache_filename);
@@ -215,31 +239,12 @@ extern "C" {
}else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "speechSynthesizer->Synthesizing: channel not found\n");
}
switch_core_session_rwunlock(session);
}
}
};
speechSynthesizer->SynthesisCompleted += [a](const SpeechSynthesisEventArgs& e) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts SynthesisCompleted\n");
a->draining = 1;
};
speechSynthesizer->SynthesisCanceled += [a](const SpeechSynthesisEventArgs& e) {
if (e.Result->Reason == ResultReason::Canceled) {
auto cancellation = SpeechSynthesisCancellationDetails::FromResult(e.Result);
a->response_code = static_cast<long int>(cancellation->ErrorCode);
a->err_msg = strdup(cancellation->ErrorDetails.c_str());
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error synthsize tex %d with error string: %s.\n", static_cast<int>(cancellation->ErrorCode), cancellation->ErrorDetails.c_str());
}
};
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts before sending synthesize request\n");
if (std::strncmp(text, "<speak", 6) == 0) {
speechSynthesizer->SpeakSsmlAsync(text);
} else {
speechSynthesizer->SpeakTextAsync(text);
}
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts sent synthesize request\n");
std::thread(start_synthesis, speechSynthesizer, text, a).detach();
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts sent synthesize request\n");
return SWITCH_STATUS_SUCCESS;
}
@@ -247,33 +252,31 @@ extern "C" {
CircularBuffer_t *cBuffer = (CircularBuffer_t *) a->circularBuffer;
std::vector<uint16_t> pcm_data;
{
switch_mutex_lock(a->mutex);
if (a->response_code > 0 && a->response_code != 200) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "azure_speech_read_tts, returning failure\n") ;
return SWITCH_STATUS_FALSE;
}
if (a->flushed) {
if (a->response_code > 0 && a->response_code != 200) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "azure_speech_read_tts, returning failure\n") ;
return SWITCH_STATUS_FALSE;
}
if (a->flushed) {
return SWITCH_STATUS_BREAK;
}
switch_mutex_lock(a->mutex);
size_t bufSize = cBuffer->size();
if (cBuffer->empty()) {
switch_mutex_unlock(a->mutex);
if (a->draining) {
return SWITCH_STATUS_BREAK;
}
if (cBuffer->empty()) {
if (a->draining) {
switch_mutex_unlock(a->mutex);
return SWITCH_STATUS_BREAK;
}
/* no audio available yet so send silence */
memset(data, 255, *datalen);
switch_mutex_unlock(a->mutex);
return SWITCH_STATUS_SUCCESS;
}
// azure returned 8000hz 16 bit data, we have to take enough data based on call sample rate.
size_t size = a->samples_rate ?
std::min((*datalen/(2 * a->samples_rate / 8000)), cBuffer->size()) :
std::min((*datalen/2), cBuffer->size());
pcm_data.insert(pcm_data.end(), cBuffer->begin(), cBuffer->begin() + size);
cBuffer->erase(cBuffer->begin(), cBuffer->begin() + size);
switch_mutex_unlock(a->mutex);
/* no audio available yet so send silence */
memset(data, 255, *datalen);
return SWITCH_STATUS_SUCCESS;
}
// azure returned 8000hz 16 bit data, we have to take enough data based on call sample rate.
size_t size = a->samples_rate ?
std::min((*datalen/(2 * a->samples_rate / 8000)), bufSize) :
std::min((*datalen/2), bufSize);
pcm_data.insert(pcm_data.end(), cBuffer->begin(), cBuffer->begin() + size);
cBuffer->erase(cBuffer->begin(), cBuffer->begin() + size);
switch_mutex_unlock(a->mutex);
size_t data_size = pcm_data.size();
@@ -314,7 +317,7 @@ extern "C" {
a->flushed = 1;
if (!download_complete) {
if (a->file) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "closing audio cache file %s because download was interrupted\n", a->cache_filename);
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "closing audio cache file %s because download was interrupted\n", a->cache_filename);
if (fclose(a->file) != 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "error closing audio cache file\n");
}
@@ -322,7 +325,7 @@ extern "C" {
}
if (a->cache_filename) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", a->cache_filename);
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", a->cache_filename);
if (unlink(a->cache_filename) != 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n",
a->cache_filename, errno, strerror(errno));
@@ -335,6 +338,9 @@ extern "C" {
switch_core_session_t* session = switch_core_session_locate(a->session_id);
if (session) {
switch_channel_t *channel = switch_core_session_get_channel(session);
/* unlock as quickly as possible */
switch_core_session_rwunlock(session);
if (channel) {
switch_event_t *event;
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_STOP) == SWITCH_STATUS_SUCCESS) {
@@ -356,7 +362,6 @@ extern "C" {
else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
}
switch_core_session_rwunlock(session);
}
}
+5 -9
View File
@@ -6,7 +6,7 @@ SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_azure_tts_shutdown);
SWITCH_MODULE_DEFINITION(mod_azure_tts, mod_azure_tts_load, mod_azure_tts_shutdown, NULL);
static void clearAzure(azure_t* a, int freeAll) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "clearAzure\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "clearAzure\n");
if (a->cache_filename) free(a->cache_filename);
if (a->api_key) free(a->api_key);
@@ -46,7 +46,7 @@ static azure_t * createOrRetrievePrivateData(switch_speech_handle_t *sh) {
sh->private_info = a;
memset(a, 0, sizeof(*a));
switch_mutex_init(&a->mutex, SWITCH_MUTEX_NESTED, sh->memory_pool);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "allocated azure_t\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "allocated azure_t\n");
}
return a;
}
@@ -56,7 +56,7 @@ switch_status_t a_speech_open(switch_speech_handle_t *sh, const char *voice_name
azure_t *a = createOrRetrievePrivateData(sh);
a->voice_name = strdup(voice_name);
a->rate = rate;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "a_speech_open voice: %s, rate %d, channels %d\n", voice_name, rate, channels);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "a_speech_open voice: %s, rate %d, channels %d\n", voice_name, rate, channels);
return azure_speech_open(a);
}
@@ -64,7 +64,7 @@ static switch_status_t a_speech_close(switch_speech_handle_t *sh, switch_speech_
{
switch_status_t rc;
azure_t *a = createOrRetrievePrivateData(sh);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "a_speech_close\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "a_speech_close\n");
switch_mutex_destroy(a->mutex);
@@ -84,8 +84,6 @@ static switch_status_t a_speech_feed_tts(switch_speech_handle_t *sh, char *text,
a->flushed = 0;
a->samples_rate = 0;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "a_speech_feed_tts\n");
return azure_speech_feed_tts(a, text, flags);
}
@@ -95,7 +93,6 @@ static switch_status_t a_speech_feed_tts(switch_speech_handle_t *sh, char *text,
static switch_status_t a_speech_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, switch_speech_flag_t *flags)
{
azure_t *a = createOrRetrievePrivateData(sh);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "a_speech_read_tts\n");
return azure_speech_read_tts(a, data, datalen, flags);
}
@@ -105,7 +102,6 @@ static switch_status_t a_speech_read_tts(switch_speech_handle_t *sh, void *data,
static void a_speech_flush_tts(switch_speech_handle_t *sh)
{
azure_t *a = createOrRetrievePrivateData(sh);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_speech_flush_tts\n");
azure_speech_flush_tts(a);
clearAzure(a, 0);
@@ -114,7 +110,7 @@ static void a_speech_flush_tts(switch_speech_handle_t *sh)
static void a_text_param_tts(switch_speech_handle_t *sh, char *param, const char *val)
{
azure_t *a = createOrRetrievePrivateData(sh);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "a_text_param_tts: %s=%s\n", param, val);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "a_text_param_tts: %s=%s\n", param, val);
if (0 == strcmp(param, "api_key")) {
if (a->api_key) free(a->api_key);
a->api_key = strdup(val);
+3 -1
View File
@@ -25,10 +25,11 @@ Stop transcription on the channel.
| variable | Description |
| --- | ----------- |
| DEEPGRAM_API_KEY | Deepgram API key used to authenticate |
| DEEPGRAM_SPEECH_TIER | https://developers.deepgram.com/documentation/features/tier/ |
| DEEPGRAM_SPEECH_CUSTOM_MODEL | custom model id |
| DEEPGRAM_SPEECH_MODEL | https://developers.deepgram.com/documentation/features/model/ |
| DEEPGRAM_SPEECH_MODEL_VERSION | https://developers.deepgram.com/documentation/features/version/ |
| DEEPGRAM_SPEECH_ENABLE_SMART_FORMAT | https://developers.deepgram.com/docs/smart-format |
| DEEPGRAM_SPEECH_ENABLE_FILLER_WORDS | https://developers.deepgram.com/docs/filler-words |
| DEEPGRAM_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION | https://developers.deepgram.com/documentation/features/punctuate/ |
| DEEPGRAM_SPEECH_PROFANITY_FILTER | https://developers.deepgram.com/documentation/features/profanity-filter/ |
| DEEPGRAM_SPEECH_REDACT | https://developers.deepgram.com/documentation/features/redact/ |
@@ -42,6 +43,7 @@ Stop transcription on the channel.
| DEEPGRAM_SPEECH_REPLACE | https://developers.deepgram.com/documentation/features/replace/ |
| DEEPGRAM_SPEECH_TAG | https://developers.deepgram.com/documentation/features/tag/ |
| DEEPGRAM_SPEECH_ENDPOINTING | https://developers.deepgram.com/documentation/features/endpointing/ |
| DEEPGRAM_SPEECH_UTTERANCE_END_MS | https://developers.deepgram.com/docs/utterance-end |
| DEEPGRAM_SPEECH_VAD_TURNOFF | https://developers.deepgram.com/documentation/features/voice-activity-detection/ |
@@ -184,6 +184,9 @@ namespace {
*
*/
}
if (var = switch_channel_get_variable(channel, "DEEPGRAM_SPEECH_ENABLE_FILLER_WORDS")) {
oss << "&filler_words=true";
}
if (var = switch_channel_get_variable(channel, "DEEPGRAM_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION")) {
oss << "&punctuate=true";
}
+8
View File
@@ -0,0 +1,8 @@
include $(top_srcdir)/build/modmake.rulesam
MODNAME=mod_deepgram_tts
mod_LTLIBRARIES = mod_deepgram_tts.la
mod_deepgram_tts_la_SOURCES = mod_deepgram_tts.c deepgram_glue.cpp
mod_deepgram_tts_la_CFLAGS = $(AM_CFLAGS)
mod_deepgram_tts_la_LIBADD = $(switch_builddir)/libfreeswitch.la
mod_deepgram_tts_la_LDFLAGS = -avoid-version -module -no-undefined -shared -lstdc++ -lboost_system -lboost_thread
+922
View File
@@ -0,0 +1,922 @@
#include "mod_deepgram_tts.h"
#include <switch.h>
#include <switch_json.h>
#include <curl/curl.h>
#include <cstdlib>
#include <boost/circular_buffer.hpp>
#include <boost/thread.hpp>
#include <boost/asio.hpp>
#include <boost/asio/ssl.hpp>
#include <boost/pool/object_pool.hpp>
#include <boost/bind/bind.hpp>
#include <boost/tokenizer.hpp>
#include <boost/foreach.hpp>
#include <boost/asio.hpp>
#include <boost/assign/list_of.hpp>
#include <boost/algorithm/string.hpp>
#include <speex/speex_resampler.h>
#define BUFFER_GROW_SIZE (80000)
typedef boost::circular_buffer<uint16_t> CircularBuffer_t;
/* Global information, common to all connections */
typedef struct
{
CURLM *multi;
int still_running;
} GlobalInfo_t;
static GlobalInfo_t global;
/* Information associated with a specific easy handle */
typedef struct
{
CURL *easy;
deepgram_t* deepgram;
char* body;
struct curl_slist *hdr_list;
GlobalInfo_t *global;
char error[CURL_ERROR_SIZE];
FILE* file;
std::chrono::time_point<std::chrono::high_resolution_clock> startTime;
bool flushed;
bool has_last_byte;
uint8_t last_byte;
} ConnInfo_t;
static boost::object_pool<ConnInfo_t> pool ;
static std::map<curl_socket_t, boost::asio::ip::tcp::socket *> socket_map;
static boost::asio::io_service io_service;
static boost::asio::deadline_timer timer(io_service);
static std::string fullDirPath;
static std::thread worker_thread;
std::string secondsToMillisecondsString(double seconds) {
// Convert to milliseconds
double milliseconds = seconds * 1000.0;
// Truncate to remove fractional part
long milliseconds_long = static_cast<long>(milliseconds);
// Convert to string
return std::to_string(milliseconds_long);
}
static CURL* createEasyHandle(void) {
CURL* easy = curl_easy_init();
if(!easy) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "curl_easy_init() failed!\n");
return nullptr ;
}
curl_easy_setopt(easy, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(easy, CURLOPT_USERAGENT, "jambonz/0.8.5");
// set connect timeout to 3 seconds and total timeout to 109 seconds
curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
curl_easy_setopt(easy, CURLOPT_TIMEOUT, 10L);
return easy ;
}
static void cleanupConn(ConnInfo_t *conn) {
auto d = conn->deepgram;
if( conn->hdr_list ) {
curl_slist_free_all(conn->hdr_list);
conn->hdr_list = nullptr ;
}
curl_easy_cleanup(conn->easy);
if (conn->file) {
if (fclose(conn->file) != 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error closing audio cache file\n");
}
conn->file = nullptr ;
}
d->conn = nullptr ;
d->draining = 1;
memset(conn, 0, sizeof(ConnInfo_t));
pool.destroy(conn) ;
}
/* Check for completed transfers, and remove their easy handles */
void check_multi_info(GlobalInfo_t *g) {
CURLMsg *msg;
int msgs_left;
ConnInfo_t *conn;
CURL *easy;
CURLcode res;
while((msg = curl_multi_info_read(g->multi, &msgs_left))) {
if(msg->msg == CURLMSG_DONE) {
long response_code;
double namelookup=0, connect=0, total=0 ;
char *ct = NULL ;
easy = msg->easy_handle;
res = msg->data.result;
curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &response_code);
curl_easy_getinfo(easy, CURLINFO_CONTENT_TYPE, &ct);
curl_easy_getinfo(easy, CURLINFO_NAMELOOKUP_TIME, &namelookup);
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connect);
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &total);
auto d = conn->deepgram;
d->response_code = response_code;
if (ct) d->ct = strdup(ct);
std::string name_lookup_ms = secondsToMillisecondsString(namelookup);
std::string connect_ms = secondsToMillisecondsString(connect);
std::string final_response_time_ms = secondsToMillisecondsString(total);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,
"mod_deepgram_tts: response: %ld, content-type %s,"
"dns(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld, "
"connect(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld, "
"total(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld\n",
response_code, ct,
(long)(namelookup), (long)(fmod(namelookup, 1.0) * 1000000),
(long)(connect), (long)(fmod(connect, 1.0) * 1000000),
(long)(total), (long)(fmod(total, 1.0) * 1000000));
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "name lookup time: %s\n", name_lookup_ms.c_str());
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "connect time: %s\n", connect_ms.c_str());
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "final response time: %s\n", final_response_time_ms.c_str());
d->name_lookup_time_ms = strdup(name_lookup_ms.c_str());
d->connect_time_ms = strdup(connect_ms.c_str());
d->final_response_time_ms = strdup(final_response_time_ms.c_str());
curl_multi_remove_handle(g->multi, easy);
cleanupConn(conn);
}
}
}
int mcode_test(const char *where, CURLMcode code) {
if(CURLM_OK != code) {
const char *s;
switch(code) {
case CURLM_CALL_MULTI_PERFORM:
s = "CURLM_CALL_MULTI_PERFORM";
break;
case CURLM_BAD_HANDLE:
s = "CURLM_BAD_HANDLE";
break;
case CURLM_BAD_EASY_HANDLE:
s = "CURLM_BAD_EASY_HANDLE";
break;
case CURLM_OUT_OF_MEMORY:
s = "CURLM_OUT_OF_MEMORY";
break;
case CURLM_INTERNAL_ERROR:
s = "CURLM_INTERNAL_ERROR";
break;
case CURLM_UNKNOWN_OPTION:
s = "CURLM_UNKNOWN_OPTION";
break;
case CURLM_LAST:
s = "CURLM_LAST";
break;
default:
s = "CURLM_unknown";
break;
case CURLM_BAD_SOCKET:
s = "CURLM_BAD_SOCKET";
break;
}
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mcode_test ERROR: %s returns %s:%d\n", where, s, code);
return -1;
}
return 0 ;
}
static void remsock(int *f, GlobalInfo_t *g) {
if(f) {
free(f);
f = NULL;
}
}
/* Called by asio when there is an action on a socket */
static void event_cb(GlobalInfo_t *g, curl_socket_t s, int action, const boost::system::error_code & error, int *fdp) {
int f = *fdp;
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X has action %d\n", s, action) ;
// Socket already POOL REMOVED.
if (f == CURL_POLL_REMOVE) {
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X removed\n", s);
remsock(fdp, g);
return;
}
if(socket_map.find(s) == socket_map.end()) {
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb: socket %#X already closed\n, s");
return;
}
/* make sure the event matches what are wanted */
if(f == action || f == CURL_POLL_INOUT) {
if(error) {
action = CURL_CSELECT_ERR;
}
CURLMcode rc = curl_multi_socket_action(g->multi, s, action, &g->still_running);
mcode_test("event_cb: curl_multi_socket_action", rc);
check_multi_info(g);
if(g->still_running <= 0) {
timer.cancel();
}
/* keep on watching.
* the socket may have been closed and/or fdp may have been changed
* in curl_multi_socket_action(), so check them both */
if(!error && socket_map.find(s) != socket_map.end() &&
(f == action || f == CURL_POLL_INOUT)) {
boost::asio::ip::tcp::socket *tcp_socket = socket_map.find(s)->second;
if(action == CURL_POLL_IN) {
tcp_socket->async_read_some(boost::asio::null_buffers(),
boost::bind(&event_cb, g, s,
action, boost::placeholders::_1, fdp));
}
if(action == CURL_POLL_OUT) {
tcp_socket->async_write_some(boost::asio::null_buffers(),
boost::bind(&event_cb, g, s,
action, boost::placeholders::_1, fdp));
}
}
}
}
/* socket functions */
static void setsock(int *fdp, curl_socket_t s, CURL *e, int act, int oldact, GlobalInfo_t *g) {
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(s);
if(it == socket_map.end()) {
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setsock: socket %#X not found\n, s");
return;
}
boost::asio::ip::tcp::socket * tcp_socket = it->second;
*fdp = act;
if(act == CURL_POLL_IN) {
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
tcp_socket->async_read_some(boost::asio::null_buffers(),
boost::bind(&event_cb, g, s,
CURL_POLL_IN, boost::placeholders::_1, fdp));
}
}
else if(act == CURL_POLL_OUT) {
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
tcp_socket->async_write_some(boost::asio::null_buffers(),
boost::bind(&event_cb, g, s,
CURL_POLL_OUT, boost::placeholders::_1, fdp));
}
}
else if(act == CURL_POLL_INOUT) {
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
tcp_socket->async_read_some(boost::asio::null_buffers(),
boost::bind(&event_cb, g, s,
CURL_POLL_IN, boost::placeholders::_1, fdp));
}
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
tcp_socket->async_write_some(boost::asio::null_buffers(),
boost::bind(&event_cb, g, s,
CURL_POLL_OUT, boost::placeholders::_1, fdp));
}
}
}
static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo_t *g) {
/* fdp is used to store current action */
int *fdp = (int *) calloc(sizeof(int), 1);
setsock(fdp, s, easy, action, 0, g);
curl_multi_assign(g->multi, s, fdp);
}
static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) {
GlobalInfo_t *g = &global;
int *actionp = (int *) sockp;
static const char *whatstr[] = { "none", "IN", "OUT", "INOUT", "REMOVE"};
if(what == CURL_POLL_REMOVE) {
*actionp = what;
}
else {
if(!actionp) {
addsock(s, e, what, g);
}
else {
setsock(actionp, s, e, what, *actionp, g);
}
}
return 0;
}
static void threadFunc() {
/* to make sure the event loop doesn't terminate when there is no work to do */
io_service.reset() ;
boost::asio::io_service::work work(io_service);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_deepgram_tts threadFunc - starting\n");
for(;;) {
try {
io_service.run() ;
break ;
}
catch( std::exception& e) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mod_deepgram_tts threadFunc - Error: %s\n", e.what());
}
}
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_deepgram_tts threadFunc - ending\n");
}
/* Called by asio when our timeout expires */
static void timer_cb(const boost::system::error_code & error, GlobalInfo_t *g)
{
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "timer_cb\n");
if(!error) {
CURLMcode rc = curl_multi_socket_action(g->multi, CURL_SOCKET_TIMEOUT, 0, &g->still_running);
mcode_test("timer_cb: curl_multi_socket_action", rc);
check_multi_info(g);
}
}
int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo_t *g) {
/* cancel running timer */
timer.cancel();
if(timeout_ms >= 0) {
// from libcurl 7.88.1-10+deb12u4 does not allow call curl_multi_socket_action or curl_multi_perform in curl_multi callback directly
timer.expires_from_now(boost::posix_time::millisec(timeout_ms ? timeout_ms : 1));
timer.async_wait(boost::bind(&timer_cb, boost::placeholders::_1, g));
}
return 0;
}
/* CURLOPT_WRITEFUNCTION */
static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
bool fireEvent = false;
uint8_t *data = (uint8_t *) ptr;
size_t bytes_received = size * nmemb;
size_t total_bytes_to_process;
auto d = conn->deepgram;
CircularBuffer_t *cBuffer = (CircularBuffer_t *) d->circularBuffer;
if (conn->flushed) {
/* this will abort the transfer */
return 0;
}
// Buffer to hold combined data if there is unprocessed byte from the last call.
std::unique_ptr<uint8_t[]> combinedData;
if (conn->has_last_byte) {
conn->has_last_byte = false; // We'll handle the last_byte now, so toggle the flag off
// Allocate memory for the new data array
combinedData.reset(new uint8_t[bytes_received + 1]);
// Prepend the last byte from previous call
combinedData[0] = conn->last_byte;
// Copy the new data following the prepended byte
memcpy(combinedData.get() + 1, data, bytes_received);
// Point our data pointer to the new array
data = combinedData.get();
total_bytes_to_process = bytes_received + 1;
} else {
total_bytes_to_process = bytes_received;
}
// If we now have an odd total, save the last byte for next time
if ((total_bytes_to_process % sizeof(int16_t)) != 0) {
conn->last_byte = data[total_bytes_to_process - 1];
conn->has_last_byte = true;
total_bytes_to_process--;
}
int16_t* inputData = reinterpret_cast<int16_t*>(data);
if (0 == d->reads++) {
fireEvent = true;
// Deepgram return PCM linear16 WAV file which contains 44 bytes headers, remove that.
inputData += 22;
total_bytes_to_process -= 44;
}
size_t numSamples = total_bytes_to_process / sizeof(int16_t);
{
switch_mutex_lock(d->mutex);
if (d->response_code > 0 && d->response_code != 200) {
std::string body((char *) ptr, bytes_received);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: received body %s\n", body.c_str());
d->err_msg = strdup(body.c_str());
switch_mutex_unlock(d->mutex);
return 0;
}
/* cache file will stay in the mp3 format for size (smaller) and simplicity */
if (conn->file) fwrite(inputData, sizeof(int16_t), numSamples, conn->file);
// Resize the buffer if necessary
if (cBuffer->capacity() - cBuffer->size() < numSamples) {
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "write_cb growing buffer\n");
//TODO: if buffer exceeds some max size, return CURL_WRITEFUNC_ERROR to abort the transfer
cBuffer->set_capacity(cBuffer->size() + std::max(numSamples, (size_t)BUFFER_GROW_SIZE));
}
/* Push the data into the buffer */
cBuffer->insert(cBuffer->end(), inputData, inputData + numSamples);
switch_mutex_unlock(d->mutex);
}
if (fireEvent && d->session_id) {
auto endTime = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(endTime - conn->startTime);
auto time_to_first_byte_ms = std::to_string(duration.count());
switch_core_session_t* session = switch_core_session_locate(d->session_id);
if (session) {
switch_channel_t *channel = switch_core_session_get_channel(session);
switch_core_session_rwunlock(session);
if (channel) {
switch_event_t *event;
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_START) == SWITCH_STATUS_SUCCESS) {
switch_channel_event_set_data(channel, event);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: firing playback-started\n");
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
if (d->reported_latency) {
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_reported_latency_ms", d->reported_latency);
}
if (d->request_id) {
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_request_id", d->request_id);
}
if (d->name_lookup_time_ms) {
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_name_lookup_time_ms", d->name_lookup_time_ms);
}
if (d->connect_time_ms) {
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_connect_time_ms", d->connect_time_ms);
}
if (d->final_response_time_ms) {
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_final_response_time_ms", d->final_response_time_ms);
}
if (d->voice_name) {
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_voice_name", d->voice_name);
}
if (d->cache_filename) {
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", d->cache_filename);
}
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
switch_event_fire(&event);
}
else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
}
}
else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
}
}
else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: session %s not found\n", d->session_id);
}
}
return size*nmemb;
}
static bool parseHeader(const std::string& str, std::string& header, std::string& value) {
std::vector<std::string> parts;
boost::split(parts, str, boost::is_any_of(":"), boost::token_compress_on);
if (parts.size() != 2)
return false;
header = boost::trim_copy(parts[0]);
value = boost::trim_copy(parts[1]);
return true;
}
static size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo_t *conn) {
size_t bytes_received = size * nitems;
const std::string prefix = "HTTP/2 ";
deepgram_t* d = conn->deepgram;
std::string header, value;
std::string input(buffer, bytes_received);
if (parseHeader(input, header, value)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s with value %s\n", header.c_str(), value.c_str());
if (0 == header.compare("dg-request-id")) d->request_id = strdup(value.c_str());
}
else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s\n", input.c_str());
if (input.rfind(prefix, 0) == 0) {
try {
d->response_code = std::stoi(input.substr(prefix.length()));
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "parsed response code: %ld\n", d->response_code);
} catch (const std::invalid_argument& e) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "header_callback: invalid response code %s\n", input.substr(prefix.length()).c_str());
}
}
}
return bytes_received;
}
/* CURLOPT_OPENSOCKETFUNCTION */
static curl_socket_t opensocket(void *clientp, curlsocktype purpose, struct curl_sockaddr *address) {
curl_socket_t sockfd = CURL_SOCKET_BAD;
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "opensocket: %d\n", purpose);
/* restrict to IPv4 */
if(purpose == CURLSOCKTYPE_IPCXN && address->family == AF_INET) {
/* create a tcp socket object */
boost::asio::ip::tcp::socket *tcp_socket = new boost::asio::ip::tcp::socket(io_service);
/* open it and get the native handle*/
boost::system::error_code ec;
tcp_socket->open(boost::asio::ip::tcp::v4(), ec);
if(ec) {
/* An error occurred */
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't open socket [%ld][%s]\n", ec, ec.message().c_str());
}
else {
sockfd = tcp_socket->native_handle();
/* save it for monitoring */
socket_map.insert(std::pair<curl_socket_t, boost::asio::ip::tcp::socket *>(sockfd, tcp_socket));
}
}
return sockfd;
}
/* CURLOPT_CLOSESOCKETFUNCTION */
static int close_socket(void *clientp, curl_socket_t item) {
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "close_socket : %#X\n", item);
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(item);
if(it != socket_map.end()) {
delete it->second;
socket_map.erase(it);
}
return 0;
}
extern "C" {
switch_status_t deepgram_speech_load() {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_loading..\n");
memset(&global, 0, sizeof(GlobalInfo_t));
global.multi = curl_multi_init();
if (!global.multi) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "deepgram_speech_load curl_multi_init() failed, exiting!\n");
return SWITCH_STATUS_FALSE;
}
curl_multi_setopt(global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
curl_multi_setopt(global.multi, CURLMOPT_SOCKETDATA, &global);
curl_multi_setopt(global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
curl_multi_setopt(global.multi, CURLMOPT_TIMERDATA, &global);
curl_multi_setopt(global.multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
/* create temp folder for cache files */
const char* baseDir = std::getenv("JAMBONZ_TMP_CACHE_FOLDER");
if (!baseDir) {
baseDir = "/var/";
}
if (strcmp(baseDir, "/") == 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", baseDir);
return SWITCH_STATUS_FALSE;
}
fullDirPath = std::string(baseDir) + "jambonz-tts-cache-files";
// Create the directory with read, write, and execute permissions for everyone
mode_t oldMask = umask(0);
int result = mkdir(fullDirPath.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
umask(oldMask);
if (result != 0) {
if (errno != EEXIST) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", fullDirPath.c_str());
fullDirPath = "";
}
else switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "folder %s already exists\n", fullDirPath.c_str());
}
else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "created folder %s\n", fullDirPath.c_str());
}
/* start worker thread that handles transfers*/
std::thread t(threadFunc) ;
worker_thread.swap( t ) ;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_loaded..\n");
return SWITCH_STATUS_SUCCESS;
}
switch_status_t deepgram_speech_unload() {
/* stop the ASIO IO service */
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deepgram_speech_unload: stopping io service\n");
io_service.stop();
/* Join the worker thread */
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deepgram_speech_unload: wait for worker thread to complete\n");
if (worker_thread.joinable()) {
worker_thread.join();
}
/* cleanup curl multi handle*/
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deepgram_speech_unload: release curl multi\n");
curl_multi_cleanup(global.multi);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deepgram_speech_unload: completed\n");
return SWITCH_STATUS_SUCCESS;
}
switch_status_t deepgram_speech_open(deepgram_t* deepgram) {
return SWITCH_STATUS_SUCCESS;
}
switch_status_t deepgram_speech_feed_tts(deepgram_t* d, char* text, switch_speech_flag_t *flags) {
CURLMcode rc;
const int MAX_CHARS = 20;
char tempText[MAX_CHARS + 4]; // +4 for the ellipsis and null terminator
if (strlen(text) > MAX_CHARS) {
strncpy(tempText, text, MAX_CHARS);
strcpy(tempText + MAX_CHARS, "...");
} else {
strcpy(tempText, text);
}
/* open cache file */
if (d->cache_audio && fullDirPath.length() > 0) {
switch_uuid_t uuid;
char uuid_str[SWITCH_UUID_FORMATTED_LENGTH + 1];
char outfile[512] = "";
int fd;
switch_uuid_get(&uuid);
switch_uuid_format(uuid_str, &uuid);
switch_snprintf(outfile, sizeof(outfile), "%s%s%s.r8", fullDirPath.c_str(), SWITCH_PATH_SEPARATOR, uuid_str);
d->cache_filename = strdup(outfile);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "writing audio cache file to %s\n", d->cache_filename);
mode_t oldMask = umask(0);
fd = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
umask(oldMask);
if (fd == -1 ) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
}
else {
d->file = fdopen(fd, "wb");
if (!d->file) {
close(fd);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
}
}
}
if (!d->api_key) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "deepgram_speech_feed_tts: no api_key provided\n");
return SWITCH_STATUS_FALSE;
}
/* format url*/
std::string url;
std::ostringstream url_stream;
// always use sample_rate=8000 for support jambonz caching system.
url_stream << "https://api.deepgram.com/v1/speak?model=" << d->voice_name << "&encoding=linear16&sample_rate=8000";
url = url_stream.str();
/* create the JSON body */
cJSON * jResult = cJSON_CreateObject();
cJSON_AddStringToObject(jResult, "text", text);
char *json = cJSON_PrintUnformatted(jResult);
cJSON_Delete(jResult);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_feed_tts: [%s] [%s]\n", url.c_str(), tempText);
ConnInfo_t *conn = pool.malloc() ;
CURL* easy = createEasyHandle();
d->conn = (void *) conn ;
conn->deepgram = d;
conn->easy = easy;
conn->global = &global;
conn->hdr_list = NULL ;
conn->file = d->file;
conn->body = json;
conn->flushed = false;
d->circularBuffer = (void *) new CircularBuffer_t(BUFFER_GROW_SIZE);
// Always use deepgram at rate 8000 for helping cache audio from jambonz.
if (d->rate != 8000) {
int err;
d->resampler = speex_resampler_init(1, 8000, d->rate, SWITCH_RESAMPLE_QUALITY, &err);
if (0 != err) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing resampler: %s.\n", speex_resampler_strerror(err));
return SWITCH_STATUS_FALSE;
}
}
std::ostringstream api_key_stream;
api_key_stream << "Authorization: Token " << d->api_key;
curl_easy_setopt(easy, CURLOPT_URL, url.c_str());
curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, write_cb);
curl_easy_setopt(easy, CURLOPT_WRITEDATA, conn);
curl_easy_setopt(easy, CURLOPT_ERRORBUFFER, conn->error);
curl_easy_setopt(easy, CURLOPT_PRIVATE, conn);
curl_easy_setopt(easy, CURLOPT_VERBOSE, 0L);
curl_easy_setopt(easy, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt(easy, CURLOPT_HEADERFUNCTION, header_callback);
curl_easy_setopt(easy, CURLOPT_HEADERDATA, conn);
/* call this function to get a socket */
curl_easy_setopt(easy, CURLOPT_OPENSOCKETFUNCTION, opensocket);
/* call this function to close a socket */
curl_easy_setopt(easy, CURLOPT_CLOSESOCKETFUNCTION, close_socket);
conn->hdr_list = curl_slist_append(conn->hdr_list, api_key_stream.str().c_str());
conn->hdr_list = curl_slist_append(conn->hdr_list, "Content-Type: application/json");
curl_easy_setopt(easy, CURLOPT_HTTPHEADER, conn->hdr_list);
curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
//curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
rc = curl_multi_add_handle(global.multi, conn->easy);
mcode_test("new_conn: curl_multi_add_handle", rc);
/* start a timer to measure the duration until we receive first byte of audio */
conn->startTime = std::chrono::high_resolution_clock::now();
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "deepgram_speech_feed_tts: called curl_multi_add_handle\n");
return SWITCH_STATUS_SUCCESS;
}
switch_status_t deepgram_speech_read_tts(deepgram_t* d, void *data, size_t *datalen, switch_speech_flag_t *flags) {
CircularBuffer_t *cBuffer = (CircularBuffer_t *) d->circularBuffer;
std::vector<uint16_t> pcm_data;
{
switch_mutex_lock(d->mutex);
ConnInfo_t *conn = (ConnInfo_t *) d->conn;
if (d->response_code > 0 && d->response_code != 200) {
switch_mutex_unlock(d->mutex);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_read_tts, returning failure\n") ;
return SWITCH_STATUS_FALSE;
}
if (conn && conn->flushed) {
switch_mutex_unlock(d->mutex);
return SWITCH_STATUS_BREAK;
}
if (cBuffer->empty()) {
if (d->draining) {
switch_mutex_unlock(d->mutex);
return SWITCH_STATUS_BREAK;
}
/* no audio available yet so send silence */
memset(data, 255, *datalen);
switch_mutex_unlock(d->mutex);
return SWITCH_STATUS_SUCCESS;
}
size_t size = std::min((*datalen/(2 * d->rate / 8000)), cBuffer->size());
pcm_data.insert(pcm_data.end(), cBuffer->begin(), cBuffer->begin() + size);
cBuffer->erase(cBuffer->begin(), cBuffer->begin() + size);
switch_mutex_unlock(d->mutex);
}
size_t data_size = pcm_data.size();
if (d->resampler) {
std::vector<int16_t> in(pcm_data.begin(), pcm_data.end());
std::vector<int16_t> out((*datalen));
spx_uint32_t in_len = data_size;
spx_uint32_t out_len = out.size();
speex_resampler_process_interleaved_int(d->resampler, in.data(), &in_len, out.data(), &out_len);
if (out_len > out.size()) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Resampler output exceeded maximum buffer size!\n");
return SWITCH_STATUS_FALSE;
}
memcpy(data, out.data(), out_len * sizeof(int16_t));
*datalen = out_len * sizeof(int16_t);
} else {
memcpy(data, pcm_data.data(), pcm_data.size() * sizeof(uint16_t));
*datalen = pcm_data.size() * sizeof(uint16_t);
}
return SWITCH_STATUS_SUCCESS;
}
switch_status_t deepgram_speech_flush_tts(deepgram_t* d) {
bool download_complete = d->response_code == 200;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;
ConnInfo_t *conn = (ConnInfo_t *) d->conn;
CircularBuffer_t *cBuffer = (CircularBuffer_t *) d->circularBuffer;
delete cBuffer;
d->circularBuffer = nullptr ;
// destroy resampler
if (d->resampler) {
speex_resampler_destroy(d->resampler);
d->resampler = NULL;
}
if (conn) {
conn->flushed = true;
if (!download_complete) {
if (conn->file) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "closing audio cache file %s because download was interrupted\n", d->cache_filename);
if (fclose(conn->file) != 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "error closing audio cache file\n");
}
conn->file = nullptr ;
}
if (d->cache_filename) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", d->cache_filename);
if (unlink(d->cache_filename) != 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n",
d->cache_filename, errno, strerror(errno));
}
free(d->cache_filename);
d->cache_filename = nullptr ;
}
}
}
if (d->session_id) {
switch_core_session_t* session = switch_core_session_locate(d->session_id);
if (session) {
switch_channel_t *channel = switch_core_session_get_channel(session);
switch_core_session_rwunlock(session);
if (channel) {
switch_event_t *event;
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_STOP) == SWITCH_STATUS_SUCCESS) {
switch_channel_event_set_data(channel, event);
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_response_code", std::to_string(d->response_code).c_str());
if (d->cache_filename && d->response_code == 200) {
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", d->cache_filename);
}
if (d->response_code != 200 && d->err_msg) {
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_error", d->err_msg);
}
switch_event_fire(&event);
}
else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
}
}
else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
}
}
}
return SWITCH_STATUS_SUCCESS;
}
switch_status_t deepgram_speech_close(deepgram_t* w) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_close\n") ;
return SWITCH_STATUS_SUCCESS;
}
}
+12
View File
@@ -0,0 +1,12 @@
#ifndef __DEEPGRAM_GLUE_H__
#define __DEEPGRAM_GLUE_H__
switch_status_t deepgram_speech_load();
switch_status_t deepgram_speech_open(deepgram_t* deepgram);
switch_status_t deepgram_speech_feed_tts(deepgram_t* deepgram, char* text, switch_speech_flag_t *flags);
switch_status_t deepgram_speech_read_tts(deepgram_t* deepgram, void *data, size_t *datalen, switch_speech_flag_t *flags);
switch_status_t deepgram_speech_flush_tts(deepgram_t* deepgram);
switch_status_t deepgram_speech_close(deepgram_t* deepgram);
switch_status_t deepgram_speech_unload();
#endif
+156
View File
@@ -0,0 +1,156 @@
#include "mod_deepgram_tts.h"
#include "deepgram_glue.h"
SWITCH_MODULE_LOAD_FUNCTION(mod_deepgram_tts_load);
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_deepgram_tts_shutdown);
SWITCH_MODULE_DEFINITION(mod_deepgram_tts, mod_deepgram_tts_load, mod_deepgram_tts_shutdown, NULL);
static void cleardeepgram(deepgram_t* d, int freeAll) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "cleardeepgram\n");
if (d->api_key) free(d->api_key);
if (d->request_id) free(d->request_id);
if (d->reported_latency) free(d->reported_latency);
if (d->ct) free(d->ct);
if (d->err_msg) free(d->err_msg);
if (d->name_lookup_time_ms) free(d->name_lookup_time_ms);
if (d->connect_time_ms) free(d->connect_time_ms);
if (d->final_response_time_ms) free(d->final_response_time_ms);
if (d->cache_filename) free(d->cache_filename);
d->api_key = NULL;
d->request_id = NULL;
d->reported_latency = NULL;
d->ct = NULL;
d->err_msg = NULL;
d->name_lookup_time_ms = NULL;
d->connect_time_ms = NULL;
d->final_response_time_ms = NULL;
d->cache_filename = NULL;
if (freeAll) {
if (d->voice_name) free(d->voice_name);
if (d->session_id) free(d->session_id);
d->voice_name = NULL;
d->session_id = NULL;
}
}
static deepgram_t * createOrRetrievePrivateData(switch_speech_handle_t *sh) {
deepgram_t *d = (deepgram_t *) sh->private_info;
if (!d) {
d = switch_core_alloc(sh->memory_pool, sizeof(*d));
sh->private_info = d;
memset(d, 0, sizeof(*d));
switch_mutex_init(&d->mutex, SWITCH_MUTEX_NESTED, sh->memory_pool);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "allocated deepgram_t\n");
}
return d;
}
switch_status_t d_speech_open(switch_speech_handle_t *sh, const char *voice_name, int rate, int channels, switch_speech_flag_t *flags)
{
deepgram_t *d = createOrRetrievePrivateData(sh);
d->voice_name = strdup(voice_name);
d->rate = rate;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_open voice: %s, rate %d, channels %d\n", voice_name, rate, channels);
return deepgram_speech_open(d);
}
static switch_status_t d_speech_close(switch_speech_handle_t *sh, switch_speech_flag_t *flags)
{
switch_status_t rc;
deepgram_t *d = createOrRetrievePrivateData(sh);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_close\n");
switch_mutex_destroy(d->mutex);
rc = deepgram_speech_close(d);
cleardeepgram(d, 1);
return rc;
}
/**
* Freeswitch will call this function to feed us text to speak
*/
static switch_status_t d_speech_feed_tts(switch_speech_handle_t *sh, char *text, switch_speech_flag_t *flags)
{
deepgram_t *d = createOrRetrievePrivateData(sh);
d->draining = 0;
d->reads = 0;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_feed_tts\n");
return deepgram_speech_feed_tts(d, text, flags);
}
/**
* Freeswitch calls periodically to get some rendered audio in L16 format. We can provide up to 8k of audio at a time.
*/
static switch_status_t d_speech_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, switch_speech_flag_t *flags)
{
deepgram_t *d = createOrRetrievePrivateData(sh);
return deepgram_speech_read_tts(d, data, datalen, flags);
}
/**
* This is called at the end, not sure exactly what we need to do here..
*/
static void d_speech_flush_tts(switch_speech_handle_t *sh)
{
deepgram_t *d = createOrRetrievePrivateData(sh);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_flush_tts\n");
deepgram_speech_flush_tts(d);
cleardeepgram(d, 0);
}
static void d_text_param_tts(switch_speech_handle_t *sh, char *param, const char *val)
{
deepgram_t *d = createOrRetrievePrivateData(sh);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_text_param_tts: %s=%s\n", param, val);
if (0 == strcmp(param, "api_key")) {
if (d->api_key) free(d->api_key);
d->api_key = strdup(val);
} else if (0 == strcmp(param, "voice")) {
if (d->voice_name) free(d->voice_name);
d->voice_name = strdup(val);
} else if (0 == strcmp(param, "session-uuid")) {
if (d->session_id) free(d->session_id);
d->session_id = strdup(val);
} else if (0 == strcmp(param, "write_cache_file") && switch_true(val)) {
d->cache_audio = 1;
}
}
static void d_numeric_param_tts(switch_speech_handle_t *sh, char *param, int val)
{
}
static void d_float_param_tts(switch_speech_handle_t *sh, char *param, double val)
{
}
SWITCH_MODULE_LOAD_FUNCTION(mod_deepgram_tts_load)
{
switch_speech_interface_t *speech_interface;
*module_interface = switch_loadable_module_create_module_interface(pool, modname);
speech_interface = switch_loadable_module_create_interface(*module_interface, SWITCH_SPEECH_INTERFACE);
speech_interface->interface_name = "deepgram";
speech_interface->speech_open = d_speech_open;
speech_interface->speech_close = d_speech_close;
speech_interface->speech_feed_tts = d_speech_feed_tts;
speech_interface->speech_read_tts = d_speech_read_tts;
speech_interface->speech_flush_tts = d_speech_flush_tts;
speech_interface->speech_text_param_tts = d_text_param_tts;
speech_interface->speech_numeric_param_tts = d_numeric_param_tts;
speech_interface->speech_float_param_tts = d_float_param_tts;
return deepgram_speech_load();
}
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_deepgram_tts_shutdown)
{
return deepgram_speech_unload();
}
+35
View File
@@ -0,0 +1,35 @@
#ifndef __MOD_DEEPGRAM_TTS_H__
#define __MOD_DEEPGRAM_TTS_H__
#include <switch.h>
#include <speex/speex_resampler.h>
typedef struct deepgram_data {
char *voice_name;
char *api_key;
/* result data */
long response_code;
char *ct;
char *reported_latency;
char *request_id;
char *name_lookup_time_ms;
char *connect_time_ms;
char *final_response_time_ms;
char *err_msg;
char *cache_filename;
char *session_id;
int rate;
int draining;
int reads;
int cache_audio;
void *conn;
void *circularBuffer;
switch_mutex_t *mutex;
FILE *file;
SpeexResamplerState *resampler;
} deepgram_t;
#endif
+1 -1
View File
@@ -15,4 +15,4 @@ endif
endif
mod_dub_la_LIBADD = $(switch_builddir)/libfreeswitch.la
mod_dub_la_LDFLAGS = -avoid-version -module -no-undefined -shared `pkg-config --libs boost` -lstdc++ -lmpg123
mod_dub_la_LDFLAGS = -avoid-version -module -no-undefined -lstdc++ -lmpg123 -lboost_system -lboost_thread
+4 -1
View File
@@ -142,6 +142,8 @@ void AudioProducerHttp::start(std::function<void(bool, const std::string&)> call
curl_easy_setopt(_easy, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)31415);
/*Add request body*/
if (!_body.empty()) curl_easy_setopt(_easy, CURLOPT_POSTFIELDS, _body.c_str());
/*Add request proxy*/
if (!_proxy.empty()) curl_easy_setopt(_easy, CURLOPT_PROXY, _proxy.c_str());
/*Add request headers*/
struct curl_slist *hdr_list = nullptr;
@@ -167,11 +169,12 @@ void AudioProducerHttp::queueHttpPostAudio(const std::string& url, int gain, boo
_gain = gain;
_loop = loop;
}
void AudioProducerHttp::queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, int gain, bool loop) {
void AudioProducerHttp::queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, const std::string& proxy, int gain, bool loop) {
_method = HttpMethod_t::HTTP_METHOD_POST;
_url = url;
_body = body;
_headers = headers;
_proxy = proxy;
_gain = gain;
_loop = loop;
}
+2 -1
View File
@@ -68,7 +68,7 @@ public:
void queueHttpGetAudio(const std::string& url, int gain = 0, bool loop = false);
void queueHttpPostAudio(const std::string& url, int gain = 0, bool loop = false);
void queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, int gain = 0, bool loop = false);
void queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, const std::string& proxy, int gain = 0, bool loop = false);
Status_t getStatus() const { return _status; }
void setStatus(Status_t status) { _status = status; }
@@ -121,6 +121,7 @@ private:
HttpMethod_t _method;
std::string _url;
std::string _body;
std::string _proxy;
std::vector<std::string> _headers;
Status_t _status;
mpg123_handle *_mh;
+3 -3
View File
@@ -98,18 +98,18 @@ extern "C" {
switch_status_t say_dub_track(struct cap_cb* cb, char* trackName, char* text, int gain) {
std::vector<std::string> headers;
std::string url, body;
std::string url, body, proxy;
Track* track = find_track_by_name(cb->tracks, trackName);
if (!track) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "play_dub_track: track %s not found\n", trackName);
return SWITCH_STATUS_FALSE;
}
if (tts_vendor_parse_text(text, url, body, headers) != SWITCH_STATUS_SUCCESS) {
if (tts_vendor_parse_text(text, url, body, headers, proxy) != SWITCH_STATUS_SUCCESS) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "say_dub_track: failed to parse text\n");
return SWITCH_STATUS_FALSE;
}
track->queueHttpPostAudio(url, body, headers, gain);
track->queueHttpPostAudio(url, body, headers, proxy, gain);
return SWITCH_STATUS_SUCCESS;
}
+2 -2
View File
@@ -114,11 +114,11 @@ void Track::queueHttpPostAudio(const std::string& url, int gain, bool loop) {
}
}
void Track::queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, int gain, bool loop) {
void Track::queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, const std::string& proxy, int gain, bool loop) {
bool startIt = false;
if (_stopping) return;
auto ap = std::make_shared<AudioProducerHttp>(_mutex, _buffer, _sampleRate);
ap->queueHttpPostAudio(url, body, headers, gain, loop);
ap->queueHttpPostAudio(url, body, headers, proxy, gain, loop);
{
std::lock_guard<std::mutex> lock(_mutex);
_apQueue.push(ap);
+1 -1
View File
@@ -14,7 +14,7 @@ public:
/* audio production methods */
void queueHttpGetAudio(const std::string& url, int gain = 0, bool loop = false);
void queueHttpPostAudio(const std::string& url, int gain = 0, bool loop = false);
void queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, int gain = 0, bool loop = false);
void queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, const std::string& proxy, int gain = 0, bool loop = false);
void queueFileAudio(const std::string& path, int gain = 0, bool loop = false);
void removeAllAudio();
+188 -2
View File
@@ -4,6 +4,186 @@
#include <switch_json.h>
#include <map>
switch_status_t whisper_parse_text(const std::map<std::string, std::string>& params, const std::string& text,
std::string& url, std::string& body, std::vector<std::string>& headers) {
std::string api_key;
std::string voice_name;
std::string model_id;
std::string speed;
for (const auto& pair : params) {
if (pair.first == "api_key") {
api_key = pair.second;
} else if (pair.first == "voice") {
voice_name = pair.second;
} else if (pair.first == "model_id") {
model_id = pair.second;
} else if (pair.first == "speed") {
speed = pair.second;
}
}
if (api_key.empty()) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "whisper_parse_text: no api_key provided\n");
return SWITCH_STATUS_FALSE;
}
if (model_id.empty()) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "whisper_parse_text: no model_id provided\n");
return SWITCH_STATUS_FALSE;
}
url = "https://api.openai.com/v1/audio/speech";
/* create the JSON body */
cJSON * jResult = cJSON_CreateObject();
cJSON_AddStringToObject(jResult, "model", model_id.c_str());
cJSON_AddStringToObject(jResult, "input", text.c_str());
cJSON_AddStringToObject(jResult, "voice", voice_name.c_str());
cJSON_AddStringToObject(jResult, "response_format", "mp3");
if (!speed.empty()) {
cJSON_AddStringToObject(jResult, "speed", speed.c_str());
}
char* _body = cJSON_PrintUnformatted(jResult);
body = _body;
cJSON_Delete(jResult);
free(_body);
// Create headers
headers.push_back("Authorization: Bearer " + api_key);
headers.push_back("Content-Type: application/json");
return SWITCH_STATUS_SUCCESS;
}
switch_status_t azure_parse_text(const std::map<std::string, std::string>& params, const std::string& text,
std::string& url, std::string& body, std::vector<std::string>& headers, std::string& proxy) {
std::string api_key;
std::string voice_name;
std::string language;
std::string region;
std::string endpoint;
std::string endpointId;
std::string http_proxy_ip;
std::string http_proxy_port;
for (const auto& pair : params) {
if (pair.first == "api_key") {
api_key = pair.second;
} else if (pair.first == "voice") {
voice_name = pair.second;
} else if (pair.first == "language") {
language = pair.second;
} else if (pair.first == "region") {
region = pair.second;
} else if (pair.first == "endpoint") {
endpoint = pair.second;
} else if (pair.first == "endpointId") {
endpointId = pair.second;
} else if (pair.first == "http_proxy_ip") {
http_proxy_ip = pair.second;
} else if (pair.first == "http_proxy_port") {
http_proxy_port = pair.second;
}
}
if (language.empty()) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "azure_parse_text: no language provided\n");
return SWITCH_STATUS_FALSE;
}
if (voice_name.empty()) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "azure_parse_text: no voice_name provided\n");
return SWITCH_STATUS_FALSE;
}
if (region.empty()) {
region = "westus";
}
/* format url*/
url = !endpoint.empty() ? endpoint : "https://" + region + ".tts.speech.microsoft.com/cognitiveservices/v1";
// Body
if (strncmp(text.c_str(), "<speak", 6) == 0) {
body = text;
} else {
std::ostringstream body_stream;
body_stream << "<speak version=\"1.0\" xmlns=\"http://www.w3.org/2001/10/synthesis\" xmlns:mstts=\"https://www.w3.org/2001/mstts\" xml:lang=\"" << language << "\">";
body_stream << "<voice name=\"" << voice_name << "\">";
body_stream << text;
body_stream << "</voice>";
body_stream << "</speak>";
body = body_stream.str();
}
// Create headers
if (!api_key.empty()) {
headers.push_back("Ocp-Apim-Subscription-Key: " + api_key);
}
if (!endpointId.empty()) {
headers.push_back("X-Microsoft-EndpointId: " + endpointId);
}
headers.push_back("Content-Type: application/ssml+xml");
headers.push_back("X-Microsoft-OutputFormat: audio-16khz-32kbitrate-mono-mp3");
// Proxy
std::ostringstream proxy_stream;
if (!http_proxy_ip.empty()) {
proxy_stream << "http://" << http_proxy_ip;
if (!http_proxy_port.empty()) {
proxy_stream << ":" << http_proxy_port;
}
}
proxy = proxy_stream.str();
return SWITCH_STATUS_SUCCESS;
}
switch_status_t deepgram_parse_text(const std::map<std::string, std::string>& params, const std::string& text,
std::string& url, std::string& body, std::vector<std::string>& headers) {
std::string api_key;
std::string voice_name;
for (const auto& pair : params) {
if (pair.first == "api_key") {
api_key = pair.second;
} else if (pair.first == "voice") {
voice_name = pair.second;
}
}
if (api_key.empty()) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "deepgram_parse_text: no api_key provided\n");
return SWITCH_STATUS_FALSE;
}
if (voice_name.empty()) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "deepgram_parse_text: no voice_name provided\n");
return SWITCH_STATUS_FALSE;
}
/* format url*/
std::ostringstream url_stream;
url_stream << "https://api.deepgram.com/v1/speak?model=" << voice_name << "&encoding=mp3";
url = url_stream.str();
/* create the JSON body */
cJSON * jResult = cJSON_CreateObject();
cJSON_AddStringToObject(jResult, "text", text.c_str());
char* _body = cJSON_PrintUnformatted(jResult);
body = _body;
cJSON_Delete(jResult);
free(_body);
// Create headers
headers.push_back("Authorization: Token " + api_key);
headers.push_back("Content-Type: application/json");
return SWITCH_STATUS_SUCCESS;
}
switch_status_t elevenlabs_parse_text(const std::map<std::string, std::string>& params, const std::string& text,
std::string& url, std::string& body, std::vector<std::string>& headers) {
@@ -87,7 +267,7 @@ switch_status_t elevenlabs_parse_text(const std::map<std::string, std::string>&
return SWITCH_STATUS_SUCCESS;
}
switch_status_t tts_vendor_parse_text(const std::string& say, std::string& url, std::string& body, std::vector<std::string>& headers) {
switch_status_t tts_vendor_parse_text(const std::string& say, std::string& url, std::string& body, std::vector<std::string>& headers, std::string& proxy) {
size_t start = say.find("{") + 1;
size_t end = say.find("}");
@@ -111,8 +291,14 @@ switch_status_t tts_vendor_parse_text(const std::string& say, std::string& url,
if (params["vendor"] == "elevenlabs") {
return elevenlabs_parse_text(params, text, url, body, headers);
} else if (params["vendor"] == "deepgram") {
return deepgram_parse_text(params, text, url, body, headers);
} else if (params["vendor"] == "microsoft") {
return azure_parse_text(params, text, url, body, headers, proxy);
} else if (params["vendor"] == "whisper") {
return whisper_parse_text(params, text, url, body, headers);
} else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "tts_vendor_parse_text: There is no available parser for text\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "tts_vendor_parse_text: There is no available parser for vendor %s\n", params["vendor"]);
return SWITCH_STATUS_FALSE;
}
}
+1 -1
View File
@@ -7,6 +7,6 @@
#include "common.h"
switch_status_t tts_vendor_parse_text(const std::string& say, std::string& url, std::string& body, std::vector<std::string>& headers);
switch_status_t tts_vendor_parse_text(const std::string& say, std::string& url, std::string& body, std::vector<std::string>& headers, std::string& proxy);
#endif
+3
View File
@@ -1,6 +1,8 @@
#include "vector_math.h"
#include <assert.h>
#include <string.h>
#include <cstdlib>
#define GRANULAR_VOLUME_MAX (50)
#define SMAX 32767
@@ -153,6 +155,7 @@ typedef union {
} vector_data_t;
#else
#pragma message("Building without vector math support")
void vector_add(int16_t* a, int16_t* b, size_t len) {
for (size_t i = 0; i < len; i++) {
a[i] += b[i];
+1 -1
View File
@@ -5,4 +5,4 @@ mod_LTLIBRARIES = mod_whisper_tts.la
mod_whisper_tts_la_SOURCES = mod_whisper_tts.c whisper_glue.cpp
mod_whisper_tts_la_CFLAGS = $(AM_CFLAGS)
mod_whisper_tts_la_LIBADD = $(switch_builddir)/libfreeswitch.la
mod_whisper_tts_la_LDFLAGS = -avoid-version -module -no-undefined -shared `pkg-config --libs boost` -lstdc++ -lmpg123
mod_whisper_tts_la_LDFLAGS = -avoid-version -module -no-undefined -shared -lstdc++ -lboost_system -lboost_thread -lmpg123
+5 -6
View File
@@ -6,7 +6,7 @@ SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_whisper_tts_shutdown);
SWITCH_MODULE_DEFINITION(mod_whisper_tts, mod_whisper_tts_load, mod_whisper_tts_shutdown, NULL);
static void clearWhisper(whisper_t* w, int freeAll) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "clearWhisper\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "clearWhisper\n");
if (w->api_key) free(w->api_key);
if (w->model_id) free(w->model_id);
if (w->speed) free(w->speed);
@@ -65,7 +65,7 @@ static switch_status_t w_speech_close(switch_speech_handle_t *sh, switch_speech_
{
switch_status_t rc;
whisper_t *w = createOrRetrievePrivateData(sh);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_speech_close\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_speech_close\n");
switch_mutex_destroy(w->mutex);
@@ -83,7 +83,7 @@ static switch_status_t w_speech_feed_tts(switch_speech_handle_t *sh, char *text,
w->draining = 0;
w->reads = 0;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_speech_feed_tts\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_speech_feed_tts\n");
return whisper_speech_feed_tts(w, text, flags);
}
@@ -94,7 +94,6 @@ static switch_status_t w_speech_feed_tts(switch_speech_handle_t *sh, char *text,
static switch_status_t w_speech_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, switch_speech_flag_t *flags)
{
whisper_t *w = createOrRetrievePrivateData(sh);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_speech_read_tts\n");
return whisper_speech_read_tts(w, data, datalen, flags);
}
@@ -104,7 +103,7 @@ static switch_status_t w_speech_read_tts(switch_speech_handle_t *sh, void *data,
static void w_speech_flush_tts(switch_speech_handle_t *sh)
{
whisper_t *w = createOrRetrievePrivateData(sh);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_speech_flush_tts\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_speech_flush_tts\n");
whisper_speech_flush_tts(w);
clearWhisper(w, 0);
@@ -113,7 +112,7 @@ static void w_speech_flush_tts(switch_speech_handle_t *sh)
static void w_text_param_tts(switch_speech_handle_t *sh, char *param, const char *val)
{
whisper_t *w = createOrRetrievePrivateData(sh);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_text_param_tts: %s=%s\n", param, val);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_text_param_tts: %s=%s\n", param, val);
if (0 == strcmp(param, "api_key")) {
if (w->api_key) free(w->api_key);
w->api_key = strdup(val);
+5 -7
View File
@@ -442,8 +442,6 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
}
{
switch_mutex_lock(w->mutex);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: received data, response %ld\n",
w->response_code);
if (w->response_code > 0 && w->response_code != 200) {
std::string body((char *) ptr, bytes_received);
@@ -482,6 +480,7 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
switch_core_session_t* session = switch_core_session_locate(w->session_id);
if (session) {
switch_channel_t *channel = switch_core_session_get_channel(session);
switch_core_session_rwunlock(session);
if (channel) {
switch_event_t *event;
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_START) == SWITCH_STATUS_SUCCESS) {
@@ -525,7 +524,6 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
}
switch_core_session_rwunlock(session);
}
else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: session %s not found\n", w->session_id);
@@ -810,6 +808,7 @@ extern "C" {
switch_codec_implementation_t read_impl;
switch_core_session_t *psession = switch_core_session_locate(w->session_id);
switch_core_session_get_read_impl(psession, &read_impl);
switch_core_session_rwunlock(psession);
uint32_t samples_per_second = !strcasecmp(read_impl.iananame, "g722") ? read_impl.actual_samples_per_second : read_impl.samples_per_second;
if (mpg123_param(mh, MPG123_FORCE_RATE, samples_per_second /*Hz*/, 0) != MPG123_OK) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_param!\n");
@@ -897,8 +896,7 @@ extern "C" {
switch_status_t whisper_speech_flush_tts(whisper_t* w) {
bool download_complete = w->response_code == 200;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "whisper_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "whisper_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;
ConnInfo_t *conn = (ConnInfo_t *) w->conn;
CircularBuffer_t *cBuffer = (CircularBuffer_t *) w->circularBuffer;
delete cBuffer;
@@ -930,6 +928,7 @@ extern "C" {
switch_core_session_t* session = switch_core_session_locate(w->session_id);
if (session) {
switch_channel_t *channel = switch_core_session_get_channel(session);
switch_core_session_rwunlock(session);
if (channel) {
switch_event_t *event;
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_STOP) == SWITCH_STATUS_SUCCESS) {
@@ -951,14 +950,13 @@ extern "C" {
else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
}
switch_core_session_rwunlock(session);
}
}
return SWITCH_STATUS_SUCCESS;
}
switch_status_t whisper_speech_close(whisper_t* w) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "whisper_speech_close\n") ;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "whisper_speech_close\n") ;
return SWITCH_STATUS_SUCCESS;
}
}