mirror of
https://github.com/jambonz/freeswitch-modules.git
synced 2026-01-25 02:08:27 +00:00
Compare commits
33 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 33750b5420 | |||
| f8604656e3 | |||
| 916332d91d | |||
| 33fee93ec7 | |||
| 41aebafd1c | |||
| bd69d476e7 | |||
| fea51d5ecf | |||
| 83a2d1d730 | |||
| 3f642467eb | |||
| 622cb14957 | |||
| 1e1a51a3d1 | |||
| 4ce95e6d27 | |||
| f0d15c57a2 | |||
| 2bbc18903c | |||
| 4aabb36a64 | |||
| ee488e4621 | |||
| 751d890d9b | |||
| e431c5d159 | |||
| a96fb2b4b2 | |||
| 94361f1d20 | |||
| d05cfb8ef0 | |||
| 8e7f4a3fab | |||
| fa4930d53e | |||
| 63e66042af | |||
| eedcc03112 | |||
| 07dddd0094 | |||
| 14297ce3a6 | |||
| 48aeedec0a | |||
| be6758c3a8 | |||
| 74bfc3152f | |||
| 9cdc5fdfca | |||
| 4e57f73c7e | |||
| 4925d31f95 |
@@ -2,9 +2,17 @@ include $(top_srcdir)/build/modmake.rulesam
|
||||
MODNAME=mod_audio_fork
|
||||
|
||||
mod_LTLIBRARIES = mod_audio_fork.la
|
||||
mod_audio_fork_la_SOURCES = mod_audio_fork.c lws_glue.cpp parser.cpp audio_pipe.cpp
|
||||
mod_audio_fork_la_SOURCES = mod_audio_fork.c lws_glue.cpp parser.cpp audio_pipe.cpp vector_math.cpp
|
||||
mod_audio_fork_la_CFLAGS = $(AM_CFLAGS)
|
||||
mod_audio_fork_la_CXXFLAGS = $(AM_CXXFLAGS) -std=c++11
|
||||
|
||||
if USE_AVX2
|
||||
mod_audio_fork_la_CXXFLAGS += -mavx2 -DUSE_AVX2
|
||||
else
|
||||
if USE_SSE2
|
||||
mod_audio_fork_la_CXXFLAGS += -msse2 -DUSE_SSE2
|
||||
endif
|
||||
endif
|
||||
|
||||
mod_audio_fork_la_LIBADD = $(switch_builddir)/libfreeswitch.la
|
||||
mod_audio_fork_la_LDFLAGS = -avoid-version -module -no-undefined -shared `pkg-config --libs libwebsockets`
|
||||
mod_audio_fork_la_LDFLAGS = -avoid-version -module -no-undefined -shared `pkg-config --libs libwebsockets` -lstdc++ -lboost_system -lboost_thread
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "audio_pipe.hpp"
|
||||
#include <switch.h>
|
||||
|
||||
#include <cassert>
|
||||
#include <iostream>
|
||||
@@ -83,7 +84,7 @@ int AudioPipe::lws_callback(struct lws *wsi,
|
||||
lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_CONNECTION_ERROR: %s, response status %d\n", in ? (char *)in : "(null)", rc);
|
||||
if (ap) {
|
||||
ap->m_state = LWS_CLIENT_FAILED;
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECT_FAIL, (char *) in);
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECT_FAIL, (char *) in, NULL, len);
|
||||
}
|
||||
else {
|
||||
lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_CONNECTION_ERROR unable to find wsi %p..\n", wsi);
|
||||
@@ -98,7 +99,7 @@ int AudioPipe::lws_callback(struct lws *wsi,
|
||||
*ppAp = ap;
|
||||
ap->m_vhd = vhd;
|
||||
ap->m_state = LWS_CLIENT_CONNECTED;
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECT_SUCCESS, NULL);
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECT_SUCCESS, NULL, NULL, len);
|
||||
}
|
||||
else {
|
||||
lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_ESTABLISHED %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi);
|
||||
@@ -114,12 +115,12 @@ int AudioPipe::lws_callback(struct lws *wsi,
|
||||
}
|
||||
if (ap->m_state == LWS_CLIENT_DISCONNECTING) {
|
||||
// closed by us
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECTION_CLOSED_GRACEFULLY, NULL);
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECTION_CLOSED_GRACEFULLY, NULL, NULL, len);
|
||||
}
|
||||
else if (ap->m_state == LWS_CLIENT_CONNECTED) {
|
||||
// closed by far end
|
||||
lwsl_notice("%s socket closed by far end\n", ap->m_uuid.c_str());
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECTION_DROPPED, NULL);
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::CONNECTION_DROPPED, NULL, NULL, len);
|
||||
}
|
||||
ap->m_state = LWS_CLIENT_DISCONNECTED;
|
||||
|
||||
@@ -133,14 +134,19 @@ int AudioPipe::lws_callback(struct lws *wsi,
|
||||
|
||||
case LWS_CALLBACK_CLIENT_RECEIVE:
|
||||
{
|
||||
|
||||
AudioPipe* ap = *ppAp;
|
||||
if (!ap) {
|
||||
lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE %s unable to find wsi %p..\n", ap->m_uuid.c_str(), wsi);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
if (lws_frame_is_binary(wsi)) {
|
||||
lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE received binary frame, discarding.\n");
|
||||
if (ap->is_bidirectional_audio_stream()) {
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::BINARY, NULL, (char *) in, len);
|
||||
} else {
|
||||
lwsl_err("AudioPipe::lws_service_thread LWS_CALLBACK_CLIENT_RECEIVE received binary frame, discarding.\n");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -180,7 +186,7 @@ int AudioPipe::lws_callback(struct lws *wsi,
|
||||
if (lws_is_final_fragment(wsi)) {
|
||||
if (nullptr != ap->m_recv_buf) {
|
||||
std::string msg((char *)ap->m_recv_buf, ap->m_recv_buf_ptr - ap->m_recv_buf);
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::MESSAGE, msg.c_str());
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), AudioPipe::MESSAGE, msg.c_str(), NULL, len);
|
||||
if (nullptr != ap->m_recv_buf) free(ap->m_recv_buf);
|
||||
}
|
||||
ap->m_recv_buf = ap->m_recv_buf_ptr = nullptr;
|
||||
@@ -458,7 +464,8 @@ bool AudioPipe::deinitialize() {
|
||||
|
||||
// instance members
|
||||
AudioPipe::AudioPipe(const char* uuid, const char* host, unsigned int port, const char* path,
|
||||
int sslFlags, size_t bufLen, size_t minFreespace, const char* username, const char* password, char* bugname, notifyHandler_t callback) :
|
||||
int sslFlags, size_t bufLen, size_t minFreespace, const char* username, const char* password, char* bugname,
|
||||
int bidirectional_audio_stream, notifyHandler_t callback) :
|
||||
m_uuid(uuid), m_host(host), m_port(port), m_path(path), m_sslFlags(sslFlags),
|
||||
m_audio_buffer_min_freespace(minFreespace), m_audio_buffer_max_len(bufLen), m_gracefulShutdown(false),
|
||||
m_audio_buffer_write_offset(LWS_PRE), m_recv_buf(nullptr), m_recv_buf_ptr(nullptr), m_bugname(bugname),
|
||||
@@ -468,12 +475,12 @@ AudioPipe::AudioPipe(const char* uuid, const char* host, unsigned int port, cons
|
||||
m_username.assign(username);
|
||||
m_password.assign(password);
|
||||
}
|
||||
|
||||
m_bidirectional_audio_stream = bidirectional_audio_stream;
|
||||
m_audio_buffer = new uint8_t[m_audio_buffer_max_len];
|
||||
}
|
||||
AudioPipe::~AudioPipe() {
|
||||
if (m_audio_buffer) delete [] m_audio_buffer;
|
||||
if (m_recv_buf) delete [] m_recv_buf;
|
||||
if (m_recv_buf) free(m_recv_buf);
|
||||
}
|
||||
|
||||
void AudioPipe::connect(void) {
|
||||
|
||||
@@ -27,10 +27,11 @@ namespace drachtio {
|
||||
CONNECT_FAIL,
|
||||
CONNECTION_DROPPED,
|
||||
CONNECTION_CLOSED_GRACEFULLY,
|
||||
MESSAGE
|
||||
MESSAGE,
|
||||
BINARY
|
||||
};
|
||||
typedef void (*log_emit_function)(int level, const char *line);
|
||||
typedef void (*notifyHandler_t)(const char *sessionId, const char* bugname, NotifyEvent_t event, const char* message);
|
||||
typedef void (*notifyHandler_t)(const char *sessionId, const char* bugname, NotifyEvent_t event, const char* message, const char* binary, size_t binary_len );
|
||||
|
||||
struct lws_per_vhost_data {
|
||||
struct lws_context *context;
|
||||
@@ -44,7 +45,8 @@ namespace drachtio {
|
||||
|
||||
// constructor
|
||||
AudioPipe(const char* uuid, const char* host, unsigned int port, const char* path, int sslFlags,
|
||||
size_t bufLen, size_t minFreespace, const char* username, const char* password, char* bugname, notifyHandler_t callback);
|
||||
size_t bufLen, size_t minFreespace, const char* username, const char* password, char* bugname,
|
||||
int bidirectional_audio, notifyHandler_t callback);
|
||||
~AudioPipe();
|
||||
|
||||
LwsState_t getLwsState(void) { return m_state; }
|
||||
@@ -83,6 +85,10 @@ namespace drachtio {
|
||||
return m_gracefulShutdown;
|
||||
}
|
||||
|
||||
bool is_bidirectional_audio_stream() {
|
||||
return m_bidirectional_audio_stream;
|
||||
}
|
||||
|
||||
void close() ;
|
||||
|
||||
// no default constructor or copying
|
||||
@@ -142,6 +148,7 @@ namespace drachtio {
|
||||
std::string m_username;
|
||||
std::string m_password;
|
||||
bool m_gracefulShutdown;
|
||||
bool m_bidirectional_audio_stream;
|
||||
};
|
||||
|
||||
} // namespace drachtio
|
||||
|
||||
+152
-18
@@ -17,9 +17,15 @@
|
||||
#include "parser.hpp"
|
||||
#include "mod_audio_fork.h"
|
||||
#include "audio_pipe.hpp"
|
||||
#include "vector_math.h"
|
||||
|
||||
#include <boost/circular_buffer.hpp>
|
||||
|
||||
typedef boost::circular_buffer<uint16_t> CircularBuffer_t;
|
||||
|
||||
#define RTP_PACKETIZATION_PERIOD 20
|
||||
#define FRAME_SIZE_8000 320 /*which means each 20ms frame as 320 bytes at 8 khz (1 channel only)*/
|
||||
#define BUFFER_GROW_SIZE (8192)
|
||||
|
||||
namespace {
|
||||
static const char *requestedBufferSecs = std::getenv("MOD_AUDIO_FORK_BUFFER_SECS");
|
||||
@@ -31,6 +37,48 @@ namespace {
|
||||
static unsigned int idxCallCount = 0;
|
||||
static uint32_t playCount = 0;
|
||||
|
||||
switch_status_t processIncomingBinary(private_t* tech_pvt, switch_core_session_t* session, const char* message, size_t dataLength) {
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->circularBuffer;
|
||||
uint8_t* data = reinterpret_cast<uint8_t*>(const_cast<char*>(message));
|
||||
uint16_t* data_uint16 = reinterpret_cast<uint16_t*>(data);
|
||||
std::vector<uint16_t> pcm_data(data_uint16, data_uint16 + dataLength / sizeof(uint16_t));
|
||||
|
||||
|
||||
if (tech_pvt->bidirectional_audio_resampler) {
|
||||
std::vector<int16_t> in(pcm_data.begin(), pcm_data.end());
|
||||
|
||||
std::vector<int16_t> out(dataLength);
|
||||
spx_uint32_t in_len = pcm_data.size();
|
||||
spx_uint32_t out_len = out.size();
|
||||
speex_resampler_process_interleaved_int(tech_pvt->bidirectional_audio_resampler, in.data(), &in_len, out.data(), &out_len);
|
||||
|
||||
if (out_len > out.size()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Resampler output exceeded maximum buffer size!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
// Resize the pcm_data to match the output length from resampler, and then copy the resampled data into it.
|
||||
pcm_data.resize(out_len);
|
||||
memcpy(pcm_data.data(), out.data(), out_len * sizeof(int16_t));
|
||||
}
|
||||
switch_mutex_lock(tech_pvt->mutex);
|
||||
|
||||
// Resize the buffer if necessary
|
||||
size_t bytesResampled = pcm_data.size() * sizeof(uint16_t);
|
||||
if (cBuffer->capacity() - cBuffer->size() < bytesResampled / sizeof(uint16_t)) {
|
||||
// If buffer exceeds some max size, you could return SWITCH_STATUS_FALSE to abort the transfer
|
||||
// if (cBuffer->size() + std::max(bytesResampled / sizeof(uint16_t), (size_t)BUFFER_GROW_SIZE) > MAX_BUFFER_SIZE) return SWITCH_STATUS_FALSE;
|
||||
|
||||
cBuffer->set_capacity(cBuffer->size() + std::max(bytesResampled / sizeof(uint16_t), (size_t)BUFFER_GROW_SIZE));
|
||||
}
|
||||
// Push the data into the buffer.
|
||||
cBuffer->insert(cBuffer->end(), pcm_data.begin(), pcm_data.end());
|
||||
|
||||
switch_mutex_unlock(tech_pvt->mutex);
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
void processIncomingMessage(private_t* tech_pvt, switch_core_session_t* session, const char* message) {
|
||||
std::string msg = message;
|
||||
std::string type;
|
||||
@@ -38,7 +86,10 @@ namespace {
|
||||
if (json) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "(%u) processIncomingMessage - received %s message\n", tech_pvt->id, type.c_str());
|
||||
cJSON* jsonData = cJSON_GetObjectItem(json, "data");
|
||||
if (0 == type.compare("playAudio")) {
|
||||
if (0 == type.compare("playAudio") &&
|
||||
// playAudio is enabled and there is no bidirectional audio from stream is enabled.
|
||||
tech_pvt->bidirectional_audio_enable &&
|
||||
!tech_pvt->bidirectional_audio_stream) {
|
||||
if (jsonData) {
|
||||
// dont send actual audio bytes in event message
|
||||
cJSON* jsonFile = NULL;
|
||||
@@ -156,7 +207,7 @@ namespace {
|
||||
}
|
||||
}
|
||||
|
||||
static void eventCallback(const char* sessionId, const char* bugname, drachtio::AudioPipe::NotifyEvent_t event, const char* message) {
|
||||
static void eventCallback(const char* sessionId, const char* bugname, drachtio::AudioPipe::NotifyEvent_t event, const char* message, const char* binary, size_t len) {
|
||||
switch_core_session_t* session = switch_core_session_locate(sessionId);
|
||||
if (session) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
@@ -198,6 +249,9 @@ namespace {
|
||||
case drachtio::AudioPipe::MESSAGE:
|
||||
processIncomingMessage(tech_pvt, session, message);
|
||||
break;
|
||||
case drachtio::AudioPipe::BINARY:
|
||||
processIncomingBinary(tech_pvt, session, binary, len);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -206,11 +260,13 @@ namespace {
|
||||
}
|
||||
switch_status_t fork_data_init(private_t *tech_pvt, switch_core_session_t *session, char * host,
|
||||
unsigned int port, char* path, int sslFlags, int sampling, int desiredSampling, int channels,
|
||||
char *bugname, char* metadata, responseHandler_t responseHandler) {
|
||||
char *bugname, char* metadata, int bidirectional_audio_enable,
|
||||
int bidirectional_audio_stream, int bidirectional_audio_sample_rate, responseHandler_t responseHandler) {
|
||||
|
||||
const char* username = nullptr;
|
||||
const char* password = nullptr;
|
||||
int err;
|
||||
int bidirectional_audio_stream_enable = bidirectional_audio_enable + bidirectional_audio_stream;
|
||||
switch_codec_implementation_t read_impl;
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
|
||||
@@ -234,13 +290,17 @@ namespace {
|
||||
tech_pvt->buffer_overrun_notified = 0;
|
||||
tech_pvt->audio_paused = 0;
|
||||
tech_pvt->graceful_shutdown = 0;
|
||||
tech_pvt->circularBuffer = (void *) new CircularBuffer_t(8192);
|
||||
tech_pvt->bidirectional_audio_enable = bidirectional_audio_enable;
|
||||
tech_pvt->bidirectional_audio_stream = bidirectional_audio_stream;
|
||||
tech_pvt->bidirectional_audio_sample_rate = bidirectional_audio_sample_rate;
|
||||
strncpy(tech_pvt->bugname, bugname, MAX_BUG_LEN);
|
||||
if (metadata) strncpy(tech_pvt->initialMetadata, metadata, MAX_METADATA_LEN);
|
||||
|
||||
size_t buflen = LWS_PRE + (FRAME_SIZE_8000 * desiredSampling / 8000 * channels * 1000 / RTP_PACKETIZATION_PERIOD * nAudioBufferSecs);
|
||||
|
||||
drachtio::AudioPipe* ap = new drachtio::AudioPipe(tech_pvt->sessionId, host, port, path, sslFlags,
|
||||
buflen, read_impl.decoded_bytes_per_packet, username, password, bugname, eventCallback);
|
||||
buflen, read_impl.decoded_bytes_per_packet, username, password, bugname, bidirectional_audio_stream_enable, eventCallback);
|
||||
if (!ap) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error allocating AudioPipe\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
@@ -262,6 +322,15 @@ namespace {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) no resampling needed for this call\n", tech_pvt->id);
|
||||
}
|
||||
|
||||
if (bidirectional_audio_sample_rate && sampling != bidirectional_audio_sample_rate) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) bidirectional audio resampling from %u to %u\n", tech_pvt->id, bidirectional_audio_sample_rate, sampling);
|
||||
tech_pvt->bidirectional_audio_resampler = speex_resampler_init(channels, bidirectional_audio_sample_rate, sampling, SWITCH_RESAMPLE_QUALITY, &err);
|
||||
if (0 != err) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error initializing bidirectional audio resampler: %s.\n", speex_resampler_strerror(err));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) fork_data_init\n", tech_pvt->id);
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
@@ -273,10 +342,19 @@ namespace {
|
||||
speex_resampler_destroy(tech_pvt->resampler);
|
||||
tech_pvt->resampler = nullptr;
|
||||
}
|
||||
if (tech_pvt->bidirectional_audio_resampler) {
|
||||
speex_resampler_destroy(tech_pvt->bidirectional_audio_resampler);
|
||||
tech_pvt->bidirectional_audio_resampler = nullptr;
|
||||
}
|
||||
if (tech_pvt->mutex) {
|
||||
switch_mutex_destroy(tech_pvt->mutex);
|
||||
tech_pvt->mutex = nullptr;
|
||||
}
|
||||
if (tech_pvt->circularBuffer) {
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->circularBuffer;
|
||||
delete cBuffer;
|
||||
tech_pvt->circularBuffer = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
void lws_logger(int level, const char *line) {
|
||||
@@ -341,7 +419,10 @@ extern "C" {
|
||||
}
|
||||
|
||||
std::string strHost(server + offset);
|
||||
std::regex re("^(.+?):?(\\d+)?(/.*)?$");
|
||||
//- `([^/:]+)` captures the hostname/IP address, match any character except in the set
|
||||
//- `:?([0-9]*)?` optionally captures a colon and the port number, if it's present.
|
||||
//- `(/.*)` captures everything else (the path).
|
||||
std::regex re("([^/:]+):?([0-9]*)?(/.*)?$");
|
||||
std::smatch matches;
|
||||
if(std::regex_search(strHost, matches, re)) {
|
||||
/*
|
||||
@@ -392,18 +473,22 @@ extern "C" {
|
||||
}
|
||||
|
||||
switch_status_t fork_session_init(switch_core_session_t *session,
|
||||
responseHandler_t responseHandler,
|
||||
uint32_t samples_per_second,
|
||||
char *host,
|
||||
unsigned int port,
|
||||
char *path,
|
||||
int sampling,
|
||||
int sslFlags,
|
||||
int channels,
|
||||
char *bugname,
|
||||
char* metadata,
|
||||
void **ppUserData)
|
||||
{
|
||||
responseHandler_t responseHandler,
|
||||
uint32_t samples_per_second,
|
||||
char *host,
|
||||
unsigned int port,
|
||||
char *path,
|
||||
int sampling,
|
||||
int sslFlags,
|
||||
int channels,
|
||||
char *bugname,
|
||||
char* metadata,
|
||||
int bidirectional_audio_enable,
|
||||
int bidirectional_audio_stream,
|
||||
int bidirectional_audio_sample_rate,
|
||||
void **ppUserData
|
||||
)
|
||||
{
|
||||
int err;
|
||||
|
||||
// allocate per-session data structure
|
||||
@@ -412,8 +497,9 @@ extern "C" {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "error allocating memory!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (SWITCH_STATUS_SUCCESS != fork_data_init(tech_pvt, session, host, port, path, sslFlags, samples_per_second, sampling, channels,
|
||||
bugname, metadata, responseHandler)) {
|
||||
bugname, metadata, bidirectional_audio_enable, bidirectional_audio_stream, bidirectional_audio_sample_rate, responseHandler)) {
|
||||
destroy_tech_pvt(tech_pvt);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
@@ -619,5 +705,53 @@ extern "C" {
|
||||
return SWITCH_TRUE;
|
||||
}
|
||||
|
||||
switch_bool_t dub_speech_frame(switch_media_bug_t *bug, private_t* tech_pvt) {
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->circularBuffer;
|
||||
if (switch_mutex_trylock(tech_pvt->mutex) == SWITCH_STATUS_SUCCESS) {
|
||||
switch_frame_t* rframe = switch_core_media_bug_get_write_replace_frame(bug);
|
||||
int16_t *fp = reinterpret_cast<int16_t*>(rframe->data);
|
||||
|
||||
rframe->channels = 1;
|
||||
rframe->datalen = rframe->samples * sizeof(int16_t);
|
||||
|
||||
int16_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
|
||||
memset(data, 0, sizeof(data));
|
||||
|
||||
int samplesToCopy = std::min(static_cast<int>(cBuffer->size()), static_cast<int>(rframe->samples));
|
||||
|
||||
std::copy_n(cBuffer->begin(), samplesToCopy, data);
|
||||
cBuffer->erase(cBuffer->begin(), cBuffer->begin() + samplesToCopy);
|
||||
|
||||
if (samplesToCopy > 0) {
|
||||
vector_add(fp, data, rframe->samples);
|
||||
}
|
||||
vector_normalize(fp, rframe->samples);
|
||||
|
||||
switch_core_media_bug_set_write_replace_frame(bug, rframe);
|
||||
switch_mutex_unlock(tech_pvt->mutex);
|
||||
}
|
||||
return SWITCH_TRUE;
|
||||
}
|
||||
|
||||
switch_status_t fork_session_stop_play(switch_core_session_t *session, char *bugname) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, bugname);
|
||||
if (!bug) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "fork_session_stop_play failed because no bug\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
private_t* tech_pvt = (private_t*) switch_core_media_bug_get_user_data(bug);
|
||||
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) tech_pvt->circularBuffer;
|
||||
|
||||
if (switch_mutex_trylock(tech_pvt->mutex) == SWITCH_STATUS_SUCCESS) {
|
||||
if (cBuffer != nullptr) {
|
||||
cBuffer->clear();
|
||||
}
|
||||
switch_mutex_unlock(tech_pvt->mutex);
|
||||
}
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -8,13 +8,16 @@ int parse_ws_uri(switch_channel_t *channel, const char* szServerUri, char* host,
|
||||
switch_status_t fork_init();
|
||||
switch_status_t fork_cleanup();
|
||||
switch_status_t fork_session_init(switch_core_session_t *session, responseHandler_t responseHandler,
|
||||
uint32_t samples_per_second, char *host, unsigned int port, char* path, int sampling, int sslFlags, int channels,
|
||||
char *bugname, char* metadata, void **ppUserData);
|
||||
uint32_t samples_per_second, char *host, unsigned int port, char* path, int sampling, int sslFlags, int channels,
|
||||
char *bugname, char* metadata, int bidirectional_audio_enable,
|
||||
int bidirectional_audio_stream, int bidirectional_audio_sample_rate, void **ppUserData);
|
||||
switch_status_t fork_session_cleanup(switch_core_session_t *session, char *bugname, char* text, int channelIsClosing);
|
||||
switch_status_t fork_session_stop_play(switch_core_session_t *session, char *bugname);
|
||||
switch_status_t fork_session_pauseresume(switch_core_session_t *session, char *bugname, int pause);
|
||||
switch_status_t fork_session_graceful_shutdown(switch_core_session_t *session, char *bugname);
|
||||
switch_status_t fork_session_send_text(switch_core_session_t *session, char *bugname, char* text);
|
||||
switch_bool_t fork_frame(switch_core_session_t *session, switch_media_bug_t *bug);
|
||||
switch_bool_t dub_speech_frame(switch_media_bug_t *bug, private_t * tech_pvt);
|
||||
switch_status_t fork_service_threads();
|
||||
switch_status_t fork_session_connect(void **ppUserData);
|
||||
#endif
|
||||
|
||||
@@ -27,40 +27,48 @@ static void responseHandler(switch_core_session_t* session, const char * eventNa
|
||||
|
||||
static switch_bool_t capture_callback(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type)
|
||||
{
|
||||
switch_bool_t ret = SWITCH_TRUE;
|
||||
switch_core_session_t *session = switch_core_media_bug_get_session(bug);
|
||||
private_t* tech_pvt = (private_t *) switch_core_media_bug_get_user_data(bug);
|
||||
switch (type) {
|
||||
case SWITCH_ABC_TYPE_INIT:
|
||||
break;
|
||||
|
||||
case SWITCH_ABC_TYPE_CLOSE:
|
||||
{
|
||||
private_t* tech_pvt = (private_t *) switch_core_media_bug_get_user_data(bug);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "Got SWITCH_ABC_TYPE_CLOSE for bug %s\n", tech_pvt->bugname);
|
||||
fork_session_cleanup(session, tech_pvt->bugname, NULL, 1);
|
||||
}
|
||||
break;
|
||||
|
||||
case SWITCH_ABC_TYPE_READ:
|
||||
return fork_frame(session, bug);
|
||||
ret = fork_frame(session, bug);
|
||||
break;
|
||||
|
||||
case SWITCH_ABC_TYPE_WRITE_REPLACE:
|
||||
ret = dub_speech_frame(bug, tech_pvt);
|
||||
break;
|
||||
|
||||
case SWITCH_ABC_TYPE_WRITE:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return SWITCH_TRUE;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static switch_status_t start_capture(switch_core_session_t *session,
|
||||
switch_media_bug_flag_t flags,
|
||||
char* host,
|
||||
unsigned int port,
|
||||
char* path,
|
||||
int sampling,
|
||||
int sslFlags,
|
||||
char* bugname,
|
||||
char* metadata)
|
||||
switch_media_bug_flag_t flags,
|
||||
char* host,
|
||||
unsigned int port,
|
||||
char* path,
|
||||
int sampling,
|
||||
int sslFlags,
|
||||
int bidirectional_audio_enable,
|
||||
int bidirectional_audio_stream,
|
||||
int bidirectional_audio_sample_rate,
|
||||
char* bugname,
|
||||
char* metadata)
|
||||
{
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_media_bug_t *bug;
|
||||
@@ -71,8 +79,8 @@ static switch_status_t start_capture(switch_core_session_t *session,
|
||||
int channels = (flags & SMBF_STEREO) ? 2 : 1;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO,
|
||||
"mod_audio_fork (%s): streaming %d sampling to %s path %s port %d tls: %s.\n",
|
||||
bugname, sampling, host, path, port, sslFlags ? "yes" : "no");
|
||||
"mod_audio_fork (%s): streaming %d sampling to %s path %s port %d tls: %s bidirectional_audio_sample_rate: %d.\n",
|
||||
bugname, sampling, host, path, port, sslFlags ? "yes" : "no", bidirectional_audio_sample_rate);
|
||||
|
||||
if (switch_channel_get_private(channel, bugname)) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "mod_audio_fork: bug %s already attached!\n", bugname);
|
||||
@@ -88,10 +96,12 @@ static switch_status_t start_capture(switch_core_session_t *session,
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "calling fork_session_init.\n");
|
||||
if (SWITCH_STATUS_FALSE == fork_session_init(session, responseHandler, read_codec->implementation->actual_samples_per_second,
|
||||
host, port, path, sampling, sslFlags, channels, bugname, metadata, &pUserData)) {
|
||||
host, port, path, sampling, sslFlags, channels, bugname, metadata, bidirectional_audio_enable, bidirectional_audio_stream,
|
||||
bidirectional_audio_sample_rate, &pUserData)) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error initializing mod_audio_fork session.\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "adding bug %s.\n", bugname);
|
||||
if ((status = switch_core_media_bug_add(session, bugname, NULL, capture_callback, pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
|
||||
return status;
|
||||
@@ -133,6 +143,16 @@ static switch_status_t do_pauseresume(switch_core_session_t *session, char* bugn
|
||||
return status;
|
||||
}
|
||||
|
||||
static switch_status_t stop_play(switch_core_session_t *session, char* bugname)
|
||||
{
|
||||
switch_status_t status = SWITCH_STATUS_SUCCESS;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "mod_audio_fork stop_play\n");
|
||||
status = fork_session_stop_play(session, bugname);
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
static switch_status_t do_graceful_shutdown(switch_core_session_t *session, char* bugname)
|
||||
{
|
||||
switch_status_t status = SWITCH_STATUS_SUCCESS;
|
||||
@@ -159,10 +179,10 @@ static switch_status_t send_text(switch_core_session_t *session, char* bugname,
|
||||
return status;
|
||||
}
|
||||
|
||||
#define FORK_API_SYNTAX "<uuid> [start | stop | send_text | pause | resume | graceful-shutdown ] [wss-url | path] [mono | mixed | stereo] [8000 | 16000 | 24000 | 32000 | 64000] [bugname] [metadata]"
|
||||
#define FORK_API_SYNTAX "<uuid> [start | stop | send_text | pause | resume | graceful-shutdown | stop_play ] [wss-url | path] [mono | mixed | stereo] [8000 | 16000 | 24000 | 32000 | 64000] [bugname] [metadata] [bidirectionalAudio_enabled] [bidirectionalAudio_stream_enabled] [bidirectionalAudio_stream_samplerate]"
|
||||
SWITCH_STANDARD_API(fork_function)
|
||||
{
|
||||
char *mycmd = NULL, *argv[7] = { 0 };
|
||||
char *mycmd = NULL, *argv[10] = { 0 };
|
||||
int argc = 0;
|
||||
switch_status_t status = SWITCH_STATUS_FALSE;
|
||||
char *bugname = MY_BUG_NAME;
|
||||
@@ -196,6 +216,9 @@ SWITCH_STANDARD_API(fork_function)
|
||||
}
|
||||
status = do_stop(lsession, bugname, text);
|
||||
}
|
||||
else if (!strcasecmp(argv[1], "stop_play")) {
|
||||
status = stop_play(lsession, bugname);
|
||||
}
|
||||
else if (!strcasecmp(argv[1], "pause")) {
|
||||
if (argc > 2) bugname = argv[2];
|
||||
status = do_pauseresume(lsession, bugname, 1);
|
||||
@@ -230,10 +253,31 @@ SWITCH_STANDARD_API(fork_function)
|
||||
char host[MAX_WS_URL_LEN], path[MAX_PATH_LEN];
|
||||
unsigned int port;
|
||||
int sslFlags;
|
||||
|
||||
int sampling = 8000;
|
||||
switch_media_bug_flag_t flags = SMBF_READ_STREAM ;
|
||||
switch_media_bug_flag_t flags = SMBF_READ_STREAM;
|
||||
char *metadata = NULL;
|
||||
if( argc > 6) {
|
||||
int bidirectional_audio_enable = 1;
|
||||
int bidirectional_audio_stream = 0;
|
||||
int bidirectional_audio_sample_rate = 0;
|
||||
// Expecting that bidirectional audio params is always received together with bugname and metadata even they are empty string
|
||||
if (argc > 9) {
|
||||
if (argv[5][0] != '\0') {
|
||||
bugname = argv[5];
|
||||
}
|
||||
if (argv[6][0] != '\0') {
|
||||
metadata = argv[6];
|
||||
}
|
||||
bidirectional_audio_enable = !strcmp(argv[7], "true") ? 1 : 0;
|
||||
bidirectional_audio_stream = !strcmp(argv[8], "true") ? 1 : 0;
|
||||
bidirectional_audio_sample_rate = atoi(argv[9]);
|
||||
|
||||
if (bidirectional_audio_enable &&
|
||||
bidirectional_audio_stream &&
|
||||
bidirectional_audio_sample_rate) {
|
||||
flags |= SMBF_WRITE_REPLACE ;
|
||||
}
|
||||
} else if( argc > 6 ) {
|
||||
bugname = argv[5];
|
||||
metadata = argv[6];
|
||||
}
|
||||
@@ -241,6 +285,7 @@ SWITCH_STANDARD_API(fork_function)
|
||||
if (argv[5][0] == '{' || argv[5][0] == '[') metadata = argv[5];
|
||||
else bugname = argv[5];
|
||||
}
|
||||
|
||||
if (0 == strcmp(argv[3], "mixed")) {
|
||||
flags |= SMBF_WRITE_STREAM ;
|
||||
}
|
||||
@@ -268,7 +313,8 @@ SWITCH_STANDARD_API(fork_function)
|
||||
else if (sampling % 8000 != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "invalid sample rate: %s\n", argv[4]);
|
||||
}
|
||||
status = start_capture(lsession, flags, host, port, path, sampling, sslFlags, bugname, metadata);
|
||||
status = start_capture(lsession, flags, host, port, path, sampling, sslFlags,
|
||||
bidirectional_audio_enable, bidirectional_audio_stream, bidirectional_audio_sample_rate, bugname, metadata);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "unsupported mod_audio_fork cmd: %s\n", argv[1]);
|
||||
|
||||
@@ -52,6 +52,11 @@ struct private_data {
|
||||
int audio_paused:1;
|
||||
int graceful_shutdown:1;
|
||||
char initialMetadata[8192];
|
||||
void *circularBuffer;
|
||||
SpeexResamplerState *bidirectional_audio_resampler;
|
||||
int bidirectional_audio_enable;
|
||||
int bidirectional_audio_stream;
|
||||
int bidirectional_audio_sample_rate;
|
||||
};
|
||||
|
||||
typedef struct private_data private_t;
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
#include "vector_math.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
#include <cstdlib>
|
||||
|
||||
#define GRANULAR_VOLUME_MAX (50)
|
||||
#define SMAX 32767
|
||||
#define SMIN (-32768)
|
||||
#define normalize_to_16bit_basic(n) if (n > SMAX) n = SMAX; else if (n < SMIN) n = SMIN;
|
||||
#define normalize_volume_granular(x) if (x > GRANULAR_VOLUME_MAX) x = GRANULAR_VOLUME_MAX; if (x < -GRANULAR_VOLUME_MAX) x = -GRANULAR_VOLUME_MAX;
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#if defined(USE_AVX2)
|
||||
#include <immintrin.h>
|
||||
#pragma message("Using AVX2 SIMD.")
|
||||
void vector_add(int16_t* a, int16_t* b, size_t len) {
|
||||
size_t i = 0;
|
||||
for (; i + 15 < len; i += 16) {
|
||||
__m256i va = _mm256_loadu_si256((const __m256i*)(a + i));
|
||||
__m256i vb = _mm256_loadu_si256((const __m256i*)(b + i));
|
||||
__m256i vc = _mm256_add_epi16(va, vb);
|
||||
_mm256_storeu_si256((__m256i*)(a + i), vc);
|
||||
}
|
||||
for (; i < len; ++i) {
|
||||
a[i] += b[i];
|
||||
}
|
||||
}
|
||||
void vector_normalize(int16_t* a, size_t len) {
|
||||
__m256i max_val = _mm256_set1_epi16(SMAX);
|
||||
__m256i min_val = _mm256_set1_epi16(SMIN);
|
||||
|
||||
size_t i = 0;
|
||||
for (; i + 15 < len; i += 16) {
|
||||
__m256i values = _mm256_loadu_si256((__m256i*)(a + i));
|
||||
__m256i gt_max = _mm256_cmpgt_epi16(values, max_val);
|
||||
__m256i lt_min = _mm256_cmpgt_epi16(min_val, values);
|
||||
values = _mm256_blendv_epi8(values, max_val, gt_max);
|
||||
values = _mm256_blendv_epi8(values, min_val, lt_min);
|
||||
_mm256_storeu_si256((__m256i*)(a + i), values);
|
||||
}
|
||||
|
||||
// Process remaining elements
|
||||
for (; i < len; ++i) {
|
||||
if (a[i] > SMAX) a[i] = SMAX;
|
||||
else if (a[i] < SMIN) a[i] = SMIN;
|
||||
}
|
||||
}
|
||||
typedef union {
|
||||
int16_t* data;
|
||||
__m256i* fp_avx2;
|
||||
} vector_data_t;
|
||||
|
||||
void vector_change_sln_volume_granular(int16_t* data, uint32_t samples, int32_t vol) {
|
||||
float newrate = 0;
|
||||
static const float pos[GRANULAR_VOLUME_MAX] = {
|
||||
1.122018, 1.258925, 1.412538, 1.584893, 1.778279, 1.995262, 2.238721, 2.511887, 2.818383, 3.162278,
|
||||
3.548134, 3.981072, 4.466835, 5.011872, 5.623413, 6.309574, 7.079458, 7.943282, 8.912509, 10.000000,
|
||||
11.220183, 12.589254, 14.125375, 15.848933, 17.782795, 19.952621, 22.387213, 25.118862, 28.183832, 31.622776,
|
||||
35.481335, 39.810719, 44.668358, 50.118729, 56.234131, 63.095726, 70.794586, 79.432816, 89.125107, 100.000000,
|
||||
112.201836, 125.892517, 141.253784, 158.489334, 177.827942, 199.526215, 223.872070, 251.188705, 281.838318, 316.227753
|
||||
};
|
||||
static const float neg[GRANULAR_VOLUME_MAX] = {
|
||||
0.891251, 0.794328, 0.707946, 0.630957, 0.562341, 0.501187, 0.446684, 0.398107, 0.354813, 0.316228,
|
||||
0.281838, 0.251189, 0.223872, 0.199526, 0.177828, 0.158489, 0.141254, 0.125893, 0.112202, 0.100000,
|
||||
0.089125, 0.079433, 0.070795, 0.063096, 0.056234, 0.050119, 0.044668, 0.039811, 0.035481, 0.031623,
|
||||
0.028184, 0.025119, 0.022387, 0.019953, 0.017783, 0.015849, 0.014125, 0.012589, 0.011220, 0.010000,
|
||||
0.008913, 0.007943, 0.007079, 0.006310, 0.005623, 0.005012, 0.004467, 0.003981, 0.003548, 0.000000 // NOTE mapped -50 dB ratio to total silence instead of 0.003162
|
||||
};
|
||||
const float* chart;
|
||||
uint32_t i = abs(vol) - 1;
|
||||
|
||||
if (vol == 0) return;
|
||||
normalize_volume_granular(vol);
|
||||
|
||||
chart = vol > 0 ? pos : neg;
|
||||
|
||||
newrate = chart[i];
|
||||
|
||||
if (newrate) {
|
||||
__m256 scale_factor_reg = _mm256_set1_ps(newrate);
|
||||
uint32_t processed_samples = samples - (samples % 8); // Ensure we process only multiples of 8
|
||||
for (uint32_t i = 0; i < processed_samples; i += 8) {
|
||||
__m128i data_ = _mm_loadu_si128((__m128i*)(data + i));
|
||||
__m256i data_32 = _mm256_cvtepi16_epi32(data_);
|
||||
|
||||
__m256 data_float = _mm256_cvtepi32_ps(data_32);
|
||||
__m256 result = _mm256_mul_ps(data_float, scale_factor_reg);
|
||||
|
||||
__m256i result_32 = _mm256_cvtps_epi32(result);
|
||||
|
||||
// Handle saturation
|
||||
__m256i min_val = _mm256_set1_epi32(SMIN);
|
||||
__m256i max_val = _mm256_set1_epi32(SMAX);
|
||||
result_32 = _mm256_min_epi32(result_32, max_val);
|
||||
result_32 = _mm256_max_epi32(result_32, min_val);
|
||||
|
||||
__m128i result_16 = _mm_packs_epi32(_mm256_castsi256_si128(result_32), _mm256_extractf128_si256(result_32, 1));
|
||||
|
||||
_mm_storeu_si128((__m128i*)(data + i), result_16);
|
||||
}
|
||||
|
||||
// Process any remaining samples
|
||||
for (uint32_t i = processed_samples; i < samples; i++) {
|
||||
int32_t tmp = (int32_t)(data[i] * newrate);
|
||||
tmp = tmp > SMAX ? SMAX : (tmp < SMIN ? SMIN : tmp);
|
||||
data[i] = (int16_t)tmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
#elif defined(USE_SSE2)
|
||||
#include <emmintrin.h>
|
||||
#pragma message("Using SSE2 SIMD.")
|
||||
void vector_add(int16_t* a, int16_t* b, size_t len) {
|
||||
size_t i = 0;
|
||||
for (; i + 7 < len; i += 8) {
|
||||
__m128i va = _mm_loadu_si128((const __m128i*)(a + i));
|
||||
__m128i vb = _mm_loadu_si128((const __m128i*)(b + i));
|
||||
__m128i vc = _mm_add_epi16(va, vb);
|
||||
_mm_storeu_si128((__m128i*)(a + i), vc);
|
||||
}
|
||||
for (; i < len; ++i) {
|
||||
a[i] += b[i];
|
||||
}
|
||||
}
|
||||
void vector_normalize(int16_t* a, size_t len) {
|
||||
__m128i max_val = _mm_set1_epi16(SMAX);
|
||||
__m128i min_val = _mm_set1_epi16(SMIN);
|
||||
|
||||
size_t i = 0;
|
||||
for (; i + 7 < len; i += 8) {
|
||||
__m128i values = _mm_loadu_si128((__m128i*)(a + i));
|
||||
__m128i gt_max = _mm_cmpgt_epi16(values, max_val);
|
||||
__m128i lt_min = _mm_cmpgt_epi16(min_val, values);
|
||||
__m128i max_masked = _mm_and_si128(gt_max, max_val);
|
||||
__m128i min_masked = _mm_and_si128(lt_min, min_val);
|
||||
__m128i other_masked = _mm_andnot_si128(_mm_or_si128(gt_max, lt_min), values);
|
||||
values = _mm_or_si128(_mm_or_si128(max_masked, min_masked), other_masked);
|
||||
_mm_storeu_si128((__m128i*)(a + i), values);
|
||||
}
|
||||
|
||||
// Process remaining elements
|
||||
for (; i < len; ++i) {
|
||||
if (a[i] > SMAX) a[i] = SMAX;
|
||||
else if (a[i] < SMIN) a[i] = SMIN;
|
||||
}
|
||||
}
|
||||
|
||||
typedef union {
|
||||
int16_t* data;
|
||||
__m128i* fp_sse2;
|
||||
} vector_data_t;
|
||||
|
||||
#else
|
||||
#pragma message("Building without vector math support")
|
||||
void vector_add(int16_t* a, int16_t* b, size_t len) {
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
a[i] += b[i];
|
||||
}
|
||||
}
|
||||
void vector_normalize(int16_t* a, size_t len) {
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
normalize_to_16bit_basic(a[i]);
|
||||
}
|
||||
}
|
||||
void vector_change_sln_volume_granular(int16_t* data, uint32_t samples, int32_t vol) {
|
||||
float newrate = 0;
|
||||
static const float pos[GRANULAR_VOLUME_MAX] = {
|
||||
1.122018, 1.258925, 1.412538, 1.584893, 1.778279, 1.995262, 2.238721, 2.511887, 2.818383, 3.162278,
|
||||
3.548134, 3.981072, 4.466835, 5.011872, 5.623413, 6.309574, 7.079458, 7.943282, 8.912509, 10.000000,
|
||||
11.220183, 12.589254, 14.125375, 15.848933, 17.782795, 19.952621, 22.387213, 25.118862, 28.183832, 31.622776,
|
||||
35.481335, 39.810719, 44.668358, 50.118729, 56.234131, 63.095726, 70.794586, 79.432816, 89.125107, 100.000000,
|
||||
112.201836, 125.892517, 141.253784, 158.489334, 177.827942, 199.526215, 223.872070, 251.188705, 281.838318, 316.227753
|
||||
};
|
||||
static const float neg[GRANULAR_VOLUME_MAX] = {
|
||||
0.891251, 0.794328, 0.707946, 0.630957, 0.562341, 0.501187, 0.446684, 0.398107, 0.354813, 0.316228,
|
||||
0.281838, 0.251189, 0.223872, 0.199526, 0.177828, 0.158489, 0.141254, 0.125893, 0.112202, 0.100000,
|
||||
0.089125, 0.079433, 0.070795, 0.063096, 0.056234, 0.050119, 0.044668, 0.039811, 0.035481, 0.031623,
|
||||
0.028184, 0.025119, 0.022387, 0.019953, 0.017783, 0.015849, 0.014125, 0.012589, 0.011220, 0.010000,
|
||||
0.008913, 0.007943, 0.007079, 0.006310, 0.005623, 0.005012, 0.004467, 0.003981, 0.003548, 0.000000 // NOTE mapped -50 dB ratio to total silence instead of 0.003162
|
||||
};
|
||||
const float* chart;
|
||||
uint32_t i;
|
||||
|
||||
if (vol == 0) return;
|
||||
|
||||
normalize_volume_granular(vol);
|
||||
|
||||
chart = vol > 0 ? pos : neg;
|
||||
|
||||
i = abs(vol) - 1;
|
||||
assert(i < GRANULAR_VOLUME_MAX);
|
||||
newrate = chart[i];
|
||||
if (newrate) {
|
||||
int32_t tmp;
|
||||
uint32_t x;
|
||||
int16_t *fp = data;
|
||||
|
||||
for (x = 0; x < samples; x++) {
|
||||
tmp = (int32_t) (fp[x] * newrate);
|
||||
normalize_to_16bit_basic(tmp);
|
||||
fp[x] = (int16_t) tmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,20 @@
|
||||
#ifndef VECTOR_MATH_H
|
||||
#define VECTOR_MATH_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
void vector_add(int16_t* a, int16_t* b, size_t len);
|
||||
void vector_normalize(int16_t* a, size_t len);
|
||||
void vector_change_sln_volume_granular(int16_t* data, uint32_t samples, int32_t vol);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -7,4 +7,4 @@ mod_aws_lex_la_CFLAGS = $(AM_CFLAGS)
|
||||
mod_aws_lex_la_CXXFLAGS = $(AM_CXXFLAGS) -std=c++11 -I${switch_srcdir}/libs/aws-sdk-cpp/aws-cpp-sdk-core/include -I${switch_srcdir}/libs/aws-sdk-cpp/aws-cpp-sdk-lexv2-runtime/include -I${switch_srcdir}/libs/aws-sdk-cpp/build/.deps/install/include
|
||||
|
||||
mod_aws_lex_la_LIBADD = $(switch_builddir)/libfreeswitch.la
|
||||
mod_aws_lex_la_LDFLAGS = -avoid-version -module -no-undefined -L${switch_srcdir}/libs/aws-sdk-cpp/build/.deps/install/lib -L${switch_srcdir}/libs/aws-sdk-cpp/build/aws-cpp-sdk-core -L${switch_srcdir}/libs/aws-sdk-cpp/build/aws-cpp-sdk-lexv2-runtime -laws-cpp-sdk-lexv2-runtime -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lpthread -lcurl -lcrypto -lssl -lz
|
||||
mod_aws_lex_la_LDFLAGS = -avoid-version -module -no-undefined -L/usr/local/lib -L${switch_srcdir}/libs/aws-sdk-cpp/build/.deps/install/lib -L${switch_srcdir}/libs/aws-sdk-cpp/build/aws-cpp-sdk-core -L${switch_srcdir}/libs/aws-sdk-cpp/build/aws-cpp-sdk-lexv2-runtime -laws-cpp-sdk-lexv2-runtime -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lpthread -lcurl -lcrypto -lssl -lz
|
||||
@@ -7,4 +7,4 @@ mod_aws_transcribe_la_CFLAGS = $(AM_CFLAGS)
|
||||
mod_aws_transcribe_la_CXXFLAGS = $(AM_CXXFLAGS) -std=c++11 -I${switch_srcdir}/libs/aws-sdk-cpp/aws-cpp-sdk-core/include -I${switch_srcdir}/libs/aws-sdk-cpp/aws-cpp-sdk-transcribestreaming/include -I${switch_srcdir}/libs/aws-sdk-cpp/build/.deps/install/include
|
||||
|
||||
mod_aws_transcribe_la_LIBADD = $(switch_builddir)/libfreeswitch.la
|
||||
mod_aws_transcribe_la_LDFLAGS = -avoid-version -module -no-undefined -L${switch_srcdir}/libs/aws-sdk-cpp/build/.deps/install/lib -L${switch_srcdir}/libs/aws-sdk-cpp/build/aws-cpp-sdk-core -L${switch_srcdir}/libs/aws-sdk-cpp/build/aws-cpp-sdk-transcribestreaming -laws-cpp-sdk-transcribestreaming -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -lpthread -lcurl -lcrypto -lssl -lz
|
||||
mod_aws_transcribe_la_LDFLAGS = -avoid-version -module -no-undefined -L/usr/local/lib -L${switch_srcdir}/libs/aws-sdk-cpp/build/.deps/install/lib -L${switch_srcdir}/libs/aws-sdk-cpp/build/aws-cpp-sdk-core -L${switch_srcdir}/libs/aws-sdk-cpp/build/aws-cpp-sdk-transcribestreaming -laws-cpp-sdk-transcribestreaming -laws-cpp-sdk-core -laws-c-event-stream -laws-checksums -laws-c-common -laws-crt-cpp -lpthread -lcurl -lcrypto -lssl -lz
|
||||
@@ -0,0 +1,8 @@
|
||||
Copyright 2023, Drachtio Communications Services, LLC
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
include $(top_srcdir)/build/modmake.rulesam
|
||||
MODNAME=mod_azure_tts
|
||||
|
||||
mod_LTLIBRARIES = mod_azure_tts.la
|
||||
mod_azure_tts_la_SOURCES = mod_azure_tts.c azure_glue.cpp
|
||||
mod_azure_tts_la_CFLAGS = $(AM_CFLAGS)
|
||||
mod_azure_tts_la_CXXFLAGS = $(AM_CXXFLAGS) -std=c++14 -I/usr/local/include/MicrosoftSpeechSDK/cxx_api -I/usr/local/include/MicrosoftSpeechSDK/c_api
|
||||
|
||||
mod_azure_tts_la_LIBADD = $(switch_builddir)/libfreeswitch.la
|
||||
mod_azure_tts_la_LDFLAGS = -avoid-version -module -no-undefined -L/usr/local/lib/MicrosoftSpeechSDK/x64 -lMicrosoft.CognitiveServices.Speech.core -shared -lstdc++ -lboost_system -lboost_thread
|
||||
@@ -0,0 +1,377 @@
|
||||
#include "mod_azure_tts.h"
|
||||
#include <switch.h>
|
||||
#include <speechapi_cxx.h>
|
||||
|
||||
#include <boost/circular_buffer.hpp>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
#include <chrono>
|
||||
#include <thread>
|
||||
|
||||
#define BUFFER_SIZE 8129
|
||||
|
||||
typedef boost::circular_buffer<uint16_t> CircularBuffer_t;
|
||||
|
||||
using namespace Microsoft::CognitiveServices::Speech;
|
||||
|
||||
static std::string fullDirPath;
|
||||
|
||||
static void start_synthesis(std::shared_ptr<SpeechSynthesizer> speechSynthesizer, const char* text, azure_t* a) {
|
||||
try {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "start_synthesis calling \n");
|
||||
auto result = std::strncmp(text, "<speak", 6) == 0 ?
|
||||
speechSynthesizer->SpeakSsmlAsync(text).get() :
|
||||
speechSynthesizer->SpeakTextAsync(text).get();
|
||||
|
||||
if (result->Reason == ResultReason::SynthesizingAudioCompleted) {
|
||||
a->response_code = 200;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "start_synthesis completed id %s, audio data - bytes: %ld, duration: %ldms\n",
|
||||
result->ResultId.c_str(), result->GetAudioLength(), result->AudioDuration.count());
|
||||
} else if (result->Reason == ResultReason::Canceled) {
|
||||
auto cancellation = SpeechSynthesisCancellationDetails::FromResult(result);
|
||||
a->response_code = static_cast<long int>(cancellation->ErrorCode);
|
||||
a->err_msg = strdup(cancellation->ErrorDetails.c_str());
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error synthesizing text %d with error string: %s.\n",
|
||||
static_cast<int>(cancellation->ErrorCode), cancellation->ErrorDetails.c_str());
|
||||
} else {
|
||||
a->response_code = 500;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error synthsize text %s (%d).\n", text, static_cast<int>(result->Reason));
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
a->response_code = 500;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "mod_azure_tts: Exception in start_synthesis %s\n", e.what());
|
||||
}
|
||||
a->draining = 1;
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
switch_status_t azure_speech_load() {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "azure_speech_loading..\n");
|
||||
|
||||
/* create temp folder for cache files */
|
||||
const char* baseDir = std::getenv("JAMBONZ_TMP_CACHE_FOLDER");
|
||||
if (!baseDir) {
|
||||
baseDir = "/tmp/";
|
||||
}
|
||||
if (strcmp(baseDir, "/") == 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", baseDir);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
fullDirPath = std::string(baseDir) + "tts-cache-files";
|
||||
|
||||
// Create the directory with read, write, and execute permissions for everyone
|
||||
mode_t oldMask = umask(0);
|
||||
int result = mkdir(fullDirPath.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
|
||||
umask(oldMask);
|
||||
if (result != 0) {
|
||||
if (errno != EEXIST) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", fullDirPath.c_str());
|
||||
fullDirPath = "";
|
||||
}
|
||||
else switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "folder %s already exists\n", fullDirPath.c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "created folder %s\n", fullDirPath.c_str());
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "azure_speech_loaded..\n");
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t azure_speech_open(azure_t* azure) {
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t azure_speech_feed_tts(azure_t* a, char* text, switch_speech_flag_t *flags) {
|
||||
const int MAX_CHARS = 20;
|
||||
char tempText[MAX_CHARS + 4]; // +4 for the ellipsis and null terminator
|
||||
|
||||
if (strlen(text) > MAX_CHARS) {
|
||||
strncpy(tempText, text, MAX_CHARS);
|
||||
strcpy(tempText + MAX_CHARS, "...");
|
||||
} else {
|
||||
strcpy(tempText, text);
|
||||
}
|
||||
|
||||
/* open cache file */
|
||||
if (a->cache_audio && fullDirPath.length() > 0) {
|
||||
switch_uuid_t uuid;
|
||||
char uuid_str[SWITCH_UUID_FORMATTED_LENGTH + 1];
|
||||
char outfile[512] = "";
|
||||
int fd;
|
||||
|
||||
switch_uuid_get(&uuid);
|
||||
switch_uuid_format(uuid_str, &uuid);
|
||||
|
||||
switch_snprintf(outfile, sizeof(outfile), "%s%s%s.r8", fullDirPath.c_str(), SWITCH_PATH_SEPARATOR, uuid_str);
|
||||
a->cache_filename = strdup(outfile);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "writing audio cache file to %s\n", a->cache_filename);
|
||||
|
||||
mode_t oldMask = umask(0);
|
||||
fd = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
|
||||
umask(oldMask);
|
||||
if (fd == -1 ) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
|
||||
}
|
||||
else {
|
||||
a->file = fdopen(fd, "wb");
|
||||
if (!a->file) {
|
||||
close(fd);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!a->api_key) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "azure_speech_feed_tts: no api_key provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (!a->language) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "azure_speech_feed_tts: no language provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (a->rate != 8000 /*Hz*/) {
|
||||
int err;
|
||||
a->resampler = speex_resampler_init(1, 8000, a->rate, SWITCH_RESAMPLE_QUALITY, &err);
|
||||
if (0 != err) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing resampler: %s.\n", speex_resampler_strerror(err));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
std::chrono::time_point<std::chrono::high_resolution_clock>* ptr = new std::chrono::time_point<std::chrono::high_resolution_clock>(std::chrono::high_resolution_clock::now());
|
||||
a->startTime = ptr;
|
||||
|
||||
a->circularBuffer = (void *) new CircularBuffer_t(BUFFER_SIZE);
|
||||
|
||||
auto speechConfig = nullptr != a->endpoint ?
|
||||
(nullptr != a->api_key ?
|
||||
SpeechConfig::FromEndpoint(a->endpoint, a->api_key) :
|
||||
SpeechConfig::FromEndpoint(a->endpoint)) :
|
||||
SpeechConfig::FromSubscription(a->api_key, a->region ? a->region : "");
|
||||
|
||||
speechConfig->SetSpeechSynthesisOutputFormat(SpeechSynthesisOutputFormat::Raw8Khz16BitMonoPcm);
|
||||
speechConfig->SetSpeechSynthesisLanguage(a->language);
|
||||
speechConfig->SetSpeechSynthesisVoiceName(a->voice_name);
|
||||
if (a->http_proxy_ip) {
|
||||
uint32_t port = a->http_proxy_port && a->http_proxy_port[0] != '\0' ? static_cast<uint32_t>(std::stoul(a->http_proxy_port)) : 80;
|
||||
speechConfig->SetProxy(a->http_proxy_ip, port);
|
||||
}
|
||||
|
||||
if (nullptr != a->endpointId) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts setting endpoint id: %s\n", a->endpointId);
|
||||
speechConfig->SetEndpointId(a->endpointId);
|
||||
}
|
||||
|
||||
try {
|
||||
auto speechSynthesizer = SpeechSynthesizer::FromConfig(speechConfig);
|
||||
|
||||
speechSynthesizer->SynthesisStarted += [a](const SpeechSynthesisEventArgs& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts SynthesisStarted\n");
|
||||
};
|
||||
|
||||
speechSynthesizer->Synthesizing += [a](const SpeechSynthesisEventArgs& e) {
|
||||
if (a->flushed) return;
|
||||
bool fireEvent = false;
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) a->circularBuffer;
|
||||
|
||||
auto audioData = e.Result->GetAudioData();
|
||||
if (a->file) {
|
||||
fwrite(audioData->data(), 1, audioData->size(), a->file);
|
||||
}
|
||||
|
||||
/**
|
||||
* this sort of reinterpretation can be dangerous as a general rule, but in this case we know that the data
|
||||
* is 16-bit PCM, so it's safe to do this and its much faster than copying the data byte by byte
|
||||
*/
|
||||
const uint16_t* begin = reinterpret_cast<const uint16_t*>(audioData->data());
|
||||
const uint16_t* end = reinterpret_cast<const uint16_t*>(audioData->data() + audioData->size());
|
||||
|
||||
/* lock as briefly as possible */
|
||||
switch_mutex_lock(a->mutex);
|
||||
if (cBuffer->capacity() - cBuffer->size() < audioData->size()) {
|
||||
cBuffer->set_capacity(cBuffer->size() + std::max( audioData->size(), (size_t)BUFFER_SIZE));
|
||||
}
|
||||
cBuffer->insert(cBuffer->end(), begin, end);
|
||||
switch_mutex_unlock(a->mutex);
|
||||
|
||||
if (0 == a->reads++) {
|
||||
fireEvent = true;
|
||||
}
|
||||
|
||||
if (fireEvent && a->session_id) {
|
||||
auto endTime = std::chrono::high_resolution_clock::now();
|
||||
auto startTime = *static_cast<std::chrono::time_point<std::chrono::high_resolution_clock>*>(a->startTime);
|
||||
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(endTime - startTime);
|
||||
auto time_to_first_byte_ms = std::to_string(duration.count());
|
||||
switch_core_session_t* session = switch_core_session_locate(a->session_id);
|
||||
if (session) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_core_session_rwunlock(session);
|
||||
if (channel) {
|
||||
switch_event_t *event;
|
||||
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_START) == SWITCH_STATUS_SUCCESS) {
|
||||
switch_channel_event_set_data(channel, event);
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
|
||||
if (a->cache_filename) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", a->cache_filename);
|
||||
}
|
||||
switch_event_fire(&event);
|
||||
} else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "speechSynthesizer->Synthesizing: failed to create event\n");
|
||||
}
|
||||
}else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "speechSynthesizer->Synthesizing: channel not found\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
std::thread(start_synthesis, speechSynthesizer, text, a).detach();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "azure_speech_feed_tts sent synthesize request\n");
|
||||
} catch (const std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mod_azure_tts: Exception: %s\n", e.what());
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t azure_speech_read_tts(azure_t* a, void *data, size_t *datalen, switch_speech_flag_t *flags) {
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) a->circularBuffer;
|
||||
std::vector<uint16_t> pcm_data;
|
||||
|
||||
if (a->response_code > 0 && a->response_code != 200) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "azure_speech_read_tts, returning failure\n") ;
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (a->flushed) {
|
||||
return SWITCH_STATUS_BREAK;
|
||||
}
|
||||
switch_mutex_lock(a->mutex);
|
||||
size_t bufSize = cBuffer->size();
|
||||
if (cBuffer->empty()) {
|
||||
switch_mutex_unlock(a->mutex);
|
||||
if (a->draining) {
|
||||
return SWITCH_STATUS_BREAK;
|
||||
}
|
||||
/* no audio available yet so send silence */
|
||||
memset(data, 255, *datalen);
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
// azure returned 8000hz 16 bit data, we have to take enough data based on call sample rate.
|
||||
size_t size = std::min((*datalen/(2 * a->rate / 8000)), bufSize);
|
||||
pcm_data.insert(pcm_data.end(), cBuffer->begin(), cBuffer->begin() + size);
|
||||
cBuffer->erase(cBuffer->begin(), cBuffer->begin() + size);
|
||||
switch_mutex_unlock(a->mutex);
|
||||
|
||||
size_t data_size = pcm_data.size();
|
||||
|
||||
if (a->resampler) {
|
||||
std::vector<int16_t> in(pcm_data.begin(), pcm_data.end());
|
||||
|
||||
std::vector<int16_t> out((*datalen));
|
||||
spx_uint32_t in_len = data_size;
|
||||
spx_uint32_t out_len = out.size();
|
||||
|
||||
speex_resampler_process_interleaved_int(a->resampler, in.data(), &in_len, out.data(), &out_len);
|
||||
|
||||
if (out_len > out.size()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Resampler output exceeded maximum buffer size!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
memcpy(data, out.data(), out_len * sizeof(int16_t));
|
||||
*datalen = out_len * sizeof(int16_t);
|
||||
} else {
|
||||
memcpy(data, pcm_data.data(), data_size * sizeof(int16_t));
|
||||
*datalen = data_size * sizeof(int16_t);
|
||||
}
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t azure_speech_flush_tts(azure_t* a) {
|
||||
bool download_complete = a->response_code == 200;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "azure_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;
|
||||
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) a->circularBuffer;
|
||||
delete cBuffer;
|
||||
a->circularBuffer = nullptr ;
|
||||
delete static_cast<std::chrono::time_point<std::chrono::high_resolution_clock>*>(a->startTime);
|
||||
a->startTime = nullptr;
|
||||
|
||||
a->flushed = 1;
|
||||
if (!download_complete) {
|
||||
if (a->file) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "closing audio cache file %s because download was interrupted\n", a->cache_filename);
|
||||
if (fclose(a->file) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "error closing audio cache file\n");
|
||||
}
|
||||
a->file = nullptr ;
|
||||
}
|
||||
|
||||
if (a->cache_filename) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", a->cache_filename);
|
||||
if (unlink(a->cache_filename) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n",
|
||||
a->cache_filename, errno, strerror(errno));
|
||||
}
|
||||
free(a->cache_filename);
|
||||
a->cache_filename = nullptr ;
|
||||
}
|
||||
}
|
||||
if (a->session_id) {
|
||||
switch_core_session_t* session = switch_core_session_locate(a->session_id);
|
||||
if (session) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
|
||||
/* unlock as quickly as possible */
|
||||
switch_core_session_rwunlock(session);
|
||||
if (channel) {
|
||||
switch_event_t *event;
|
||||
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_STOP) == SWITCH_STATUS_SUCCESS) {
|
||||
switch_channel_event_set_data(channel, event);
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_azure_response_code", std::to_string(a->response_code).c_str());
|
||||
if (a->cache_filename && download_complete) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", a->cache_filename);
|
||||
}
|
||||
if (!download_complete && a->err_msg) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_error", a->err_msg);
|
||||
}
|
||||
switch_event_fire(&event);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t azure_speech_close(azure_t* a) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "azure_speech_close\n") ;
|
||||
if (a->resampler) {
|
||||
speex_resampler_destroy(a->resampler);
|
||||
}
|
||||
|
||||
a->resampler = NULL;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t azure_speech_unload() {
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#ifndef __AZURE_TTS_GLUE_H__
|
||||
#define __AZURE_TTS_GLUE_H__
|
||||
|
||||
switch_status_t azure_speech_load();
|
||||
switch_status_t azure_speech_open(azure_t* azure);
|
||||
switch_status_t azure_speech_feed_tts(azure_t* azure, char* text, switch_speech_flag_t *flags);
|
||||
switch_status_t azure_speech_read_tts(azure_t* azure, void *data, size_t *datalen, switch_speech_flag_t *flags);
|
||||
switch_status_t azure_speech_flush_tts(azure_t* azure);
|
||||
switch_status_t azure_speech_close(azure_t* azure);
|
||||
switch_status_t azure_speech_unload();
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,173 @@
|
||||
#include "mod_azure_tts.h"
|
||||
#include "azure_glue.h"
|
||||
|
||||
SWITCH_MODULE_LOAD_FUNCTION(mod_azure_tts_load);
|
||||
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_azure_tts_shutdown);
|
||||
SWITCH_MODULE_DEFINITION(mod_azure_tts, mod_azure_tts_load, mod_azure_tts_shutdown, NULL);
|
||||
|
||||
static void clearAzure(azure_t* a, int freeAll) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "clearAzure\n");
|
||||
|
||||
if (a->cache_filename) free(a->cache_filename);
|
||||
if (a->api_key) free(a->api_key);
|
||||
if (a->language) free(a->language);
|
||||
if (a->region) free(a->region);
|
||||
if (a->endpoint) free(a->endpoint);
|
||||
if (a->endpointId) free(a->endpointId);
|
||||
if (a->err_msg) free(a->err_msg);
|
||||
if (a->http_proxy_ip) free(a->http_proxy_ip);
|
||||
if (a->http_proxy_port) free(a->http_proxy_port);
|
||||
|
||||
|
||||
a->cache_filename = NULL;
|
||||
a->api_key = NULL;
|
||||
a->language = NULL;
|
||||
a->region = NULL;
|
||||
a->endpoint = NULL;
|
||||
a->endpointId = NULL;
|
||||
a->err_msg = NULL;
|
||||
a->http_proxy_ip = NULL;
|
||||
a->http_proxy_port = NULL;
|
||||
|
||||
|
||||
if (freeAll) {
|
||||
if (a->voice_name) free(a->voice_name);
|
||||
if (a->session_id) free(a->session_id);
|
||||
a->voice_name = NULL;
|
||||
a->session_id = NULL;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static azure_t * createOrRetrievePrivateData(switch_speech_handle_t *sh) {
|
||||
azure_t *a = (azure_t *) sh->private_info;
|
||||
if (!a) {
|
||||
a = switch_core_alloc(sh->memory_pool, sizeof(*a));
|
||||
sh->private_info = a;
|
||||
memset(a, 0, sizeof(*a));
|
||||
switch_mutex_init(&a->mutex, SWITCH_MUTEX_NESTED, sh->memory_pool);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "allocated azure_t\n");
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
switch_status_t a_speech_open(switch_speech_handle_t *sh, const char *voice_name, int rate, int channels, switch_speech_flag_t *flags)
|
||||
{
|
||||
azure_t *a = createOrRetrievePrivateData(sh);
|
||||
a->voice_name = strdup(voice_name);
|
||||
a->rate = rate;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "a_speech_open voice: %s, rate %d, channels %d\n", voice_name, rate, channels);
|
||||
return azure_speech_open(a);
|
||||
}
|
||||
|
||||
static switch_status_t a_speech_close(switch_speech_handle_t *sh, switch_speech_flag_t *flags)
|
||||
{
|
||||
switch_status_t rc;
|
||||
azure_t *a = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "a_speech_close\n");
|
||||
|
||||
switch_mutex_destroy(a->mutex);
|
||||
|
||||
rc = azure_speech_close(a);
|
||||
clearAzure(a, 1);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeswitch will call this function to feed us text to speak
|
||||
*/
|
||||
static switch_status_t a_speech_feed_tts(switch_speech_handle_t *sh, char *text, switch_speech_flag_t *flags)
|
||||
{
|
||||
azure_t *a = createOrRetrievePrivateData(sh);
|
||||
a->draining = 0;
|
||||
a->reads = 0;
|
||||
a->flushed = 0;
|
||||
|
||||
return azure_speech_feed_tts(a, text, flags);
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeswitch calls periodically to get some rendered audio in L16 format. We can provide up to 8k of audio at a time.
|
||||
*/
|
||||
static switch_status_t a_speech_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, switch_speech_flag_t *flags)
|
||||
{
|
||||
azure_t *a = createOrRetrievePrivateData(sh);
|
||||
return azure_speech_read_tts(a, data, datalen, flags);
|
||||
}
|
||||
|
||||
/**
|
||||
* This is called at the end, not sure exactly what we need to do here..
|
||||
*/
|
||||
static void a_speech_flush_tts(switch_speech_handle_t *sh)
|
||||
{
|
||||
azure_t *a = createOrRetrievePrivateData(sh);
|
||||
azure_speech_flush_tts(a);
|
||||
|
||||
clearAzure(a, 0);
|
||||
}
|
||||
|
||||
static void a_text_param_tts(switch_speech_handle_t *sh, char *param, const char *val)
|
||||
{
|
||||
azure_t *a = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "a_text_param_tts: %s=%s\n", param, val);
|
||||
if (0 == strcmp(param, "api_key")) {
|
||||
if (a->api_key) free(a->api_key);
|
||||
a->api_key = strdup(val);
|
||||
} else if (0 == strcmp(param, "region")) {
|
||||
if (a->region) free(a->region);
|
||||
a->region = strdup(val);
|
||||
} else if (0 == strcmp(param, "voice")) {
|
||||
if (a->voice_name) free(a->voice_name);
|
||||
a->voice_name = strdup(val);
|
||||
} else if (0 == strcmp(param, "language")) {
|
||||
if (a->language) free(a->language);
|
||||
a->language = strdup(val);
|
||||
} else if (0 == strcmp(param, "endpoint")) {
|
||||
if (a->endpoint) free(a->endpoint);
|
||||
a->endpoint = strdup(val);
|
||||
} else if (0 == strcmp(param, "endpointId")) {
|
||||
if (a->endpointId) free(a->endpointId);
|
||||
a->endpointId = strdup(val);
|
||||
} else if (0 == strcmp(param, "http_proxy_ip")) {
|
||||
if (a->http_proxy_ip) free(a->http_proxy_ip);
|
||||
a->http_proxy_ip = strdup(val);
|
||||
} else if (0 == strcmp(param, "http_proxy_port")) {
|
||||
if (a->http_proxy_port) free(a->http_proxy_port);
|
||||
a->http_proxy_port = strdup(val);
|
||||
} else if (0 == strcmp(param, "session-uuid")) {
|
||||
if (a->session_id) free(a->session_id);
|
||||
a->session_id = strdup(val);
|
||||
} else if (0 == strcmp(param, "write_cache_file") && switch_true(val)) {
|
||||
a->cache_audio = 1;
|
||||
}
|
||||
}
|
||||
|
||||
static void a_numeric_param_tts(switch_speech_handle_t *sh, char *param, int val)
|
||||
{
|
||||
}
|
||||
static void a_float_param_tts(switch_speech_handle_t *sh, char *param, double val)
|
||||
{
|
||||
}
|
||||
|
||||
SWITCH_MODULE_LOAD_FUNCTION(mod_azure_tts_load)
|
||||
{
|
||||
switch_speech_interface_t *speech_interface;
|
||||
|
||||
*module_interface = switch_loadable_module_create_module_interface(pool, modname);
|
||||
speech_interface = switch_loadable_module_create_interface(*module_interface, SWITCH_SPEECH_INTERFACE);
|
||||
speech_interface->interface_name = "microsoft";
|
||||
speech_interface->speech_open = a_speech_open;
|
||||
speech_interface->speech_close = a_speech_close;
|
||||
speech_interface->speech_feed_tts = a_speech_feed_tts;
|
||||
speech_interface->speech_read_tts = a_speech_read_tts;
|
||||
speech_interface->speech_flush_tts = a_speech_flush_tts;
|
||||
speech_interface->speech_text_param_tts = a_text_param_tts;
|
||||
speech_interface->speech_numeric_param_tts = a_numeric_param_tts;
|
||||
speech_interface->speech_float_param_tts = a_float_param_tts;
|
||||
return azure_speech_load();
|
||||
}
|
||||
|
||||
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_azure_tts_shutdown)
|
||||
{
|
||||
return azure_speech_unload();
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
#ifndef __MOD_AZURE_TTS_H__
|
||||
#define __MOD_AZURE_TTS_H__
|
||||
|
||||
#include <switch.h>
|
||||
#include <speex/speex_resampler.h>
|
||||
|
||||
typedef struct azure_data {
|
||||
char *voice_name;
|
||||
char *api_key;
|
||||
char *region;
|
||||
char *language;
|
||||
char *endpoint;
|
||||
char *endpointId;
|
||||
char *http_proxy_ip;
|
||||
char *http_proxy_port;
|
||||
|
||||
/* result data */
|
||||
long response_code;
|
||||
char *session_id;
|
||||
char *cache_filename;
|
||||
char *err_msg;
|
||||
|
||||
int rate;
|
||||
int draining;
|
||||
int reads;
|
||||
int cache_audio;
|
||||
int flushed;
|
||||
|
||||
void *startTime;
|
||||
|
||||
FILE *file;
|
||||
SpeexResamplerState *resampler;
|
||||
void *circularBuffer;
|
||||
switch_mutex_t *mutex;
|
||||
} azure_t;
|
||||
|
||||
#endif
|
||||
@@ -25,10 +25,11 @@ Stop transcription on the channel.
|
||||
| variable | Description |
|
||||
| --- | ----------- |
|
||||
| DEEPGRAM_API_KEY | Deepgram API key used to authenticate |
|
||||
| DEEPGRAM_SPEECH_TIER | https://developers.deepgram.com/documentation/features/tier/ |
|
||||
| DEEPGRAM_SPEECH_CUSTOM_MODEL | custom model id |
|
||||
| DEEPGRAM_SPEECH_MODEL | https://developers.deepgram.com/documentation/features/model/ |
|
||||
| DEEPGRAM_SPEECH_MODEL_VERSION | https://developers.deepgram.com/documentation/features/version/ |
|
||||
| DEEPGRAM_SPEECH_ENABLE_SMART_FORMAT | https://developers.deepgram.com/docs/smart-format |
|
||||
| DEEPGRAM_SPEECH_ENABLE_FILLER_WORDS | https://developers.deepgram.com/docs/filler-words |
|
||||
| DEEPGRAM_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION | https://developers.deepgram.com/documentation/features/punctuate/ |
|
||||
| DEEPGRAM_SPEECH_PROFANITY_FILTER | https://developers.deepgram.com/documentation/features/profanity-filter/ |
|
||||
| DEEPGRAM_SPEECH_REDACT | https://developers.deepgram.com/documentation/features/redact/ |
|
||||
@@ -42,6 +43,7 @@ Stop transcription on the channel.
|
||||
| DEEPGRAM_SPEECH_REPLACE | https://developers.deepgram.com/documentation/features/replace/ |
|
||||
| DEEPGRAM_SPEECH_TAG | https://developers.deepgram.com/documentation/features/tag/ |
|
||||
| DEEPGRAM_SPEECH_ENDPOINTING | https://developers.deepgram.com/documentation/features/endpointing/ |
|
||||
| DEEPGRAM_SPEECH_UTTERANCE_END_MS | https://developers.deepgram.com/docs/utterance-end |
|
||||
| DEEPGRAM_SPEECH_VAD_TURNOFF | https://developers.deepgram.com/documentation/features/voice-activity-detection/ |
|
||||
|
||||
|
||||
|
||||
@@ -176,7 +176,9 @@ int AudioPipe::lws_callback(struct lws *wsi,
|
||||
if (lws_is_final_fragment(wsi)) {
|
||||
if (nullptr != ap->m_recv_buf) {
|
||||
std::string msg((char *)ap->m_recv_buf, ap->m_recv_buf_ptr - ap->m_recv_buf);
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), deepgram::AudioPipe::MESSAGE, msg.c_str(), ap->isFinished());
|
||||
if (!ap->m_silence_disconnect) {
|
||||
ap->m_callback(ap->m_uuid.c_str(), ap->m_bugname.c_str(), deepgram::AudioPipe::MESSAGE, msg.c_str(), ap->isFinished());
|
||||
}
|
||||
if (nullptr != ap->m_recv_buf) free(ap->m_recv_buf);
|
||||
}
|
||||
ap->m_recv_buf = ap->m_recv_buf_ptr = nullptr;
|
||||
@@ -451,7 +453,7 @@ AudioPipe::AudioPipe(const char* uuid, const char* bugname, const char* host, un
|
||||
m_uuid(uuid), m_host(host), m_port(port), m_path(path), m_finished(false), m_bugname(bugname),
|
||||
m_audio_buffer_min_freespace(minFreespace), m_audio_buffer_max_len(bufLen), m_gracefulShutdown(false),
|
||||
m_audio_buffer_write_offset(LWS_PRE), m_recv_buf(nullptr), m_recv_buf_ptr(nullptr), m_useTls(useTls),
|
||||
m_state(LWS_CLIENT_IDLE), m_wsi(nullptr), m_vhd(nullptr), m_callback(callback) {
|
||||
m_state(LWS_CLIENT_IDLE), m_wsi(nullptr), m_vhd(nullptr), m_callback(callback), m_silence_disconnect(false) {
|
||||
|
||||
if (apiKey) m_apiKey = apiKey;
|
||||
else m_apiKey = "";
|
||||
@@ -516,6 +518,11 @@ void AudioPipe::finish() {
|
||||
bufferForSending("{\"type\": \"CloseStream\"}");
|
||||
}
|
||||
|
||||
void AudioPipe::finish_in_silence() {
|
||||
m_silence_disconnect = true;
|
||||
finish();
|
||||
}
|
||||
|
||||
void AudioPipe::waitForClose() {
|
||||
std::shared_future<void> sf(m_promise.get_future());
|
||||
sf.wait();
|
||||
|
||||
@@ -76,6 +76,7 @@ namespace deepgram {
|
||||
|
||||
void close() ;
|
||||
void finish();
|
||||
void finish_in_silence();
|
||||
void waitForClose();
|
||||
void setClosed() { m_promise.set_value(); }
|
||||
bool isFinished() { return m_finished;}
|
||||
@@ -138,6 +139,7 @@ namespace deepgram {
|
||||
std::string m_bugname;
|
||||
std::promise<void> m_promise;
|
||||
bool m_useTls;
|
||||
bool m_silence_disconnect;
|
||||
};
|
||||
|
||||
} // namespace deepgram
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
|
||||
#define RTP_PACKETIZATION_PERIOD 20
|
||||
#define FRAME_SIZE_8000 320 /*which means each 20ms frame as 320 bytes at 8 khz (1 channel only)*/
|
||||
#define DEEPGRAM_KEEP_ALIVE_INTERVAL_SECOND 8
|
||||
|
||||
namespace {
|
||||
static bool hasDefaultCredentials = false;
|
||||
@@ -86,13 +87,17 @@ namespace {
|
||||
static const char* emptyTranscript =
|
||||
"\"is_final\":false,\"speech_final\":false,\"channel\":{\"alternatives\":[{\"transcript\":\"\",\"confidence\":0.0,\"words\":[]}]}";
|
||||
|
||||
static void reaper(private_t *tech_pvt) {
|
||||
static void reaper(private_t *tech_pvt, bool silence_disconnect) {
|
||||
std::shared_ptr<deepgram::AudioPipe> pAp;
|
||||
pAp.reset((deepgram::AudioPipe *)tech_pvt->pAudioPipe);
|
||||
tech_pvt->pAudioPipe = nullptr;
|
||||
|
||||
std::thread t([pAp, tech_pvt]{
|
||||
pAp->finish();
|
||||
std::thread t([pAp, tech_pvt, silence_disconnect]{
|
||||
if (silence_disconnect) {
|
||||
pAp->finish_in_silence();
|
||||
} else {
|
||||
pAp->finish();
|
||||
}
|
||||
pAp->waitForClose();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s (%u) got remote close\n", tech_pvt->sessionId, tech_pvt->id);
|
||||
});
|
||||
@@ -112,6 +117,8 @@ namespace {
|
||||
tech_pvt->resampler = NULL;
|
||||
}
|
||||
|
||||
// NB: do not destroy the mutex here, that is caller responsibility
|
||||
|
||||
/*
|
||||
if (tech_pvt->vad) {
|
||||
switch_vad_destroy(&tech_pvt->vad);
|
||||
@@ -184,6 +191,9 @@ namespace {
|
||||
*
|
||||
*/
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "DEEPGRAM_SPEECH_ENABLE_FILLER_WORDS")) {
|
||||
oss << "&filler_words=true";
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "DEEPGRAM_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION")) {
|
||||
oss << "&punctuate=true";
|
||||
}
|
||||
@@ -292,9 +302,20 @@ namespace {
|
||||
break;
|
||||
case deepgram::AudioPipe::CONNECTION_DROPPED:
|
||||
// first thing: we can no longer access the AudioPipe
|
||||
tech_pvt->pAudioPipe = nullptr;
|
||||
tech_pvt->responseHandler(session, TRANSCRIBE_EVENT_DISCONNECT, NULL, tech_pvt->bugname, finished);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connection (%s) dropped from far end\n", tech_pvt->bugname);
|
||||
|
||||
/**
|
||||
* this is a bit tricky. If we just closed a previos connection it may be returning final transcripts
|
||||
* and then a close event here as it is shutting down (in the reaper thread above).
|
||||
* In this scenario, the fact that the connection is dropped is not significant.
|
||||
*/
|
||||
if (finished) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "old connection (%s) gracefully closed by Deepgram\n", tech_pvt->bugname);
|
||||
}
|
||||
else {
|
||||
tech_pvt->pAudioPipe = nullptr;
|
||||
tech_pvt->responseHandler(session, TRANSCRIBE_EVENT_DISCONNECT, NULL, tech_pvt->bugname, finished);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connection (%s) dropped from far end\n", tech_pvt->bugname);
|
||||
}
|
||||
break;
|
||||
case deepgram::AudioPipe::CONNECTION_CLOSED_GRACEFULLY:
|
||||
// first thing: we can no longer access the AudioPipe
|
||||
@@ -326,60 +347,82 @@ namespace {
|
||||
|
||||
int err;
|
||||
int useTls = true;
|
||||
std::string host;
|
||||
int port = 443;
|
||||
size_t buflen = LWS_PRE + (FRAME_SIZE_8000 * desiredSampling / 8000 * channels * 1000 / RTP_PACKETIZATION_PERIOD * nAudioBufferSecs);
|
||||
|
||||
std::ostringstream configuration_stream;
|
||||
switch_codec_implementation_t read_impl;
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
|
||||
switch_core_session_get_read_impl(session, &read_impl);
|
||||
|
||||
memset(tech_pvt, 0, sizeof(private_t));
|
||||
|
||||
std::string path;
|
||||
constructPath(session, path, desiredSampling, channels, lang, interim);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "path: %s\n", path.c_str());
|
||||
|
||||
strncpy(tech_pvt->sessionId, switch_core_session_get_uuid(session), MAX_SESSION_ID);
|
||||
|
||||
const char* endpoint = switch_channel_get_variable(channel, "DEEPGRAM_URI");
|
||||
if (endpoint != nullptr) {
|
||||
std::string ep(endpoint);
|
||||
|
||||
useTls = switch_true(switch_channel_get_variable(channel, "DEEPGRAM_USE_TLS"));
|
||||
|
||||
size_t pos = ep.find(':');
|
||||
host = ep;
|
||||
if (pos != std::string::npos) {
|
||||
std::string host = ep.substr(0, pos);
|
||||
std::string port = ep.substr(pos + 1);
|
||||
strncpy(tech_pvt->host, host.c_str(), MAX_WS_URL_LEN);
|
||||
tech_pvt->port = ::atoi(port.c_str());
|
||||
}
|
||||
else {
|
||||
strncpy(tech_pvt->host, ep.c_str(), MAX_WS_URL_LEN);
|
||||
tech_pvt->port = 443;
|
||||
host = ep.substr(0, pos);
|
||||
std::string strPort = ep.substr(pos + 1);
|
||||
port = ::atoi(strPort.c_str());
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO,
|
||||
"connecting to deepgram on-prem %s port %d, using tls? (%s)\n", tech_pvt->host, tech_pvt->port, useTls ? "yes" : "no");
|
||||
}
|
||||
else {
|
||||
strncpy(tech_pvt->host, "api.deepgram.com", MAX_WS_URL_LEN);
|
||||
tech_pvt->port = 443;
|
||||
"connecting to deepgram on-prem %s port %d, using tls? (%s)\n", host.c_str(), port, useTls ? "yes" : "no");
|
||||
} else {
|
||||
host = "api.deepgram.com";
|
||||
}
|
||||
|
||||
strncpy(tech_pvt->path, path.c_str(), MAX_PATH_LEN);
|
||||
const char* apiKey = switch_channel_get_variable(channel, "DEEPGRAM_API_KEY");
|
||||
if (!apiKey && defaultApiKey) {
|
||||
apiKey = defaultApiKey;
|
||||
} else if (!apiKey && endpoint == nullptr) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "no deepgram api key provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
configuration_stream <<
|
||||
host << ":" <<
|
||||
port << ";" <<
|
||||
path << ";" <<
|
||||
buflen << ";" <<
|
||||
read_impl.decoded_bytes_per_packet << ";" <<
|
||||
apiKey << ";" <<
|
||||
useTls;
|
||||
|
||||
if (tech_pvt->pAudioPipe) {
|
||||
// stop sending keep alive
|
||||
tech_pvt->is_keep_alive = 0;
|
||||
if (0 != strcmp(tech_pvt->configuration, configuration_stream.str().c_str())) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "fork_data_init: stop existing deepgram connection, old configuration %s, new configuration %s\n",
|
||||
tech_pvt->configuration, configuration_stream.str().c_str());
|
||||
reaper(tech_pvt, true);
|
||||
} else {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "fork_data_init: enable existing deepgram connection\n");
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
} else {
|
||||
memset(tech_pvt, 0, sizeof(private_t));
|
||||
}
|
||||
|
||||
strncpy(tech_pvt->sessionId, switch_core_session_get_uuid(session), MAX_SESSION_ID);
|
||||
strncpy(tech_pvt->host, host.c_str(), MAX_WS_URL_LEN);
|
||||
tech_pvt->port = port;
|
||||
strncpy(tech_pvt->path, path.c_str(), MAX_PATH_LEN);
|
||||
strncpy(tech_pvt->configuration, configuration_stream.str().c_str(), MAX_PATH_LEN) ;
|
||||
tech_pvt->sampling = desiredSampling;
|
||||
tech_pvt->responseHandler = responseHandler;
|
||||
tech_pvt->channels = channels;
|
||||
tech_pvt->id = ++idxCallCount;
|
||||
tech_pvt->buffer_overrun_notified = 0;
|
||||
tech_pvt->is_keep_alive = 0;
|
||||
strncpy(tech_pvt->bugname, bugname, MAX_BUG_LEN);
|
||||
|
||||
size_t buflen = LWS_PRE + (FRAME_SIZE_8000 * desiredSampling / 8000 * channels * 1000 / RTP_PACKETIZATION_PERIOD * nAudioBufferSecs);
|
||||
|
||||
const char* apiKey = switch_channel_get_variable(channel, "DEEPGRAM_API_KEY");
|
||||
if (!apiKey && defaultApiKey) apiKey = defaultApiKey;
|
||||
else if (!apiKey && endpoint == nullptr) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "no deepgram api key provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
deepgram::AudioPipe* ap = new deepgram::AudioPipe(tech_pvt->sessionId, bugname, tech_pvt->host, tech_pvt->port, tech_pvt->path,
|
||||
buflen, read_impl.decoded_bytes_per_packet, apiKey, useTls, eventCallback);
|
||||
@@ -390,18 +433,26 @@ namespace {
|
||||
|
||||
tech_pvt->pAudioPipe = static_cast<void *>(ap);
|
||||
|
||||
switch_mutex_init(&tech_pvt->mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session));
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connecting now\n");
|
||||
ap->connect();
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connection in progress\n");
|
||||
|
||||
if (desiredSampling != sampling) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) resampling from %u to %u\n", tech_pvt->id, sampling, desiredSampling);
|
||||
tech_pvt->resampler = speex_resampler_init(channels, sampling, desiredSampling, SWITCH_RESAMPLE_QUALITY, &err);
|
||||
if (0 != err) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error initializing resampler: %s.\n", speex_resampler_strerror(err));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (!tech_pvt->mutex) {
|
||||
switch_mutex_init(&tech_pvt->mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session));
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) no resampling needed for this call\n", tech_pvt->id);
|
||||
|
||||
if (!tech_pvt->resampler) {
|
||||
if (desiredSampling != sampling) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) resampling from %u to %u\n", tech_pvt->id, sampling, desiredSampling);
|
||||
tech_pvt->resampler = speex_resampler_init(channels, sampling, desiredSampling, SWITCH_RESAMPLE_QUALITY, &err);
|
||||
if (0 != err) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error initializing resampler: %s.\n", speex_resampler_strerror(err));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) no resampling needed for this call\n", tech_pvt->id);
|
||||
}
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) fork_data_init\n", tech_pvt->id);
|
||||
@@ -458,11 +509,22 @@ extern "C" {
|
||||
switch_status_t dg_transcribe_session_init(switch_core_session_t *session,
|
||||
responseHandler_t responseHandler, uint32_t samples_per_second, uint32_t channels,
|
||||
char* lang, int interim, char* bugname, void **ppUserData)
|
||||
{
|
||||
{
|
||||
int err;
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, bugname);
|
||||
private_t* tech_pvt;
|
||||
if (bug) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "reuse existing kep alive deepgram connection\n");
|
||||
tech_pvt = (private_t*) switch_core_media_bug_get_user_data(bug);
|
||||
} else {
|
||||
tech_pvt = (private_t *) switch_core_session_alloc(session, sizeof(private_t));
|
||||
tech_pvt->pAudioPipe = NULL;
|
||||
tech_pvt->is_keep_alive = 0;
|
||||
tech_pvt->mutex = NULL;
|
||||
tech_pvt->resampler = NULL;
|
||||
}
|
||||
|
||||
// allocate per-session data structure
|
||||
private_t* tech_pvt = (private_t *) switch_core_session_alloc(session, sizeof(private_t));
|
||||
if (!tech_pvt) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "error allocating memory!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
@@ -475,16 +537,13 @@ extern "C" {
|
||||
|
||||
*ppUserData = tech_pvt;
|
||||
|
||||
deepgram::AudioPipe *pAudioPipe = static_cast<deepgram::AudioPipe *>(tech_pvt->pAudioPipe);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connecting now\n");
|
||||
pAudioPipe->connect();
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "connection in progress\n");
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t dg_transcribe_session_stop(switch_core_session_t *session,int channelIsClosing, char* bugname) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, bugname);
|
||||
const bool use_single_connection = switch_true(std::getenv("DEEPGRAM_SPEECH_USE_SINGLE_CONNECTION"));
|
||||
if (!bug) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "dg_transcribe_session_stop: no bug - websocket conection already closed\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
@@ -495,6 +554,12 @@ extern "C" {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "(%u) dg_transcribe_session_stop\n", id);
|
||||
|
||||
if (!tech_pvt) return SWITCH_STATUS_FALSE;
|
||||
if (use_single_connection && !channelIsClosing) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "dg_transcribe_session_stop: call is running, use_single_connection is true, keep alive is activated\n", id);
|
||||
tech_pvt->is_keep_alive = 1;
|
||||
tech_pvt->frame_count = 0;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// close connection and get final responses
|
||||
switch_mutex_lock(tech_pvt->mutex);
|
||||
@@ -502,7 +567,7 @@ extern "C" {
|
||||
if (!channelIsClosing) switch_core_media_bug_remove(session, &bug);
|
||||
|
||||
deepgram::AudioPipe *pAudioPipe = static_cast<deepgram::AudioPipe *>(tech_pvt->pAudioPipe);
|
||||
if (pAudioPipe) reaper(tech_pvt);
|
||||
if (pAudioPipe) reaper(tech_pvt, false);
|
||||
destroy_tech_pvt(tech_pvt);
|
||||
switch_mutex_unlock(tech_pvt->mutex);
|
||||
switch_mutex_destroy(tech_pvt->mutex);
|
||||
@@ -516,9 +581,31 @@ extern "C" {
|
||||
size_t inuse = 0;
|
||||
bool dirty = false;
|
||||
char *p = (char *) "{\"msg\": \"buffer overrun\"}";
|
||||
char *keep_alive = (char *) "{\"type\": \"KeepAlive\"}";
|
||||
|
||||
if (!tech_pvt) return SWITCH_TRUE;
|
||||
|
||||
|
||||
// Keep sending keep alive if there is no transcribe activity
|
||||
if (tech_pvt->is_keep_alive && tech_pvt->pAudioPipe) {
|
||||
deepgram::AudioPipe *pAudioPipe = static_cast<deepgram::AudioPipe *>(tech_pvt->pAudioPipe);
|
||||
if (++tech_pvt->frame_count * 20 /*ms*/ / 1000 >= DEEPGRAM_KEEP_ALIVE_INTERVAL_SECOND) {
|
||||
tech_pvt->frame_count = 0;
|
||||
pAudioPipe->bufferForSending(keep_alive);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "dg_transcribe_frame: sending %s to deepgram\n", keep_alive);
|
||||
}
|
||||
// remove media bug buffered data
|
||||
while (true) {
|
||||
unsigned char data[SWITCH_RECOMMENDED_BUFFER_SIZE] = {0};
|
||||
switch_frame_t frame = { 0 };
|
||||
frame.data = data;
|
||||
frame.buflen = SWITCH_RECOMMENDED_BUFFER_SIZE;
|
||||
switch_status_t rv = switch_core_media_bug_read(bug, &frame, SWITCH_TRUE);
|
||||
if (rv != SWITCH_STATUS_SUCCESS) break;
|
||||
}
|
||||
return SWITCH_TRUE;
|
||||
}
|
||||
|
||||
if (switch_mutex_trylock(tech_pvt->mutex) == SWITCH_STATUS_SUCCESS) {
|
||||
if (!tech_pvt->pAudioPipe) {
|
||||
switch_mutex_unlock(tech_pvt->mutex);
|
||||
@@ -529,7 +616,6 @@ extern "C" {
|
||||
switch_mutex_unlock(tech_pvt->mutex);
|
||||
return SWITCH_TRUE;
|
||||
}
|
||||
|
||||
pAudioPipe->lockAudioBuffer();
|
||||
size_t available = pAudioPipe->binarySpaceAvailable();
|
||||
if (NULL == tech_pvt->resampler) {
|
||||
@@ -571,7 +657,6 @@ extern "C" {
|
||||
if (frame.datalen) {
|
||||
spx_uint32_t out_len = available >> 1; // space for samples which are 2 bytes
|
||||
spx_uint32_t in_len = frame.samples;
|
||||
|
||||
speex_resampler_process_interleaved_int(tech_pvt->resampler,
|
||||
(const spx_int16_t *) frame.data,
|
||||
(spx_uint32_t *) &in_len,
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
*/
|
||||
#include "mod_deepgram_transcribe.h"
|
||||
#include "dg_transcribe_glue.h"
|
||||
#include <stdio.h>
|
||||
|
||||
/* Prototypes */
|
||||
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_deepgram_transcribe_shutdown);
|
||||
@@ -72,8 +73,10 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
|
||||
switch_codec_implementation_t read_impl = { 0 };
|
||||
void *pUserData;
|
||||
uint32_t samples_per_second;
|
||||
int use_single_connection = switch_true(getenv("DEEPGRAM_SPEECH_USE_SINGLE_CONNECTION"));
|
||||
bug = switch_channel_get_private(channel, bugname);
|
||||
|
||||
if (switch_channel_get_private(channel, bugname)) {
|
||||
if (bug && !use_single_connection) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing bug from previous transcribe\n");
|
||||
do_stop(session, bugname);
|
||||
}
|
||||
@@ -90,11 +93,13 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing dg speech session.\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if ((status = switch_core_media_bug_add(session, "dg_transcribe", NULL, capture_callback, pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
|
||||
return status;
|
||||
if (!bug || !use_single_connection) {
|
||||
if ((status = switch_core_media_bug_add(session, "dg_transcribe", NULL, capture_callback, pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
|
||||
return status;
|
||||
}
|
||||
switch_channel_set_private(channel, bugname, bug);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "added media bug for dg transcribe\n");
|
||||
}
|
||||
switch_channel_set_private(channel, bugname, bug);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "added media bug for dg transcribe\n");
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
@@ -39,6 +39,9 @@ struct private_data {
|
||||
unsigned int id;
|
||||
int buffer_overrun_notified:1;
|
||||
int is_finished:1;
|
||||
int is_keep_alive;
|
||||
int frame_count;
|
||||
char configuration[MAX_PATH_LEN];
|
||||
};
|
||||
|
||||
typedef struct private_data private_t;
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
include $(top_srcdir)/build/modmake.rulesam
|
||||
MODNAME=mod_deepgram_tts
|
||||
|
||||
mod_LTLIBRARIES = mod_deepgram_tts.la
|
||||
mod_deepgram_tts_la_SOURCES = mod_deepgram_tts.c deepgram_glue.cpp
|
||||
mod_deepgram_tts_la_CFLAGS = $(AM_CFLAGS)
|
||||
mod_deepgram_tts_la_LIBADD = $(switch_builddir)/libfreeswitch.la
|
||||
mod_deepgram_tts_la_LDFLAGS = -avoid-version -module -no-undefined -shared -lstdc++ -lboost_system -lboost_thread
|
||||
@@ -0,0 +1,954 @@
|
||||
#include "mod_deepgram_tts.h"
|
||||
#include <switch.h>
|
||||
#include <switch_json.h>
|
||||
#include <curl/curl.h>
|
||||
#include <cstdlib>
|
||||
|
||||
#include <boost/circular_buffer.hpp>
|
||||
#include <boost/thread.hpp>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/asio/ssl.hpp>
|
||||
#include <boost/pool/object_pool.hpp>
|
||||
#include <boost/bind/bind.hpp>
|
||||
#include <boost/tokenizer.hpp>
|
||||
#include <boost/foreach.hpp>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/assign/list_of.hpp>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
|
||||
#include <speex/speex_resampler.h>
|
||||
|
||||
#define BUFFER_GROW_SIZE (80000)
|
||||
|
||||
typedef boost::circular_buffer<uint16_t> CircularBuffer_t;
|
||||
/* Global information, common to all connections */
|
||||
typedef struct
|
||||
{
|
||||
CURLM *multi;
|
||||
int still_running;
|
||||
} GlobalInfo_t;
|
||||
static GlobalInfo_t global;
|
||||
|
||||
/* Information associated with a specific easy handle */
|
||||
typedef struct
|
||||
{
|
||||
CURL *easy;
|
||||
deepgram_t* deepgram;
|
||||
char* body;
|
||||
struct curl_slist *hdr_list;
|
||||
GlobalInfo_t *global;
|
||||
char error[CURL_ERROR_SIZE];
|
||||
FILE* file;
|
||||
std::chrono::time_point<std::chrono::high_resolution_clock> startTime;
|
||||
bool flushed;
|
||||
|
||||
bool has_last_byte;
|
||||
uint8_t last_byte;
|
||||
} ConnInfo_t;
|
||||
|
||||
|
||||
static boost::object_pool<ConnInfo_t> pool ;
|
||||
static std::map<curl_socket_t, boost::asio::ip::tcp::socket *> socket_map;
|
||||
static boost::asio::io_service io_service;
|
||||
static boost::asio::deadline_timer timer(io_service);
|
||||
static std::string fullDirPath;
|
||||
static std::thread worker_thread;
|
||||
|
||||
std::string secondsToMillisecondsString(double seconds) {
|
||||
// Convert to milliseconds
|
||||
double milliseconds = seconds * 1000.0;
|
||||
|
||||
// Truncate to remove fractional part
|
||||
long milliseconds_long = static_cast<long>(milliseconds);
|
||||
|
||||
// Convert to string
|
||||
return std::to_string(milliseconds_long);
|
||||
}
|
||||
|
||||
static CURL* createEasyHandle(void) {
|
||||
CURL* easy = curl_easy_init();
|
||||
if(!easy) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "curl_easy_init() failed!\n");
|
||||
return nullptr ;
|
||||
}
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(easy, CURLOPT_USERAGENT, "jambonz/0.8.5");
|
||||
|
||||
// set connect timeout to 3 seconds and total timeout to 109 seconds
|
||||
curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
|
||||
curl_easy_setopt(easy, CURLOPT_TIMEOUT, 10L);
|
||||
|
||||
return easy ;
|
||||
}
|
||||
|
||||
static void cleanupConn(ConnInfo_t *conn) {
|
||||
auto d = conn->deepgram;
|
||||
|
||||
if( conn->hdr_list ) {
|
||||
curl_slist_free_all(conn->hdr_list);
|
||||
conn->hdr_list = nullptr ;
|
||||
}
|
||||
curl_easy_cleanup(conn->easy);
|
||||
|
||||
if (conn->file) {
|
||||
if (fclose(conn->file) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error closing audio cache file\n");
|
||||
}
|
||||
conn->file = nullptr ;
|
||||
}
|
||||
|
||||
d->conn = nullptr ;
|
||||
d->draining = 1;
|
||||
|
||||
memset(conn, 0, sizeof(ConnInfo_t));
|
||||
pool.destroy(conn) ;
|
||||
}
|
||||
|
||||
/* Check for completed transfers, and remove their easy handles */
|
||||
void check_multi_info(GlobalInfo_t *g) {
|
||||
CURLMsg *msg;
|
||||
int msgs_left;
|
||||
ConnInfo_t *conn;
|
||||
CURL *easy;
|
||||
CURLcode res;
|
||||
|
||||
while((msg = curl_multi_info_read(g->multi, &msgs_left))) {
|
||||
if(msg->msg == CURLMSG_DONE) {
|
||||
long response_code;
|
||||
double namelookup=0, connect=0, total=0 ;
|
||||
char *ct = NULL ;
|
||||
|
||||
easy = msg->easy_handle;
|
||||
res = msg->data.result;
|
||||
curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
|
||||
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &response_code);
|
||||
curl_easy_getinfo(easy, CURLINFO_CONTENT_TYPE, &ct);
|
||||
|
||||
curl_easy_getinfo(easy, CURLINFO_NAMELOOKUP_TIME, &namelookup);
|
||||
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connect);
|
||||
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &total);
|
||||
|
||||
auto d = conn->deepgram;
|
||||
d->response_code = response_code;
|
||||
if (ct) d->ct = strdup(ct);
|
||||
|
||||
std::string name_lookup_ms = secondsToMillisecondsString(namelookup);
|
||||
std::string connect_ms = secondsToMillisecondsString(connect);
|
||||
std::string final_response_time_ms = secondsToMillisecondsString(total);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,
|
||||
"mod_deepgram_tts: response: %ld, content-type %s,"
|
||||
"dns(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld, "
|
||||
"connect(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld, "
|
||||
"total(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld\n",
|
||||
response_code, ct,
|
||||
(long)(namelookup), (long)(fmod(namelookup, 1.0) * 1000000),
|
||||
(long)(connect), (long)(fmod(connect, 1.0) * 1000000),
|
||||
(long)(total), (long)(fmod(total, 1.0) * 1000000));
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "name lookup time: %s\n", name_lookup_ms.c_str());
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "connect time: %s\n", connect_ms.c_str());
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "final response time: %s\n", final_response_time_ms.c_str());
|
||||
|
||||
d->name_lookup_time_ms = strdup(name_lookup_ms.c_str());
|
||||
d->connect_time_ms = strdup(connect_ms.c_str());
|
||||
d->final_response_time_ms = strdup(final_response_time_ms.c_str());
|
||||
|
||||
curl_multi_remove_handle(g->multi, easy);
|
||||
cleanupConn(conn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int mcode_test(const char *where, CURLMcode code) {
|
||||
if(CURLM_OK != code) {
|
||||
const char *s;
|
||||
switch(code) {
|
||||
case CURLM_CALL_MULTI_PERFORM:
|
||||
s = "CURLM_CALL_MULTI_PERFORM";
|
||||
break;
|
||||
case CURLM_BAD_HANDLE:
|
||||
s = "CURLM_BAD_HANDLE";
|
||||
break;
|
||||
case CURLM_BAD_EASY_HANDLE:
|
||||
s = "CURLM_BAD_EASY_HANDLE";
|
||||
break;
|
||||
case CURLM_OUT_OF_MEMORY:
|
||||
s = "CURLM_OUT_OF_MEMORY";
|
||||
break;
|
||||
case CURLM_INTERNAL_ERROR:
|
||||
s = "CURLM_INTERNAL_ERROR";
|
||||
break;
|
||||
case CURLM_UNKNOWN_OPTION:
|
||||
s = "CURLM_UNKNOWN_OPTION";
|
||||
break;
|
||||
case CURLM_LAST:
|
||||
s = "CURLM_LAST";
|
||||
break;
|
||||
default:
|
||||
s = "CURLM_unknown";
|
||||
break;
|
||||
case CURLM_BAD_SOCKET:
|
||||
s = "CURLM_BAD_SOCKET";
|
||||
break;
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mcode_test ERROR: %s returns %s:%d\n", where, s, code);
|
||||
|
||||
return -1;
|
||||
}
|
||||
return 0 ;
|
||||
}
|
||||
|
||||
static void remsock(int *f, GlobalInfo_t *g) {
|
||||
if(f) {
|
||||
free(f);
|
||||
f = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/* Called by asio when there is an action on a socket */
|
||||
static void event_cb(GlobalInfo_t *g, curl_socket_t s, int action, const boost::system::error_code & error, int *fdp) {
|
||||
int f = *fdp;
|
||||
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X has action %d\n", s, action) ;
|
||||
|
||||
// Socket already POOL REMOVED.
|
||||
if (f == CURL_POLL_REMOVE) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X removed\n", s);
|
||||
remsock(fdp, g);
|
||||
return;
|
||||
}
|
||||
|
||||
if(socket_map.find(s) == socket_map.end()) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb: socket %#X already closed\n, s");
|
||||
return;
|
||||
}
|
||||
|
||||
/* make sure the event matches what are wanted */
|
||||
if(f == action || f == CURL_POLL_INOUT) {
|
||||
if(error) {
|
||||
action = CURL_CSELECT_ERR;
|
||||
}
|
||||
CURLMcode rc = curl_multi_socket_action(g->multi, s, action, &g->still_running);
|
||||
|
||||
mcode_test("event_cb: curl_multi_socket_action", rc);
|
||||
check_multi_info(g);
|
||||
|
||||
if(g->still_running <= 0) {
|
||||
timer.cancel();
|
||||
}
|
||||
|
||||
/* keep on watching.
|
||||
* the socket may have been closed and/or fdp may have been changed
|
||||
* in curl_multi_socket_action(), so check them both */
|
||||
if(!error && socket_map.find(s) != socket_map.end() &&
|
||||
(f == action || f == CURL_POLL_INOUT)) {
|
||||
boost::asio::ip::tcp::socket *tcp_socket = socket_map.find(s)->second;
|
||||
|
||||
if(action == CURL_POLL_IN) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
action, boost::placeholders::_1, fdp));
|
||||
}
|
||||
if(action == CURL_POLL_OUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
action, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* socket functions */
|
||||
static void setsock(int *fdp, curl_socket_t s, CURL *e, int act, int oldact, GlobalInfo_t *g) {
|
||||
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(s);
|
||||
|
||||
if(it == socket_map.end()) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setsock: socket %#X not found\n, s");
|
||||
return;
|
||||
}
|
||||
|
||||
boost::asio::ip::tcp::socket * tcp_socket = it->second;
|
||||
|
||||
*fdp = act;
|
||||
|
||||
if(act == CURL_POLL_IN) {
|
||||
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_IN, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
else if(act == CURL_POLL_OUT) {
|
||||
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_OUT, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
else if(act == CURL_POLL_INOUT) {
|
||||
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_IN, boost::placeholders::_1, fdp));
|
||||
}
|
||||
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_OUT, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo_t *g) {
|
||||
/* fdp is used to store current action */
|
||||
int *fdp = (int *) calloc(sizeof(int), 1);
|
||||
|
||||
setsock(fdp, s, easy, action, 0, g);
|
||||
curl_multi_assign(g->multi, s, fdp);
|
||||
}
|
||||
|
||||
static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) {
|
||||
GlobalInfo_t *g = &global;
|
||||
|
||||
int *actionp = (int *) sockp;
|
||||
static const char *whatstr[] = { "none", "IN", "OUT", "INOUT", "REMOVE"};
|
||||
|
||||
if(what == CURL_POLL_REMOVE) {
|
||||
*actionp = what;
|
||||
}
|
||||
else {
|
||||
if(!actionp) {
|
||||
addsock(s, e, what, g);
|
||||
}
|
||||
else {
|
||||
setsock(actionp, s, e, what, *actionp, g);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void threadFunc() {
|
||||
/* to make sure the event loop doesn't terminate when there is no work to do */
|
||||
io_service.reset() ;
|
||||
boost::asio::io_service::work work(io_service);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_deepgram_tts threadFunc - starting\n");
|
||||
|
||||
for(;;) {
|
||||
|
||||
try {
|
||||
io_service.run() ;
|
||||
break ;
|
||||
}
|
||||
catch( std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mod_deepgram_tts threadFunc - Error: %s\n", e.what());
|
||||
}
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_deepgram_tts threadFunc - ending\n");
|
||||
}
|
||||
|
||||
|
||||
/* Called by asio when our timeout expires */
|
||||
static void timer_cb(const boost::system::error_code & error, GlobalInfo_t *g)
|
||||
{
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "timer_cb\n");
|
||||
|
||||
if(!error) {
|
||||
CURLMcode rc = curl_multi_socket_action(g->multi, CURL_SOCKET_TIMEOUT, 0, &g->still_running);
|
||||
mcode_test("timer_cb: curl_multi_socket_action", rc);
|
||||
check_multi_info(g);
|
||||
}
|
||||
}
|
||||
|
||||
int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo_t *g) {
|
||||
|
||||
/* cancel running timer */
|
||||
timer.cancel();
|
||||
|
||||
if(timeout_ms >= 0) {
|
||||
// from libcurl 7.88.1-10+deb12u4 does not allow call curl_multi_socket_action or curl_multi_perform in curl_multi callback directly
|
||||
timer.expires_from_now(boost::posix_time::millisec(timeout_ms ? timeout_ms : 1));
|
||||
timer.async_wait(boost::bind(&timer_cb, boost::placeholders::_1, g));
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* CURLOPT_WRITEFUNCTION */
|
||||
static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
|
||||
bool fireEvent = false;
|
||||
uint8_t *data = (uint8_t *) ptr;
|
||||
size_t bytes_received = size * nmemb;
|
||||
size_t total_bytes_to_process;
|
||||
auto d = conn->deepgram;
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) d->circularBuffer;
|
||||
|
||||
if (conn->flushed || cBuffer == nullptr) {
|
||||
/* this will abort the transfer */
|
||||
return 0;
|
||||
}
|
||||
// Buffer to hold combined data if there is unprocessed byte from the last call.
|
||||
std::unique_ptr<uint8_t[]> combinedData;
|
||||
|
||||
if (conn->has_last_byte) {
|
||||
conn->has_last_byte = false; // We'll handle the last_byte now, so toggle the flag off
|
||||
|
||||
// Allocate memory for the new data array
|
||||
combinedData.reset(new uint8_t[bytes_received + 1]);
|
||||
|
||||
// Prepend the last byte from previous call
|
||||
combinedData[0] = conn->last_byte;
|
||||
|
||||
// Copy the new data following the prepended byte
|
||||
memcpy(combinedData.get() + 1, data, bytes_received);
|
||||
|
||||
// Point our data pointer to the new array
|
||||
data = combinedData.get();
|
||||
|
||||
total_bytes_to_process = bytes_received + 1;
|
||||
} else {
|
||||
total_bytes_to_process = bytes_received;
|
||||
}
|
||||
|
||||
// If we now have an odd total, save the last byte for next time
|
||||
if ((total_bytes_to_process % sizeof(int16_t)) != 0) {
|
||||
conn->last_byte = data[total_bytes_to_process - 1];
|
||||
conn->has_last_byte = true;
|
||||
total_bytes_to_process--;
|
||||
}
|
||||
|
||||
int16_t* inputData = reinterpret_cast<int16_t*>(data);
|
||||
if (0 == d->reads++) {
|
||||
fireEvent = true;
|
||||
// Deepgram return PCM linear16 WAV file which contains 44 bytes headers, remove that.
|
||||
inputData += 22;
|
||||
total_bytes_to_process -= 44;
|
||||
}
|
||||
size_t numSamples = total_bytes_to_process / sizeof(int16_t);
|
||||
{
|
||||
switch_mutex_lock(d->mutex);
|
||||
|
||||
if (d->response_code > 0 && d->response_code != 200) {
|
||||
std::string body((char *) ptr, bytes_received);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: received body %s\n", body.c_str());
|
||||
d->err_msg = strdup(body.c_str());
|
||||
switch_mutex_unlock(d->mutex);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* cache file will stay in the mp3 format for size (smaller) and simplicity */
|
||||
if (conn->file) fwrite(inputData, sizeof(int16_t), numSamples, conn->file);
|
||||
|
||||
// Resize the buffer if necessary
|
||||
if (cBuffer->capacity() - cBuffer->size() < numSamples) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "write_cb growing buffer\n");
|
||||
|
||||
//TODO: if buffer exceeds some max size, return CURL_WRITEFUNC_ERROR to abort the transfer
|
||||
cBuffer->set_capacity(cBuffer->size() + std::max(numSamples, (size_t)BUFFER_GROW_SIZE));
|
||||
}
|
||||
|
||||
/* Push the data into the buffer */
|
||||
cBuffer->insert(cBuffer->end(), inputData, inputData + numSamples);
|
||||
|
||||
switch_mutex_unlock(d->mutex);
|
||||
}
|
||||
if (fireEvent && d->session_id) {
|
||||
auto endTime = std::chrono::high_resolution_clock::now();
|
||||
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(endTime - conn->startTime);
|
||||
auto time_to_first_byte_ms = std::to_string(duration.count());
|
||||
switch_core_session_t* session = switch_core_session_locate(d->session_id);
|
||||
if (session) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_core_session_rwunlock(session);
|
||||
if (channel) {
|
||||
switch_event_t *event;
|
||||
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_START) == SWITCH_STATUS_SUCCESS) {
|
||||
switch_channel_event_set_data(channel, event);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: firing playback-started\n");
|
||||
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
|
||||
if (d->reported_model_name) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_reported_model_name", d->reported_model_name);
|
||||
}
|
||||
if (d->reported_model_uuid) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_reported_model_uuid", d->reported_model_uuid);
|
||||
}
|
||||
if (d->reported_char_count) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_reported_char_count", d->reported_char_count);
|
||||
}
|
||||
if (d->request_id) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_request_id", d->request_id);
|
||||
}
|
||||
if (d->name_lookup_time_ms) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_name_lookup_time_ms", d->name_lookup_time_ms);
|
||||
}
|
||||
if (d->connect_time_ms) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_connect_time_ms", d->connect_time_ms);
|
||||
}
|
||||
if (d->final_response_time_ms) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_final_response_time_ms", d->final_response_time_ms);
|
||||
}
|
||||
if (d->voice_name) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_voice_name", d->voice_name);
|
||||
}
|
||||
if (d->cache_filename) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", d->cache_filename);
|
||||
}
|
||||
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
|
||||
switch_event_fire(&event);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: session %s not found\n", d->session_id);
|
||||
}
|
||||
}
|
||||
return size*nmemb;
|
||||
}
|
||||
|
||||
static bool parseHeader(const std::string& str, std::string& header, std::string& value) {
|
||||
std::vector<std::string> parts;
|
||||
boost::split(parts, str, boost::is_any_of(":"), boost::token_compress_on);
|
||||
|
||||
if (parts.size() != 2)
|
||||
return false;
|
||||
|
||||
header = boost::trim_copy(parts[0]);
|
||||
value = boost::trim_copy(parts[1]);
|
||||
return true;
|
||||
}
|
||||
|
||||
static int extract_response_code(const std::string& input) {
|
||||
std::size_t space_pos = input.find(' ');
|
||||
if (space_pos == std::string::npos) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid HTTP response format %s\n", input.c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::size_t code_start_pos = space_pos + 1;
|
||||
std::size_t code_end_pos = input.find(' ', code_start_pos);
|
||||
if (code_end_pos == std::string::npos) {
|
||||
code_end_pos = input.length();
|
||||
}
|
||||
|
||||
std::string code_str = input.substr(code_start_pos, code_end_pos - code_start_pos);
|
||||
int response_code = std::stoi(code_str);
|
||||
return response_code;
|
||||
}
|
||||
|
||||
static size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo_t *conn) {
|
||||
size_t bytes_received = size * nitems;
|
||||
const std::string prefix = "HTTP/";
|
||||
deepgram_t* d = conn->deepgram;
|
||||
std::string header, value;
|
||||
std::string input(buffer, bytes_received);
|
||||
if (parseHeader(input, header, value)) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s with value %s\n", header.c_str(), value.c_str());
|
||||
if (0 == header.compare("dg-request-id")) d->request_id = strdup(value.c_str());
|
||||
else if (0 == header.compare("dg-model-name")) d->reported_model_name = strdup(value.c_str());
|
||||
else if (0 == header.compare("dg-model-uuid")) d->reported_model_uuid = strdup(value.c_str());
|
||||
else if (0 == header.compare("dg-char-count")) d->reported_char_count = strdup(value.c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: %s\n", input.c_str());
|
||||
if (input.rfind(prefix, 0) == 0) {
|
||||
try {
|
||||
d->response_code = extract_response_code(input);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: parsed response code: %ld\n", d->response_code);
|
||||
} catch (const std::invalid_argument& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "header_callback: invalid response code %s\n", input.substr(prefix.length()).c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
return bytes_received;
|
||||
}
|
||||
|
||||
/* CURLOPT_OPENSOCKETFUNCTION */
|
||||
static curl_socket_t opensocket(void *clientp, curlsocktype purpose, struct curl_sockaddr *address) {
|
||||
curl_socket_t sockfd = CURL_SOCKET_BAD;
|
||||
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "opensocket: %d\n", purpose);
|
||||
/* restrict to IPv4 */
|
||||
if(purpose == CURLSOCKTYPE_IPCXN && address->family == AF_INET) {
|
||||
/* create a tcp socket object */
|
||||
boost::asio::ip::tcp::socket *tcp_socket = new boost::asio::ip::tcp::socket(io_service);
|
||||
|
||||
/* open it and get the native handle*/
|
||||
boost::system::error_code ec;
|
||||
tcp_socket->open(boost::asio::ip::tcp::v4(), ec);
|
||||
|
||||
if(ec) {
|
||||
/* An error occurred */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't open socket [%ld][%s]\n", ec, ec.message().c_str());
|
||||
}
|
||||
else {
|
||||
sockfd = tcp_socket->native_handle();
|
||||
|
||||
/* save it for monitoring */
|
||||
socket_map.insert(std::pair<curl_socket_t, boost::asio::ip::tcp::socket *>(sockfd, tcp_socket));
|
||||
}
|
||||
}
|
||||
return sockfd;
|
||||
}
|
||||
|
||||
/* CURLOPT_CLOSESOCKETFUNCTION */
|
||||
static int close_socket(void *clientp, curl_socket_t item) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "close_socket : %#X\n", item);
|
||||
|
||||
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(item);
|
||||
if(it != socket_map.end()) {
|
||||
delete it->second;
|
||||
socket_map.erase(it);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
extern "C" {
|
||||
switch_status_t deepgram_speech_load() {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_loading..\n");
|
||||
memset(&global, 0, sizeof(GlobalInfo_t));
|
||||
global.multi = curl_multi_init();
|
||||
|
||||
if (!global.multi) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "deepgram_speech_load curl_multi_init() failed, exiting!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
curl_multi_setopt(global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_SOCKETDATA, &global);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_TIMERDATA, &global);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
||||
|
||||
/* create temp folder for cache files */
|
||||
const char* baseDir = std::getenv("JAMBONZ_TMP_CACHE_FOLDER");
|
||||
if (!baseDir) {
|
||||
baseDir = "/tmp/";
|
||||
}
|
||||
if (strcmp(baseDir, "/") == 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", baseDir);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
fullDirPath = std::string(baseDir) + "tts-cache-files";
|
||||
|
||||
// Create the directory with read, write, and execute permissions for everyone
|
||||
mode_t oldMask = umask(0);
|
||||
int result = mkdir(fullDirPath.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
|
||||
umask(oldMask);
|
||||
if (result != 0) {
|
||||
if (errno != EEXIST) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", fullDirPath.c_str());
|
||||
fullDirPath = "";
|
||||
}
|
||||
else switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "folder %s already exists\n", fullDirPath.c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "created folder %s\n", fullDirPath.c_str());
|
||||
}
|
||||
|
||||
/* start worker thread that handles transfers*/
|
||||
std::thread t(threadFunc) ;
|
||||
worker_thread.swap( t ) ;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_loaded..\n");
|
||||
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t deepgram_speech_unload() {
|
||||
/* stop the ASIO IO service */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deepgram_speech_unload: stopping io service\n");
|
||||
io_service.stop();
|
||||
|
||||
/* Join the worker thread */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deepgram_speech_unload: wait for worker thread to complete\n");
|
||||
if (worker_thread.joinable()) {
|
||||
worker_thread.join();
|
||||
}
|
||||
|
||||
/* cleanup curl multi handle*/
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deepgram_speech_unload: release curl multi\n");
|
||||
curl_multi_cleanup(global.multi);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deepgram_speech_unload: completed\n");
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t deepgram_speech_open(deepgram_t* deepgram) {
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t deepgram_speech_feed_tts(deepgram_t* d, char* text, switch_speech_flag_t *flags) {
|
||||
CURLMcode rc;
|
||||
|
||||
const int MAX_CHARS = 20;
|
||||
char tempText[MAX_CHARS + 4]; // +4 for the ellipsis and null terminator
|
||||
|
||||
if (strlen(text) > MAX_CHARS) {
|
||||
strncpy(tempText, text, MAX_CHARS);
|
||||
strcpy(tempText + MAX_CHARS, "...");
|
||||
} else {
|
||||
strcpy(tempText, text);
|
||||
}
|
||||
|
||||
/* open cache file */
|
||||
if (d->cache_audio && fullDirPath.length() > 0) {
|
||||
switch_uuid_t uuid;
|
||||
char uuid_str[SWITCH_UUID_FORMATTED_LENGTH + 1];
|
||||
char outfile[512] = "";
|
||||
int fd;
|
||||
|
||||
switch_uuid_get(&uuid);
|
||||
switch_uuid_format(uuid_str, &uuid);
|
||||
|
||||
switch_snprintf(outfile, sizeof(outfile), "%s%s%s.r8", fullDirPath.c_str(), SWITCH_PATH_SEPARATOR, uuid_str);
|
||||
d->cache_filename = strdup(outfile);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "writing audio cache file to %s\n", d->cache_filename);
|
||||
|
||||
mode_t oldMask = umask(0);
|
||||
fd = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
|
||||
umask(oldMask);
|
||||
if (fd == -1 ) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
|
||||
}
|
||||
else {
|
||||
d->file = fdopen(fd, "wb");
|
||||
if (!d->file) {
|
||||
close(fd);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!d->api_key) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "deepgram_speech_feed_tts: no api_key provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
/* format url*/
|
||||
std::string url;
|
||||
std::ostringstream url_stream;
|
||||
// always use sample_rate=8000 for support jambonz caching system.
|
||||
url_stream << "https://api.deepgram.com/v1/speak?model=" << d->voice_name << "&encoding=linear16&sample_rate=8000";
|
||||
url = url_stream.str();
|
||||
|
||||
/* create the JSON body */
|
||||
cJSON * jResult = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(jResult, "text", text);
|
||||
char *json = cJSON_PrintUnformatted(jResult);
|
||||
|
||||
cJSON_Delete(jResult);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_feed_tts: [%s] [%s]\n", url.c_str(), tempText);
|
||||
|
||||
ConnInfo_t *conn = pool.malloc() ;
|
||||
|
||||
CURL* easy = createEasyHandle();
|
||||
d->conn = (void *) conn ;
|
||||
conn->deepgram = d;
|
||||
conn->easy = easy;
|
||||
conn->global = &global;
|
||||
conn->hdr_list = NULL ;
|
||||
conn->file = d->file;
|
||||
conn->body = json;
|
||||
conn->flushed = false;
|
||||
conn->has_last_byte = false;
|
||||
conn->last_byte = 0;
|
||||
|
||||
|
||||
d->circularBuffer = (void *) new CircularBuffer_t(BUFFER_GROW_SIZE);
|
||||
// Always use deepgram at rate 8000 for helping cache audio from jambonz.
|
||||
if (d->rate != 8000) {
|
||||
int err;
|
||||
d->resampler = speex_resampler_init(1, 8000, d->rate, SWITCH_RESAMPLE_QUALITY, &err);
|
||||
if (0 != err) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing resampler: %s.\n", speex_resampler_strerror(err));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
std::ostringstream api_key_stream;
|
||||
api_key_stream << "Authorization: Token " << d->api_key;
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, write_cb);
|
||||
curl_easy_setopt(easy, CURLOPT_WRITEDATA, conn);
|
||||
curl_easy_setopt(easy, CURLOPT_ERRORBUFFER, conn->error);
|
||||
curl_easy_setopt(easy, CURLOPT_PRIVATE, conn);
|
||||
curl_easy_setopt(easy, CURLOPT_VERBOSE, 0L);
|
||||
curl_easy_setopt(easy, CURLOPT_NOPROGRESS, 1L);
|
||||
curl_easy_setopt(easy, CURLOPT_HEADERFUNCTION, header_callback);
|
||||
curl_easy_setopt(easy, CURLOPT_HEADERDATA, conn);
|
||||
|
||||
/* call this function to get a socket */
|
||||
curl_easy_setopt(easy, CURLOPT_OPENSOCKETFUNCTION, opensocket);
|
||||
|
||||
/* call this function to close a socket */
|
||||
curl_easy_setopt(easy, CURLOPT_CLOSESOCKETFUNCTION, close_socket);
|
||||
|
||||
conn->hdr_list = curl_slist_append(conn->hdr_list, api_key_stream.str().c_str());
|
||||
conn->hdr_list = curl_slist_append(conn->hdr_list, "Content-Type: application/json");
|
||||
curl_easy_setopt(easy, CURLOPT_HTTPHEADER, conn->hdr_list);
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
|
||||
//curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
|
||||
|
||||
// libcurl adding random byte to the response body that creates white noise to audio file
|
||||
// https://github.com/curl/curl/issues/10525
|
||||
const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
|
||||
curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
|
||||
|
||||
rc = curl_multi_add_handle(global.multi, conn->easy);
|
||||
mcode_test("new_conn: curl_multi_add_handle", rc);
|
||||
|
||||
/* start a timer to measure the duration until we receive first byte of audio */
|
||||
conn->startTime = std::chrono::high_resolution_clock::now();
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "deepgram_speech_feed_tts: called curl_multi_add_handle\n");
|
||||
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t deepgram_speech_read_tts(deepgram_t* d, void *data, size_t *datalen, switch_speech_flag_t *flags) {
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) d->circularBuffer;
|
||||
std::vector<uint16_t> pcm_data;
|
||||
|
||||
{
|
||||
switch_mutex_lock(d->mutex);
|
||||
ConnInfo_t *conn = (ConnInfo_t *) d->conn;
|
||||
if (d->response_code > 0 && d->response_code != 200) {
|
||||
switch_mutex_unlock(d->mutex);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_read_tts, returning failure\n") ;
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (conn && conn->flushed) {
|
||||
switch_mutex_unlock(d->mutex);
|
||||
return SWITCH_STATUS_BREAK;
|
||||
}
|
||||
if (cBuffer->empty()) {
|
||||
if (d->draining) {
|
||||
switch_mutex_unlock(d->mutex);
|
||||
return SWITCH_STATUS_BREAK;
|
||||
}
|
||||
/* no audio available yet so send silence */
|
||||
memset(data, 255, *datalen);
|
||||
switch_mutex_unlock(d->mutex);
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
size_t size = std::min((*datalen/(2 * d->rate / 8000)), cBuffer->size());
|
||||
pcm_data.insert(pcm_data.end(), cBuffer->begin(), cBuffer->begin() + size);
|
||||
cBuffer->erase(cBuffer->begin(), cBuffer->begin() + size);
|
||||
switch_mutex_unlock(d->mutex);
|
||||
}
|
||||
|
||||
size_t data_size = pcm_data.size();
|
||||
|
||||
if (d->resampler) {
|
||||
std::vector<int16_t> in(pcm_data.begin(), pcm_data.end());
|
||||
|
||||
std::vector<int16_t> out((*datalen));
|
||||
spx_uint32_t in_len = data_size;
|
||||
spx_uint32_t out_len = out.size();
|
||||
speex_resampler_process_interleaved_int(d->resampler, in.data(), &in_len, out.data(), &out_len);
|
||||
|
||||
if (out_len > out.size()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Resampler output exceeded maximum buffer size!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
memcpy(data, out.data(), out_len * sizeof(int16_t));
|
||||
*datalen = out_len * sizeof(int16_t);
|
||||
} else {
|
||||
memcpy(data, pcm_data.data(), pcm_data.size() * sizeof(uint16_t));
|
||||
*datalen = pcm_data.size() * sizeof(uint16_t);
|
||||
}
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t deepgram_speech_flush_tts(deepgram_t* d) {
|
||||
bool download_complete = d->response_code == 200;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;
|
||||
|
||||
ConnInfo_t *conn = (ConnInfo_t *) d->conn;
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) d->circularBuffer;
|
||||
delete cBuffer;
|
||||
d->circularBuffer = nullptr ;
|
||||
|
||||
// destroy resampler
|
||||
if (d->resampler) {
|
||||
speex_resampler_destroy(d->resampler);
|
||||
d->resampler = NULL;
|
||||
}
|
||||
|
||||
if (conn) {
|
||||
conn->flushed = true;
|
||||
if (!download_complete) {
|
||||
if (conn->file) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "closing audio cache file %s because download was interrupted\n", d->cache_filename);
|
||||
if (fclose(conn->file) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "error closing audio cache file\n");
|
||||
}
|
||||
conn->file = nullptr ;
|
||||
}
|
||||
|
||||
if (d->cache_filename) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", d->cache_filename);
|
||||
if (unlink(d->cache_filename) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n",
|
||||
d->cache_filename, errno, strerror(errno));
|
||||
}
|
||||
free(d->cache_filename);
|
||||
d->cache_filename = nullptr ;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (d->session_id) {
|
||||
switch_core_session_t* session = switch_core_session_locate(d->session_id);
|
||||
if (session) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_core_session_rwunlock(session);
|
||||
if (channel) {
|
||||
switch_event_t *event;
|
||||
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_STOP) == SWITCH_STATUS_SUCCESS) {
|
||||
switch_channel_event_set_data(channel, event);
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_deepgram_response_code", std::to_string(d->response_code).c_str());
|
||||
if (d->cache_filename && d->response_code == 200) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", d->cache_filename);
|
||||
}
|
||||
if (d->response_code != 200 && d->err_msg) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_error", d->err_msg);
|
||||
}
|
||||
switch_event_fire(&event);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t deepgram_speech_close(deepgram_t* w) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "deepgram_speech_close\n") ;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#ifndef __DEEPGRAM_GLUE_H__
|
||||
#define __DEEPGRAM_GLUE_H__
|
||||
|
||||
switch_status_t deepgram_speech_load();
|
||||
switch_status_t deepgram_speech_open(deepgram_t* deepgram);
|
||||
switch_status_t deepgram_speech_feed_tts(deepgram_t* deepgram, char* text, switch_speech_flag_t *flags);
|
||||
switch_status_t deepgram_speech_read_tts(deepgram_t* deepgram, void *data, size_t *datalen, switch_speech_flag_t *flags);
|
||||
switch_status_t deepgram_speech_flush_tts(deepgram_t* deepgram);
|
||||
switch_status_t deepgram_speech_close(deepgram_t* deepgram);
|
||||
switch_status_t deepgram_speech_unload();
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,160 @@
|
||||
#include "mod_deepgram_tts.h"
|
||||
#include "deepgram_glue.h"
|
||||
|
||||
SWITCH_MODULE_LOAD_FUNCTION(mod_deepgram_tts_load);
|
||||
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_deepgram_tts_shutdown);
|
||||
SWITCH_MODULE_DEFINITION(mod_deepgram_tts, mod_deepgram_tts_load, mod_deepgram_tts_shutdown, NULL);
|
||||
|
||||
static void cleardeepgram(deepgram_t* d, int freeAll) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "cleardeepgram\n");
|
||||
if (d->api_key) free(d->api_key);
|
||||
|
||||
if (d->request_id) free(d->request_id);
|
||||
if (d->reported_model_name) free(d->reported_model_name);
|
||||
if (d->reported_model_uuid) free(d->reported_model_uuid);
|
||||
if (d->reported_char_count) free(d->reported_char_count);
|
||||
if (d->ct) free(d->ct);
|
||||
if (d->err_msg) free(d->err_msg);
|
||||
if (d->name_lookup_time_ms) free(d->name_lookup_time_ms);
|
||||
if (d->connect_time_ms) free(d->connect_time_ms);
|
||||
if (d->final_response_time_ms) free(d->final_response_time_ms);
|
||||
if (d->cache_filename) free(d->cache_filename);
|
||||
|
||||
|
||||
d->api_key = NULL;
|
||||
d->request_id = NULL;
|
||||
|
||||
d->reported_model_name = NULL;
|
||||
d->reported_model_uuid = NULL;
|
||||
d->reported_char_count = NULL;
|
||||
d->ct = NULL;
|
||||
d->err_msg = NULL;
|
||||
d->name_lookup_time_ms = NULL;
|
||||
d->connect_time_ms = NULL;
|
||||
d->final_response_time_ms = NULL;
|
||||
d->cache_filename = NULL;
|
||||
|
||||
if (freeAll) {
|
||||
if (d->voice_name) free(d->voice_name);
|
||||
if (d->session_id) free(d->session_id);
|
||||
d->voice_name = NULL;
|
||||
d->session_id = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static deepgram_t * createOrRetrievePrivateData(switch_speech_handle_t *sh) {
|
||||
deepgram_t *d = (deepgram_t *) sh->private_info;
|
||||
if (!d) {
|
||||
d = switch_core_alloc(sh->memory_pool, sizeof(*d));
|
||||
sh->private_info = d;
|
||||
memset(d, 0, sizeof(*d));
|
||||
switch_mutex_init(&d->mutex, SWITCH_MUTEX_NESTED, sh->memory_pool);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "allocated deepgram_t\n");
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
switch_status_t d_speech_open(switch_speech_handle_t *sh, const char *voice_name, int rate, int channels, switch_speech_flag_t *flags)
|
||||
{
|
||||
deepgram_t *d = createOrRetrievePrivateData(sh);
|
||||
d->voice_name = strdup(voice_name);
|
||||
d->rate = rate;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_open voice: %s, rate %d, channels %d\n", voice_name, rate, channels);
|
||||
return deepgram_speech_open(d);
|
||||
}
|
||||
|
||||
static switch_status_t d_speech_close(switch_speech_handle_t *sh, switch_speech_flag_t *flags)
|
||||
{
|
||||
switch_status_t rc;
|
||||
deepgram_t *d = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_close\n");
|
||||
|
||||
switch_mutex_destroy(d->mutex);
|
||||
|
||||
rc = deepgram_speech_close(d);
|
||||
cleardeepgram(d, 1);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeswitch will call this function to feed us text to speak
|
||||
*/
|
||||
static switch_status_t d_speech_feed_tts(switch_speech_handle_t *sh, char *text, switch_speech_flag_t *flags)
|
||||
{
|
||||
deepgram_t *d = createOrRetrievePrivateData(sh);
|
||||
d->draining = 0;
|
||||
d->reads = 0;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_feed_tts\n");
|
||||
|
||||
return deepgram_speech_feed_tts(d, text, flags);
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeswitch calls periodically to get some rendered audio in L16 format. We can provide up to 8k of audio at a time.
|
||||
*/
|
||||
static switch_status_t d_speech_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, switch_speech_flag_t *flags)
|
||||
{
|
||||
deepgram_t *d = createOrRetrievePrivateData(sh);
|
||||
return deepgram_speech_read_tts(d, data, datalen, flags);
|
||||
}
|
||||
|
||||
/**
|
||||
* This is called at the end, not sure exactly what we need to do here..
|
||||
*/
|
||||
static void d_speech_flush_tts(switch_speech_handle_t *sh)
|
||||
{
|
||||
deepgram_t *d = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_flush_tts\n");
|
||||
deepgram_speech_flush_tts(d);
|
||||
|
||||
cleardeepgram(d, 0);
|
||||
}
|
||||
|
||||
static void d_text_param_tts(switch_speech_handle_t *sh, char *param, const char *val)
|
||||
{
|
||||
deepgram_t *d = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_text_param_tts: %s=%s\n", param, val);
|
||||
if (0 == strcmp(param, "api_key")) {
|
||||
if (d->api_key) free(d->api_key);
|
||||
d->api_key = strdup(val);
|
||||
} else if (0 == strcmp(param, "voice")) {
|
||||
if (d->voice_name) free(d->voice_name);
|
||||
d->voice_name = strdup(val);
|
||||
} else if (0 == strcmp(param, "session-uuid")) {
|
||||
if (d->session_id) free(d->session_id);
|
||||
d->session_id = strdup(val);
|
||||
} else if (0 == strcmp(param, "write_cache_file") && switch_true(val)) {
|
||||
d->cache_audio = 1;
|
||||
}
|
||||
}
|
||||
|
||||
static void d_numeric_param_tts(switch_speech_handle_t *sh, char *param, int val)
|
||||
{
|
||||
}
|
||||
static void d_float_param_tts(switch_speech_handle_t *sh, char *param, double val)
|
||||
{
|
||||
}
|
||||
|
||||
SWITCH_MODULE_LOAD_FUNCTION(mod_deepgram_tts_load)
|
||||
{
|
||||
switch_speech_interface_t *speech_interface;
|
||||
|
||||
*module_interface = switch_loadable_module_create_module_interface(pool, modname);
|
||||
speech_interface = switch_loadable_module_create_interface(*module_interface, SWITCH_SPEECH_INTERFACE);
|
||||
speech_interface->interface_name = "deepgram";
|
||||
speech_interface->speech_open = d_speech_open;
|
||||
speech_interface->speech_close = d_speech_close;
|
||||
speech_interface->speech_feed_tts = d_speech_feed_tts;
|
||||
speech_interface->speech_read_tts = d_speech_read_tts;
|
||||
speech_interface->speech_flush_tts = d_speech_flush_tts;
|
||||
speech_interface->speech_text_param_tts = d_text_param_tts;
|
||||
speech_interface->speech_numeric_param_tts = d_numeric_param_tts;
|
||||
speech_interface->speech_float_param_tts = d_float_param_tts;
|
||||
return deepgram_speech_load();
|
||||
}
|
||||
|
||||
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_deepgram_tts_shutdown)
|
||||
{
|
||||
return deepgram_speech_unload();
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
#ifndef __MOD_DEEPGRAM_TTS_H__
|
||||
#define __MOD_DEEPGRAM_TTS_H__
|
||||
|
||||
#include <switch.h>
|
||||
#include <speex/speex_resampler.h>
|
||||
|
||||
typedef struct deepgram_data {
|
||||
char *voice_name;
|
||||
char *api_key;
|
||||
|
||||
/* result data */
|
||||
long response_code;
|
||||
char *ct;
|
||||
// Deepgram hedaers
|
||||
//dg-model-name
|
||||
char *reported_model_name;
|
||||
//dg-model-uuid
|
||||
char *reported_model_uuid;
|
||||
//dg-char-count
|
||||
char *reported_char_count;
|
||||
//dg-request-id
|
||||
char *request_id;
|
||||
char *name_lookup_time_ms;
|
||||
char *connect_time_ms;
|
||||
char *final_response_time_ms;
|
||||
char *err_msg;
|
||||
char *cache_filename;
|
||||
char *session_id;
|
||||
|
||||
int rate;
|
||||
int draining;
|
||||
int reads;
|
||||
int cache_audio;
|
||||
|
||||
void *conn;
|
||||
void *circularBuffer;
|
||||
switch_mutex_t *mutex;
|
||||
FILE *file;
|
||||
SpeexResamplerState *resampler;
|
||||
} deepgram_t;
|
||||
|
||||
#endif
|
||||
+10
-2
@@ -2,9 +2,17 @@ include $(top_srcdir)/build/modmake.rulesam
|
||||
MODNAME=mod_dub
|
||||
|
||||
mod_LTLIBRARIES = mod_dub.la
|
||||
mod_dub_la_SOURCES = file_loader.cpp audio_downloader.cpp mod_dub.c dub_glue.cpp
|
||||
mod_dub_la_SOURCES = ap_file.cpp ap_http.cpp track.cpp dub_glue.cpp tts_vendor_parser.cpp mod_dub.c vector_math.cpp mpg_decode.cpp
|
||||
mod_dub_la_CFLAGS = $(AM_CFLAGS)
|
||||
mod_dub_la_CXXFLAGS = $(AM_CXXFLAGS) -std=c++17
|
||||
|
||||
if USE_AVX2
|
||||
mod_dub_la_CXXFLAGS += -mavx2 -DUSE_AVX2
|
||||
else
|
||||
if USE_SSE2
|
||||
mod_dub_la_CXXFLAGS += -msse2 -DUSE_SSE2
|
||||
endif
|
||||
endif
|
||||
|
||||
mod_dub_la_LIBADD = $(switch_builddir)/libfreeswitch.la
|
||||
mod_dub_la_LDFLAGS = -avoid-version -module -no-undefined -shared `pkg-config --libs boost` -lstdc++ -lmpg123
|
||||
mod_dub_la_LDFLAGS = -avoid-version -module -no-undefined -lstdc++ -lmpg123 -lboost_system -lboost_thread
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
#ifndef __AP_H__
|
||||
#define __AP_H__
|
||||
|
||||
#include <mutex>
|
||||
#include <functional>
|
||||
#include "common.h"
|
||||
|
||||
class AudioProducer {
|
||||
public:
|
||||
AudioProducer(
|
||||
std::mutex& mutex,
|
||||
CircularBuffer_t& circularBuffer,
|
||||
int sampleRate
|
||||
) : _mutex(mutex), _buffer(circularBuffer), _sampleRate(sampleRate), _notified(false), _loop(false), _gain(0) {}
|
||||
virtual ~AudioProducer() {}
|
||||
|
||||
virtual void notifyDone(bool error, const std::string& errorMsg) {
|
||||
if (!_notified) {
|
||||
_notified = true;
|
||||
if (_callback) _callback(error, errorMsg);
|
||||
}
|
||||
}
|
||||
virtual void start(std::function<void(bool, const std::string&)> callback) = 0;
|
||||
virtual void stop() = 0;
|
||||
|
||||
bool isLoopedAudio() const { return _loop; }
|
||||
|
||||
protected:
|
||||
std::mutex& _mutex;
|
||||
CircularBuffer_t& _buffer;
|
||||
int _sampleRate;
|
||||
int _gain;
|
||||
bool _loop;
|
||||
std::function<void(bool, const std::string&)> _callback;
|
||||
bool _notified;
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,199 @@
|
||||
#include "switch.h"
|
||||
#include "ap_file.h"
|
||||
#include "mpg_decode.h"
|
||||
|
||||
#define INIT_BUFFER_SIZE (80000)
|
||||
#define BUFFER_GROW_SIZE (80000)
|
||||
#define BUFFER_THROTTLE_LOW (40000)
|
||||
#define BUFFER_THROTTLE_HIGH (160000)
|
||||
|
||||
bool AudioProducerFile::initialized = false;
|
||||
boost::asio::io_service AudioProducerFile::io_service;
|
||||
std::thread AudioProducerFile::worker_thread;
|
||||
|
||||
void AudioProducerFile::threadFunc() {
|
||||
io_service.reset() ;
|
||||
boost::asio::io_service::work work(io_service);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "file_loader threadFunc - starting\n");
|
||||
|
||||
for(;;) {
|
||||
|
||||
try {
|
||||
io_service.run() ;
|
||||
break ;
|
||||
}
|
||||
catch( std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "file_loader threadFunc - Error: %s\n", e.what());
|
||||
}
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "file_loader threadFunc - ending\n");
|
||||
}
|
||||
|
||||
void AudioProducerFile::_init() {
|
||||
if (!initialized) {
|
||||
initialized = true;
|
||||
if (mpg123_init() != MPG123_OK) {
|
||||
throw std::runtime_error("AudioProducerFile::AudioProducerFile: failed to initiate MPG123");
|
||||
return ;
|
||||
}
|
||||
|
||||
/* start worker thread */
|
||||
std::thread t(threadFunc) ;
|
||||
worker_thread.swap( t ) ;
|
||||
}
|
||||
}
|
||||
|
||||
void AudioProducerFile::_deinit() {
|
||||
if (initialized) {
|
||||
initialized = false;
|
||||
io_service.stop();
|
||||
if (worker_thread.joinable()) {
|
||||
worker_thread.join();
|
||||
}
|
||||
mpg123_exit();
|
||||
}
|
||||
}
|
||||
AudioProducerFile::AudioProducerFile(
|
||||
std::mutex& mutex,
|
||||
CircularBuffer_t& circularBuffer,
|
||||
int sampleRate
|
||||
) : AudioProducer(mutex, circularBuffer, sampleRate), _timer(io_service), _mh(nullptr), _fp(nullptr) {
|
||||
|
||||
AudioProducerFile::_init();
|
||||
}
|
||||
|
||||
AudioProducerFile::~AudioProducerFile() {
|
||||
reset();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "AudioProducerFile::~AudioProducerFile %p\n", (void *)this);
|
||||
}
|
||||
|
||||
void AudioProducerFile::queueFileAudio(const std::string& path, int gain, bool loop) {
|
||||
_path = path;
|
||||
_gain = gain;
|
||||
_loop = loop;
|
||||
|
||||
/* we only handle mp3 or r8 files atm */
|
||||
size_t pos = path.find_last_of('.');
|
||||
if (pos == std::string::npos) {
|
||||
throw std::runtime_error("file " + path + " has no extension");
|
||||
}
|
||||
auto filetype = path.substr(pos + 1);
|
||||
if (0 == filetype.compare("mp3")) _type = FILE_TYPE_MP3;
|
||||
else if (0 == filetype.compare("r8")) _type = FILE_TYPE_R8;
|
||||
else throw std::runtime_error("file " + path + " has unsupported extension " + filetype);
|
||||
}
|
||||
|
||||
void AudioProducerFile::start(std::function<void(bool, const std::string&)> callback) {
|
||||
int mhError = 0;
|
||||
|
||||
_callback = callback;
|
||||
|
||||
/* allocate handle for mpeg decoding */
|
||||
_mh = mpg123_new("auto", &mhError);
|
||||
if (!_mh) {
|
||||
const char *mhErr = mpg123_plain_strerror(mhError);
|
||||
throw std::runtime_error("Error allocating mpg123 handle! " + std::string(mhErr));
|
||||
}
|
||||
|
||||
if (mpg123_open_feed(_mh) != MPG123_OK) throw std::runtime_error("Error mpg123_open_feed!");
|
||||
if (mpg123_format_all(_mh) != MPG123_OK) throw std::runtime_error("Error mpg123_format_all!");
|
||||
if (mpg123_param(_mh, MPG123_FORCE_RATE, _sampleRate, 0) != MPG123_OK) throw std::runtime_error("Error forcing resample to 8k!");
|
||||
if (mpg123_param(_mh, MPG123_FLAGS, MPG123_MONO_MIX, 0) != MPG123_OK) throw std::runtime_error("Error forcing single channel!");
|
||||
|
||||
_fp = fopen(_path.c_str(), "rb");
|
||||
if (!_fp) throw std::runtime_error("Error opening file " + _path);
|
||||
|
||||
_status = Status_t::STATUS_AWAITING_RESTART;
|
||||
|
||||
/* do the initial read in the worker thread so we don't block here */
|
||||
_timer.expires_from_now(boost::posix_time::millisec(1));
|
||||
_timer.async_wait(boost::bind(&AudioProducerFile::read_cb, this, boost::placeholders::_1));
|
||||
}
|
||||
|
||||
void AudioProducerFile::read_cb(const boost::system::error_code& error) {
|
||||
if (_status == Status_t::STATUS_STOPPED) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: session gone\n");
|
||||
return;
|
||||
}
|
||||
if (_status == Status_t::STATUS_AWAITING_RESTART) {
|
||||
_status = Status_t::STATUS_IN_PROGRESS;
|
||||
}
|
||||
if (!error) {
|
||||
size_t size = _buffer.size();
|
||||
if (size < BUFFER_THROTTLE_LOW) {
|
||||
std::vector<int16_t> pcm_data;
|
||||
int8_t buf[INIT_BUFFER_SIZE];
|
||||
|
||||
size_t bytesRead = ::fread(buf, sizeof(int8_t), INIT_BUFFER_SIZE, _fp);
|
||||
if (bytesRead <= 0) {
|
||||
if (::feof(_fp)) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %p eof\n", (void *) this);
|
||||
else if (::ferror(_fp)) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "read_cb: %p error reading file\n", (void *) this);
|
||||
else switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "read_cb: %p unknown error reading file\n", (void *) this);
|
||||
_status = Status_t::STATUS_COMPLETE;
|
||||
return;
|
||||
}
|
||||
|
||||
if (_type == FileType_t::FILE_TYPE_MP3) pcm_data = convert_mp3_to_linear(_mh, _gain, buf, bytesRead);
|
||||
else pcm_data = std::vector<int16_t>(reinterpret_cast<int16_t*>(buf), reinterpret_cast<int16_t*>(buf) + bytesRead / 2);
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
|
||||
// Resize the buffer if necessary
|
||||
size_t capacity = _buffer.capacity();
|
||||
if (capacity - size < pcm_data.size()) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb %p growing buffer, size now %ld\n", (void *) this, size);
|
||||
_buffer.set_capacity(size + std::max(pcm_data.size(), (size_t)BUFFER_GROW_SIZE));
|
||||
}
|
||||
|
||||
/* Push the data into the buffer */
|
||||
_buffer.insert(_buffer.end(), pcm_data.data(), pcm_data.data() + pcm_data.size());
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %p wrote data, buffer size is now %ld\n", (void *) this, _buffer.size());
|
||||
}
|
||||
|
||||
if (bytesRead < INIT_BUFFER_SIZE) {
|
||||
_status = Status_t::STATUS_COMPLETE;
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %u reached end of file, status is %s\n", (void *) this, status2String(_status));
|
||||
}
|
||||
}
|
||||
|
||||
if (_status == Status_t::STATUS_COMPLETE) {
|
||||
cleanup(Status_t::STATUS_COMPLETE);
|
||||
}
|
||||
else {
|
||||
// read more in 2 seconds
|
||||
_timer.expires_from_now(boost::posix_time::millisec(2000));
|
||||
_timer.async_wait(boost::bind(&AudioProducerFile::read_cb, this, boost::placeholders::_1));
|
||||
}
|
||||
} else {
|
||||
cleanup(Status_t::STATUS_FAILED, error.message());
|
||||
}
|
||||
}
|
||||
|
||||
void AudioProducerFile::stop() {
|
||||
cleanup(Status_t::STATUS_STOPPED, "");
|
||||
}
|
||||
|
||||
void AudioProducerFile::reset() {
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
if (_fp) {
|
||||
fclose(_fp);
|
||||
_fp = nullptr;
|
||||
}
|
||||
if (_mh) {
|
||||
mpg123_close(_mh);
|
||||
mpg123_delete(_mh);
|
||||
_mh = nullptr;
|
||||
}
|
||||
}
|
||||
_timer.cancel();
|
||||
_status = Status_t::STATUS_NONE;
|
||||
}
|
||||
|
||||
void AudioProducerFile::cleanup(Status_t status, std::string errMsg) {
|
||||
reset();
|
||||
_status = status;
|
||||
notifyDone(status != Status_t::STATUS_COMPLETE, errMsg);
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
#ifndef __AP_FILE_H__
|
||||
#define __AP_FILE_H__
|
||||
|
||||
#include <thread>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/bind/bind.hpp>
|
||||
|
||||
#include <mpg123.h>
|
||||
|
||||
#include "ap.h"
|
||||
|
||||
class AudioProducerFile : public AudioProducer {
|
||||
public:
|
||||
|
||||
typedef enum
|
||||
{
|
||||
STATUS_NONE = 0,
|
||||
STATUS_FAILED,
|
||||
STATUS_IN_PROGRESS,
|
||||
STATUS_PAUSED,
|
||||
STATUS_COMPLETE,
|
||||
STATUS_AWAITING_RESTART,
|
||||
STATUS_STOPPED
|
||||
} Status_t;
|
||||
|
||||
typedef enum {
|
||||
FILE_TYPE_MP3 = 0,
|
||||
FILE_TYPE_R8
|
||||
} FileType_t;
|
||||
|
||||
const char* status2String(Status_t status)
|
||||
{
|
||||
static const char* statusStrings[] = {
|
||||
"STATUS_NONE",
|
||||
"STATUS_FAILED",
|
||||
"STATUS_IN_PROGRESS",
|
||||
"STATUS_PAUSED",
|
||||
"STATUS_COMPLETE",
|
||||
"STATUS_AWAITING_RESTART",
|
||||
"STATUS_STOPPED"
|
||||
};
|
||||
|
||||
if (status >= 0 && status < sizeof(statusStrings) / sizeof(statusStrings[0]))
|
||||
{
|
||||
return statusStrings[status];
|
||||
}
|
||||
else
|
||||
{
|
||||
return "UNKNOWN_STATUS";
|
||||
}
|
||||
}
|
||||
|
||||
AudioProducerFile(
|
||||
std::mutex& mutex,
|
||||
CircularBuffer_t& circularBuffer,
|
||||
int sampleRate
|
||||
);
|
||||
virtual ~AudioProducerFile();
|
||||
virtual void start(std::function<void(bool, const std::string&)> callback);
|
||||
virtual void stop();
|
||||
void cleanup(Status_t status, std::string errMsg = "");
|
||||
void reset();
|
||||
|
||||
void queueFileAudio(const std::string& path, int gain = 0, bool loop = false);
|
||||
|
||||
void read_cb(const boost::system::error_code& error);
|
||||
|
||||
static bool initialized;
|
||||
static std::thread worker_thread;
|
||||
static boost::asio::io_service io_service;
|
||||
static void threadFunc();
|
||||
|
||||
private:
|
||||
|
||||
static void _init();
|
||||
static void _deinit();
|
||||
|
||||
void stop_file_load();
|
||||
|
||||
std::string _path;
|
||||
Status_t _status;
|
||||
FileType_t _type;
|
||||
mpg123_handle *_mh;
|
||||
boost::asio::deadline_timer _timer;
|
||||
FILE* _fp;
|
||||
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,669 @@
|
||||
#include <functional>
|
||||
#include <mpg123.h>
|
||||
#include "switch.h"
|
||||
#include "ap_http.h"
|
||||
#include "mpg_decode.h"
|
||||
|
||||
#include <boost/asio/ssl.hpp>
|
||||
#include <boost/pool/object_pool.hpp>
|
||||
#include <boost/bind/bind.hpp>
|
||||
#include <boost/tokenizer.hpp>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
#include <boost/assign/list_of.hpp>
|
||||
|
||||
#define BUFFER_GROW_SIZE (80000)
|
||||
#define BUFFER_THROTTLE_LOW (40000)
|
||||
#define BUFFER_THROTTLE_HIGH (160000)
|
||||
|
||||
bool AudioProducerHttp::initialized = false;
|
||||
boost::asio::io_service AudioProducerHttp::io_service;
|
||||
std::thread AudioProducerHttp::worker_thread;
|
||||
GlobalInfo_t AudioProducerHttp::global;
|
||||
std::map<curl_socket_t, boost::asio::ip::tcp::socket *> AudioProducerHttp::socket_map;
|
||||
boost::asio::deadline_timer AudioProducerHttp::multi_timer(AudioProducerHttp::io_service);
|
||||
|
||||
void AudioProducerHttp::threadFunc() {
|
||||
/* to make sure the event loop doesn't terminate when there is no work to do */
|
||||
io_service.reset() ;
|
||||
boost::asio::io_service::work work(io_service);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "ap_http threadFunc - starting\n");
|
||||
|
||||
for(;;) {
|
||||
|
||||
try {
|
||||
io_service.run() ;
|
||||
break ;
|
||||
}
|
||||
catch( std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "ap_http threadFunc - Error: %s\n", e.what());
|
||||
}
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "ap_http threadFunc - ending\n");
|
||||
}
|
||||
|
||||
void AudioProducerHttp::_init() {
|
||||
if (!initialized) {
|
||||
initialized = true;
|
||||
if (mpg123_init() != MPG123_OK) {
|
||||
throw std::runtime_error("AudioProducerFile::AudioProducerFile: failed to initiate MPG123");
|
||||
return ;
|
||||
}
|
||||
|
||||
memset(&global, 0, sizeof(GlobalInfo_t));
|
||||
global.multi = curl_multi_init();
|
||||
if (!global.multi) {
|
||||
throw std::runtime_error("AudioProducerHttp::_init: failed to initiate CURL multi");
|
||||
return ;
|
||||
}
|
||||
|
||||
curl_multi_setopt(global.multi, CURLMOPT_SOCKETFUNCTION, &AudioProducerHttp::sock_cb);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_SOCKETDATA, &global);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_TIMERFUNCTION, &AudioProducerHttp::multi_timer_cb);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_TIMERDATA, &global);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
||||
|
||||
/* start worker thread */
|
||||
std::thread t(threadFunc) ;
|
||||
worker_thread.swap( t ) ;
|
||||
}
|
||||
}
|
||||
|
||||
void AudioProducerHttp::_deinit() {
|
||||
if (initialized) {
|
||||
initialized = false;
|
||||
io_service.stop();
|
||||
if (worker_thread.joinable()) {
|
||||
worker_thread.join();
|
||||
}
|
||||
/* cleanup curl multi handle*/
|
||||
curl_multi_cleanup(global.multi);
|
||||
|
||||
mpg123_exit();
|
||||
}
|
||||
}
|
||||
|
||||
AudioProducerHttp::AudioProducerHttp(
|
||||
std::mutex& mutex,
|
||||
CircularBuffer_t& circularBuffer,
|
||||
int sampleRate
|
||||
) : AudioProducer(mutex, circularBuffer, sampleRate), _status(Status_t::STATUS_NONE), _mh(nullptr), _easy(nullptr),
|
||||
_error{0}, _response_code(0), _timer(io_service) {
|
||||
|
||||
AudioProducerHttp::_init();
|
||||
}
|
||||
|
||||
AudioProducerHttp::~AudioProducerHttp() {
|
||||
reset();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "AudioProducerFile::~AudioProducerFile %p\n", (void *)this);
|
||||
}
|
||||
|
||||
void AudioProducerHttp::start(std::function<void(bool, const std::string&)> callback) {
|
||||
int mhError = 0;
|
||||
|
||||
_callback = callback;
|
||||
memset(_error, 0, sizeof(_error));
|
||||
|
||||
/* allocate handle for mpeg decoding */
|
||||
_mh = mpg123_new("auto", &mhError);
|
||||
if (!_mh) {
|
||||
const char *mhErr = mpg123_plain_strerror(mhError);
|
||||
throw std::runtime_error("Error allocating mpg123 handle! " + std::string(mhErr));
|
||||
}
|
||||
|
||||
if (mpg123_open_feed(_mh) != MPG123_OK) throw std::runtime_error("Error mpg123_open_feed!\n");
|
||||
if (mpg123_format_all(_mh) != MPG123_OK) throw std::runtime_error("Error mpg123_format_all!\n");
|
||||
if (mpg123_param(_mh, MPG123_FORCE_RATE, _sampleRate, 0) != MPG123_OK) throw std::runtime_error("Error forcing resample to 8k!\n");
|
||||
if (mpg123_param(_mh, MPG123_FLAGS, MPG123_MONO_MIX, 0) != MPG123_OK) throw std::runtime_error("Error forcing single channel!\n");
|
||||
|
||||
_easy = createEasyHandle();
|
||||
if (!_easy) throw std::runtime_error("Error creating easy handle!\n");
|
||||
|
||||
curl_easy_setopt(_easy, CURLOPT_WRITEFUNCTION, &AudioProducerHttp::static_write_cb);
|
||||
curl_easy_setopt(_easy, CURLOPT_URL, _url.c_str());
|
||||
curl_easy_setopt(_easy, CURLOPT_HTTPGET, _method == HttpMethod_t::HTTP_METHOD_GET ? 1L : 0L);
|
||||
curl_easy_setopt(_easy, CURLOPT_WRITEDATA, this);
|
||||
curl_easy_setopt(_easy, CURLOPT_ERRORBUFFER, _error);
|
||||
curl_easy_setopt(_easy, CURLOPT_PRIVATE, this);
|
||||
curl_easy_setopt(_easy, CURLOPT_VERBOSE, 0L);
|
||||
curl_easy_setopt(_easy, CURLOPT_NOPROGRESS, 1L);
|
||||
curl_easy_setopt(_easy, CURLOPT_HEADERFUNCTION, &AudioProducerHttp::static_header_callback);
|
||||
curl_easy_setopt(_easy, CURLOPT_HEADERDATA, this);
|
||||
|
||||
/* call this function to get a socket */
|
||||
curl_easy_setopt(_easy, CURLOPT_OPENSOCKETFUNCTION, &AudioProducerHttp::open_socket);
|
||||
|
||||
/* call this function to close a socket */
|
||||
curl_easy_setopt(_easy, CURLOPT_CLOSESOCKETFUNCTION, &AudioProducerHttp::close_socket);
|
||||
|
||||
// libcurl adding random byte to the response body that creates white noise to audio file
|
||||
// https://github.com/curl/curl/issues/10525
|
||||
const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
|
||||
curl_easy_setopt(_easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
|
||||
|
||||
/* keep the speed down so we don't have to buffer large amounts*/
|
||||
curl_easy_setopt(_easy, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)31415);
|
||||
/*Add request body*/
|
||||
if (!_body.empty()) curl_easy_setopt(_easy, CURLOPT_POSTFIELDS, _body.c_str());
|
||||
/*Add request proxy*/
|
||||
if (!_proxy.empty()) curl_easy_setopt(_easy, CURLOPT_PROXY, _proxy.c_str());
|
||||
|
||||
/*Add request headers*/
|
||||
struct curl_slist *hdr_list = nullptr;
|
||||
for(const auto& header : _headers) {
|
||||
hdr_list = curl_slist_append(hdr_list, header.c_str());
|
||||
}
|
||||
if (hdr_list) curl_easy_setopt(_easy, CURLOPT_HTTPHEADER, hdr_list);
|
||||
|
||||
_status = Status_t::STATUS_AWAITING_RESTART;
|
||||
|
||||
_timer.expires_from_now(boost::posix_time::millisec(1));
|
||||
_timer.async_wait(boost::bind(&AudioProducerHttp::addCurlHandle, this, boost::placeholders::_1));
|
||||
}
|
||||
void AudioProducerHttp::queueHttpGetAudio(const std::string& url, int gain, bool loop) {
|
||||
_method = HttpMethod_t::HTTP_METHOD_GET;
|
||||
_url = url;
|
||||
_gain = gain;
|
||||
_loop = loop;
|
||||
}
|
||||
void AudioProducerHttp::queueHttpPostAudio(const std::string& url, int gain, bool loop) {
|
||||
_method = HttpMethod_t::HTTP_METHOD_POST;
|
||||
_url = url;
|
||||
_gain = gain;
|
||||
_loop = loop;
|
||||
}
|
||||
void AudioProducerHttp::queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, const std::string& proxy, int gain, bool loop) {
|
||||
_method = HttpMethod_t::HTTP_METHOD_POST;
|
||||
_url = url;
|
||||
_body = body;
|
||||
_headers = headers;
|
||||
_proxy = proxy;
|
||||
_gain = gain;
|
||||
_loop = loop;
|
||||
}
|
||||
|
||||
void AudioProducerHttp::addCurlHandle(const boost::system::error_code& error) {
|
||||
if (_status == Status_t::STATUS_AWAITING_RESTART) {
|
||||
auto rc = curl_multi_add_handle(global.multi, _easy);
|
||||
if (mcode_test("new_conn: curl_multi_add_handle", rc) < 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "AudioProducerHttp::addCurlHandle: Error adding easy handle to multi handle\n");
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "AudioProducerHttp::addCurlHandle retrieving from %s\n", _url.c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CURL* AudioProducerHttp::createEasyHandle(void) {
|
||||
CURL* easy = curl_easy_init();
|
||||
if(!easy) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "curl_easy_init() failed!\n");
|
||||
return nullptr ;
|
||||
}
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(easy, CURLOPT_USERAGENT, "jambonz/0.8.5");
|
||||
|
||||
// set connect timeout to 3 seconds and no total timeout as files could be large
|
||||
curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
|
||||
curl_easy_setopt(easy, CURLOPT_TIMEOUT, 0L); // no timeout
|
||||
|
||||
return easy ;
|
||||
}
|
||||
|
||||
size_t AudioProducerHttp::static_write_cb(char* ptr, size_t size, size_t nmemb, void* userdata) {
|
||||
return static_cast<AudioProducerHttp*>(userdata)->write_cb(ptr, size, nmemb);
|
||||
}
|
||||
|
||||
size_t AudioProducerHttp::write_cb(void *ptr, size_t size, size_t nmemb) {
|
||||
int8_t *data = (int8_t *) ptr;
|
||||
size_t bytes_received = size * nmemb;
|
||||
std::vector<int16_t> pcm_data;
|
||||
if (_status == Status_t::STATUS_STOPPING || _status == Status_t::STATUS_STOPPED) {
|
||||
_timer.cancel();
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,
|
||||
// "AudioProducerHttp::write_cb: aborting transfer, status %s, mutex %p, buffer %p\n", status2String(_status));
|
||||
/* this will abort the transfer */
|
||||
return 0;
|
||||
}
|
||||
if (_response_code > 0 && _response_code != 200) {
|
||||
std::string body((char *) ptr, bytes_received);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "AudioProducerHttp::write_cb: received body %s\n", body.c_str());
|
||||
_err_msg = body;
|
||||
_status = Status_t::STATUS_FAILED;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* throttle after reaching high water mark */
|
||||
size_t bufSize = _buffer.size();
|
||||
if (bufSize > BUFFER_THROTTLE_HIGH) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "AudioProducerHttp::write_cb: throttling download, buffer size is %ld\n", _buffer.size());
|
||||
|
||||
// check back in 2 seconds
|
||||
_timer.expires_from_now(boost::posix_time::millisec(2000));
|
||||
_timer.async_wait(boost::bind(&AudioProducerHttp::throttling_cb, this, boost::placeholders::_1));
|
||||
|
||||
_status = Status_t::STATUS_DOWNLOAD_PAUSED;
|
||||
return CURL_WRITEFUNC_PAUSE;
|
||||
}
|
||||
|
||||
pcm_data = convert_mp3_to_linear(_mh, _gain, data, bytes_received);
|
||||
size_t samples = pcm_data.size();
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
|
||||
// Resize the buffer if necessary
|
||||
if (_buffer.capacity() - bufSize < samples) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "AudioProducerHttp::write_cb growing buffer, size now %ld\n", _buffer.size());
|
||||
|
||||
//TODO: if buffer exceeds some max size, return CURL_WRITEFUNC_ERROR to abort the transfer
|
||||
_buffer.set_capacity(_buffer.size() + std::max(samples, (size_t)BUFFER_GROW_SIZE));
|
||||
}
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "AudioProducerHttp::write_cb: writing %ld samples to buffer\n", pcm_data.size());
|
||||
|
||||
/* Push the data into the buffer */
|
||||
_buffer.insert(_buffer.end(), pcm_data.data(), pcm_data.data() + pcm_data.size());
|
||||
return bytes_received;
|
||||
|
||||
}
|
||||
|
||||
size_t AudioProducerHttp::static_header_callback(char *buffer, size_t size, size_t nitems, void* userdata) {
|
||||
return static_cast<AudioProducerHttp*>(userdata)->header_callback(buffer, size, nitems);
|
||||
}
|
||||
|
||||
size_t AudioProducerHttp::header_callback(char *buffer, size_t size, size_t nitems) {
|
||||
size_t bytes_received = size * nitems;
|
||||
const std::string prefix = "HTTP/";
|
||||
std::string header, value;
|
||||
std::string input(buffer, bytes_received);
|
||||
if (parseHeader(input, header, value)) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: %s with value %s\n", header.c_str(), value.c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: %s\n", input.c_str());
|
||||
if (input.rfind(prefix, 0) == 0) {
|
||||
try {
|
||||
_response_code = extract_response_code(input);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: parsed response code: %ld\n", _response_code);
|
||||
} catch (const std::invalid_argument& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "header_callback: invalid response code %s\n", input.substr(prefix.length()).c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
return bytes_received;
|
||||
}
|
||||
|
||||
void AudioProducerHttp::throttling_cb(const boost::system::error_code& error) {
|
||||
if (_status == Status_t::STATUS_STOPPING) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "throttling_cb: session gone, resume download so we can complete\n");
|
||||
curl_easy_pause(_easy, CURLPAUSE_CONT);
|
||||
return;
|
||||
}
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "throttling_cb: status is %s\n", status2String(_status));
|
||||
|
||||
if (!error) {
|
||||
auto size = _buffer.size();
|
||||
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "throttling_cb: size is now %ld\n", size);
|
||||
if (size < BUFFER_THROTTLE_LOW) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "throttling_cb: resuming download\n");
|
||||
curl_easy_pause(_easy, CURLPAUSE_CONT);
|
||||
return;
|
||||
}
|
||||
|
||||
// check back in 2 seconds
|
||||
_timer.expires_from_now(boost::posix_time::millisec(2000));
|
||||
_timer.async_wait(boost::bind(&AudioProducerHttp::throttling_cb, this, boost::placeholders::_1));
|
||||
|
||||
} else if (125 == error.value()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "throttling_cb: timer canceled\n");
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "throttling_cb: error (%d): %s\n", error.value(), error.message().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
int AudioProducerHttp::sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) {
|
||||
GlobalInfo_t *g = &global;
|
||||
|
||||
int *actionp = (int *) sockp;
|
||||
static const char *whatstr[] = { "none", "IN", "OUT", "INOUT", "REMOVE"};
|
||||
|
||||
if(what == CURL_POLL_REMOVE) {
|
||||
*actionp = what;
|
||||
}
|
||||
else {
|
||||
if(!actionp) {
|
||||
addsock(s, e, what, g);
|
||||
}
|
||||
else {
|
||||
setsock(actionp, s, e, what, *actionp, g);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void AudioProducerHttp::timer_cb(const boost::system::error_code & error, GlobalInfo_t *g)
|
||||
{
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "timer_cb\n");
|
||||
|
||||
if(!error) {
|
||||
CURLMcode rc = curl_multi_socket_action(g->multi, CURL_SOCKET_TIMEOUT, 0, &g->still_running);
|
||||
mcode_test("timer_cb: curl_multi_socket_action", rc);
|
||||
check_multi_info(g);
|
||||
}
|
||||
}
|
||||
|
||||
int AudioProducerHttp::multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo_t *g) {
|
||||
|
||||
/* cancel running timer */
|
||||
multi_timer.cancel();
|
||||
|
||||
if(timeout_ms >= 0) {
|
||||
// from libcurl 7.88.1-10+deb12u4 does not allow call curl_multi_socket_action or curl_multi_perform in curl_multi callback directly
|
||||
multi_timer.expires_from_now(boost::posix_time::millisec(timeout_ms ? timeout_ms : 1));
|
||||
multi_timer.async_wait(boost::bind(&timer_cb, boost::placeholders::_1, g));
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
curl_socket_t AudioProducerHttp::open_socket(void *clientp, curlsocktype purpose, struct curl_sockaddr *address) {
|
||||
curl_socket_t sockfd = CURL_SOCKET_BAD;
|
||||
|
||||
/* restrict to IPv4 */
|
||||
if(purpose == CURLSOCKTYPE_IPCXN && address->family == AF_INET) {
|
||||
/* create a tcp socket object */
|
||||
boost::asio::ip::tcp::socket *tcp_socket = new boost::asio::ip::tcp::socket(io_service);
|
||||
|
||||
/* open it and get the native handle*/
|
||||
boost::system::error_code ec;
|
||||
tcp_socket->open(boost::asio::ip::tcp::v4(), ec);
|
||||
|
||||
if(ec) {
|
||||
/* An error occurred */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't open socket [%ld][%s]\n", ec, ec.message().c_str());
|
||||
}
|
||||
else {
|
||||
sockfd = tcp_socket->native_handle();
|
||||
|
||||
/* save it for monitoring */
|
||||
socket_map.insert(std::pair<curl_socket_t, boost::asio::ip::tcp::socket *>(sockfd, tcp_socket));
|
||||
}
|
||||
}
|
||||
return sockfd;
|
||||
}
|
||||
|
||||
int AudioProducerHttp::close_socket(void *clientp, curl_socket_t item) {
|
||||
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(item);
|
||||
if(it != socket_map.end()) {
|
||||
delete it->second;
|
||||
socket_map.erase(it);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int AudioProducerHttp::mcode_test(const char *where, CURLMcode code) {
|
||||
if(CURLM_OK != code) {
|
||||
const char *s;
|
||||
switch(code) {
|
||||
case CURLM_CALL_MULTI_PERFORM:
|
||||
s = "CURLM_CALL_MULTI_PERFORM";
|
||||
break;
|
||||
case CURLM_BAD_HANDLE:
|
||||
s = "CURLM_BAD_HANDLE";
|
||||
break;
|
||||
case CURLM_BAD_EASY_HANDLE:
|
||||
s = "CURLM_BAD_EASY_HANDLE";
|
||||
break;
|
||||
case CURLM_OUT_OF_MEMORY:
|
||||
s = "CURLM_OUT_OF_MEMORY";
|
||||
break;
|
||||
case CURLM_INTERNAL_ERROR:
|
||||
s = "CURLM_INTERNAL_ERROR";
|
||||
break;
|
||||
case CURLM_UNKNOWN_OPTION:
|
||||
s = "CURLM_UNKNOWN_OPTION";
|
||||
break;
|
||||
case CURLM_LAST:
|
||||
s = "CURLM_LAST";
|
||||
break;
|
||||
default:
|
||||
s = "CURLM_unknown";
|
||||
break;
|
||||
case CURLM_BAD_SOCKET:
|
||||
s = "CURLM_BAD_SOCKET";
|
||||
break;
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mcode_test ERROR: %s returns %s:%d\n", where, s, code);
|
||||
|
||||
return -1;
|
||||
}
|
||||
return 0 ;
|
||||
}
|
||||
|
||||
bool AudioProducerHttp::parseHeader(const std::string& str, std::string& header, std::string& value) {
|
||||
std::vector<std::string> parts;
|
||||
boost::split(parts, str, boost::is_any_of(":"), boost::token_compress_on);
|
||||
|
||||
if (parts.size() != 2)
|
||||
return false;
|
||||
|
||||
header = boost::trim_copy(parts[0]);
|
||||
value = boost::trim_copy(parts[1]);
|
||||
return true;
|
||||
}
|
||||
|
||||
int AudioProducerHttp::extract_response_code(const std::string& input) {
|
||||
std::size_t space_pos = input.find(' ');
|
||||
if (space_pos == std::string::npos) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid HTTP response format %s\n", input.c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::size_t code_start_pos = space_pos + 1;
|
||||
std::size_t code_end_pos = input.find(' ', code_start_pos);
|
||||
if (code_end_pos == std::string::npos) {
|
||||
code_end_pos = input.length();
|
||||
}
|
||||
|
||||
std::string code_str = input.substr(code_start_pos, code_end_pos - code_start_pos);
|
||||
int response_code = std::stoi(code_str);
|
||||
return response_code;
|
||||
}
|
||||
|
||||
void AudioProducerHttp::setsock(int *fdp, curl_socket_t s, CURL *e, int act, int oldact, GlobalInfo_t *g) {
|
||||
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(s);
|
||||
|
||||
if(it == socket_map.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
boost::asio::ip::tcp::socket * tcp_socket = it->second;
|
||||
|
||||
*fdp = act;
|
||||
|
||||
if(act == CURL_POLL_IN) {
|
||||
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_IN, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
else if(act == CURL_POLL_OUT) {
|
||||
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_OUT, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
else if(act == CURL_POLL_INOUT) {
|
||||
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_IN, boost::placeholders::_1, fdp));
|
||||
}
|
||||
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_OUT, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Called by asio when there is an action on a socket */
|
||||
void AudioProducerHttp::event_cb(GlobalInfo_t *g, curl_socket_t s, int action, const boost::system::error_code & error, int *fdp) {
|
||||
int f = *fdp;
|
||||
|
||||
|
||||
// Socket already POOL REMOVED.
|
||||
if (f == CURL_POLL_REMOVE) {
|
||||
remsock(fdp, g);
|
||||
return;
|
||||
}
|
||||
|
||||
if(socket_map.find(s) == socket_map.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* make sure the event matches what are wanted */
|
||||
if(f == action || f == CURL_POLL_INOUT) {
|
||||
if(error) {
|
||||
action = CURL_CSELECT_ERR;
|
||||
}
|
||||
CURLMcode rc = curl_multi_socket_action(g->multi, s, action, &g->still_running);
|
||||
|
||||
mcode_test("event_cb: curl_multi_socket_action", rc);
|
||||
check_multi_info(g);
|
||||
|
||||
if(g->still_running <= 0) {
|
||||
multi_timer.cancel();
|
||||
}
|
||||
|
||||
/* keep on watching.
|
||||
* the socket may have been closed and/or fdp may have been changed
|
||||
* in curl_multi_socket_action(), so check them both */
|
||||
if(!error && socket_map.find(s) != socket_map.end() &&
|
||||
(f == action || f == CURL_POLL_INOUT)) {
|
||||
boost::asio::ip::tcp::socket *tcp_socket = socket_map.find(s)->second;
|
||||
|
||||
if(action == CURL_POLL_IN) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
action, boost::placeholders::_1, fdp));
|
||||
}
|
||||
if(action == CURL_POLL_OUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
action, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AudioProducerHttp::remsock(int *f, GlobalInfo_t *g) {
|
||||
if(f) {
|
||||
free(f);
|
||||
f = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
void AudioProducerHttp::addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo_t *g) {
|
||||
/* fdp is used to store current action */
|
||||
int *fdp = (int *) calloc(sizeof(int), 1);
|
||||
|
||||
setsock(fdp, s, easy, action, 0, g);
|
||||
curl_multi_assign(g->multi, s, fdp);
|
||||
}
|
||||
|
||||
/* Check for completed transfers, and remove their easy handles */
|
||||
void AudioProducerHttp::check_multi_info(GlobalInfo_t *g) {
|
||||
CURLMsg *msg;
|
||||
int msgs_left;
|
||||
AudioProducerHttp *ap;
|
||||
CURL *easy;
|
||||
CURLcode res;
|
||||
|
||||
while((msg = curl_multi_info_read(g->multi, &msgs_left))) {
|
||||
if(msg->msg == CURLMSG_DONE) {
|
||||
long response_code;
|
||||
double namelookup=0, connect=0, total=0 ;
|
||||
char *ct = NULL ;
|
||||
|
||||
easy = msg->easy_handle;
|
||||
res = msg->data.result;
|
||||
curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ap);
|
||||
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &response_code);
|
||||
curl_easy_getinfo(easy, CURLINFO_CONTENT_TYPE, &ct);
|
||||
|
||||
curl_easy_getinfo(easy, CURLINFO_NAMELOOKUP_TIME, &namelookup);
|
||||
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connect);
|
||||
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &total);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "curl done, response code %d, status %s\n", response_code, status2String(ap->getStatus()));
|
||||
|
||||
bool restart = ap->isLoopedAudio() && ap->getStatus() != Status_t::STATUS_STOPPING && response_code == 200;
|
||||
if (restart) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "restarting looped audio\n");
|
||||
ap->getTimer().expires_from_now(boost::posix_time::millisec(1000));
|
||||
ap->getTimer().async_wait(boost::bind(&AudioProducerHttp::static_restart_cb, boost::placeholders::_1, ap));
|
||||
}
|
||||
else {
|
||||
ap->cleanup(Status_t::STATUS_DOWNLOAD_COMPLETE, (int) response_code);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AudioProducerHttp::static_restart_cb(const boost::system::error_code& error, void* userdata) {
|
||||
static_cast<AudioProducerHttp*>(userdata)->restart_cb(error);
|
||||
}
|
||||
|
||||
void AudioProducerHttp::restart_cb(const boost::system::error_code& error) {
|
||||
if (_status == Status_t::STATUS_STOPPING) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "restart_cb: session gone\n");
|
||||
return;
|
||||
}
|
||||
if (!error) {
|
||||
reset();
|
||||
start(_callback);
|
||||
}
|
||||
else if (error.value() == boost::asio::error::operation_aborted) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "restart_cb: %s, cancelling retrieve\n", error.message().c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "restart_cb: error (%d): %s\n", error.value(), error.message().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void AudioProducerHttp::stop() {
|
||||
cleanup(Status_t::STATUS_STOPPED, 0);
|
||||
return;
|
||||
}
|
||||
|
||||
void AudioProducerHttp::reset() {
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
if (_easy) {
|
||||
curl_multi_remove_handle(global.multi, _easy);
|
||||
curl_easy_cleanup(_easy);
|
||||
_easy = nullptr;
|
||||
}
|
||||
if (_mh) {
|
||||
mpg123_close(_mh);
|
||||
mpg123_delete(_mh);
|
||||
_mh = nullptr;
|
||||
}
|
||||
}
|
||||
_err_msg.clear();
|
||||
_response_code = 0;
|
||||
_timer.cancel();
|
||||
_status = Status_t::STATUS_NONE;
|
||||
}
|
||||
|
||||
void AudioProducerHttp::cleanup(Status_t status, int response_code) {
|
||||
std::string errMsg = response_code > 200 ? "http response: " + std::to_string(response_code) : "";
|
||||
reset();
|
||||
_status = status;
|
||||
notifyDone(!errMsg.empty(), errMsg);
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
#ifndef __AP_HTTP_H__
|
||||
#define __AP_HTTP_H__
|
||||
|
||||
#include "ap.h"
|
||||
#include <curl/curl.h>
|
||||
#include <mpg123.h>
|
||||
#include <boost/asio.hpp>
|
||||
|
||||
|
||||
typedef struct
|
||||
{
|
||||
CURLM *multi;
|
||||
int still_running;
|
||||
} GlobalInfo_t;
|
||||
|
||||
class AudioProducerHttp : public AudioProducer {
|
||||
public:
|
||||
|
||||
typedef enum
|
||||
{
|
||||
STATUS_NONE = 0,
|
||||
STATUS_FAILED,
|
||||
STATUS_DOWNLOAD_IN_PROGRESS,
|
||||
STATUS_DOWNLOAD_PAUSED,
|
||||
STATUS_DOWNLOAD_COMPLETE,
|
||||
STATUS_AWAITING_RESTART,
|
||||
STATUS_STOPPING,
|
||||
STATUS_STOPPED
|
||||
} Status_t;
|
||||
|
||||
typedef enum {
|
||||
HTTP_METHOD_GET = 0,
|
||||
HTTP_METHOD_POST
|
||||
} HttpMethod_t;
|
||||
|
||||
static const char* status2String(Status_t status) {
|
||||
static const char* statusStrings[] = {
|
||||
"STATUS_NONE",
|
||||
"STATUS_FAILED",
|
||||
"STATUS_DOWNLOAD_IN_PROGRESS",
|
||||
"STATUS_DOWNLOAD_PAUSED",
|
||||
"STATUS_DOWNLOAD_COMPLETE",
|
||||
"STATUS_AWAITING_RESTART",
|
||||
"STATUS_STOPPED"
|
||||
};
|
||||
|
||||
if (status >= 0 && status < sizeof(statusStrings) / sizeof(statusStrings[0]))
|
||||
{
|
||||
return statusStrings[status];
|
||||
}
|
||||
else
|
||||
{
|
||||
return "UNKNOWN_STATUS";
|
||||
}
|
||||
}
|
||||
AudioProducerHttp(
|
||||
std::mutex& mutex,
|
||||
CircularBuffer_t& circularBuffer,
|
||||
int sampleRate
|
||||
);
|
||||
virtual ~AudioProducerHttp();
|
||||
|
||||
virtual void start(std::function<void(bool, const std::string&)> callback);
|
||||
void addCurlHandle(const boost::system::error_code& error);
|
||||
virtual void stop();
|
||||
void cleanup(Status_t status, int response_code);
|
||||
void reset();
|
||||
|
||||
void queueHttpGetAudio(const std::string& url, int gain = 0, bool loop = false);
|
||||
void queueHttpPostAudio(const std::string& url, int gain = 0, bool loop = false);
|
||||
void queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, const std::string& proxy, int gain = 0, bool loop = false);
|
||||
|
||||
Status_t getStatus() const { return _status; }
|
||||
void setStatus(Status_t status) { _status = status; }
|
||||
|
||||
boost::asio::deadline_timer& getTimer() { return _timer; }
|
||||
|
||||
static bool initialized;
|
||||
static std::thread worker_thread;
|
||||
static boost::asio::io_service io_service;
|
||||
static void threadFunc();
|
||||
static std::map<curl_socket_t, boost::asio::ip::tcp::socket *> socket_map;
|
||||
static boost::asio::deadline_timer multi_timer;
|
||||
|
||||
|
||||
static GlobalInfo_t global;
|
||||
|
||||
static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp);
|
||||
static int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo_t *g);
|
||||
static void timer_cb(const boost::system::error_code & error, GlobalInfo_t *g);
|
||||
static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo_t *g);
|
||||
static void setsock(int *fdp, curl_socket_t s, CURL *e, int act, int oldact, GlobalInfo_t *g);
|
||||
static int mcode_test(const char *where, CURLMcode code);
|
||||
static void check_multi_info(GlobalInfo_t *g);
|
||||
static void event_cb(GlobalInfo_t *g, curl_socket_t s, int action, const boost::system::error_code & error, int *fdp);
|
||||
static void remsock(int *f, GlobalInfo_t *g);
|
||||
|
||||
static size_t static_write_cb(char* ptr, size_t size, size_t nmemb, void* userdata);
|
||||
size_t write_cb(void *ptr, size_t size, size_t nmemb);
|
||||
|
||||
static size_t static_header_callback(char *buffer, size_t size, size_t nitems, void* userdata);
|
||||
size_t header_callback(char *buffer, size_t size, size_t nitems);
|
||||
|
||||
void throttling_cb(const boost::system::error_code& error);
|
||||
|
||||
static void static_restart_cb(const boost::system::error_code& error, void* userdata);
|
||||
void restart_cb(const boost::system::error_code& error);
|
||||
|
||||
static int close_socket(void *clientp, curl_socket_t item);
|
||||
static curl_socket_t open_socket(void *clientp, curlsocktype purpose, struct curl_sockaddr *address);
|
||||
|
||||
private:
|
||||
|
||||
static void _init();
|
||||
static void _deinit();
|
||||
|
||||
CURL* createEasyHandle();
|
||||
bool parseHeader(const std::string& str, std::string& header, std::string& value);
|
||||
int extract_response_code(const std::string& input);
|
||||
|
||||
HttpMethod_t _method;
|
||||
std::string _url;
|
||||
std::string _body;
|
||||
std::string _proxy;
|
||||
std::vector<std::string> _headers;
|
||||
Status_t _status;
|
||||
mpg123_handle *_mh;
|
||||
CURL *_easy;
|
||||
char _error[CURL_ERROR_SIZE]; // curl error buffer
|
||||
std::string _err_msg;
|
||||
int _response_code;
|
||||
boost::asio::deadline_timer _timer;
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -1,885 +0,0 @@
|
||||
#include "audio_downloader.h"
|
||||
|
||||
#include <boost/thread.hpp>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/asio/ssl.hpp>
|
||||
#include <boost/pool/object_pool.hpp>
|
||||
#include <boost/bind/bind.hpp>
|
||||
#include <boost/tokenizer.hpp>
|
||||
#include <boost/foreach.hpp>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/assign/list_of.hpp>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
|
||||
#include <map>
|
||||
|
||||
#include <mpg123.h>
|
||||
|
||||
#include <curl/curl.h>
|
||||
|
||||
#define BUFFER_GROW_SIZE (80000)
|
||||
#define BUFFER_THROTTLE_LOW (40000)
|
||||
#define BUFFER_THROTTLE_HIGH (160000)
|
||||
|
||||
static uint16_t currDownloadId = 0;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
CURLM *multi;
|
||||
int still_running;
|
||||
} GlobalInfo_t;
|
||||
static GlobalInfo_t global;
|
||||
|
||||
typedef enum
|
||||
{
|
||||
STATUS_NONE = 0,
|
||||
STATUS_FAILED,
|
||||
STATUS_DOWNLOAD_IN_PROGRESS,
|
||||
STATUS_DOWNLOAD_PAUSED,
|
||||
STATUS_DOWNLOAD_COMPLETE,
|
||||
STATUS_AWAITING_RESTART,
|
||||
STATUS_STOPPING,
|
||||
STATUS_STOPPED
|
||||
} Status_t;
|
||||
|
||||
static const char* status2String(Status_t status)
|
||||
{
|
||||
static const char* statusStrings[] = {
|
||||
"STATUS_NONE",
|
||||
"STATUS_FAILED",
|
||||
"STATUS_DOWNLOAD_IN_PROGRESS",
|
||||
"STATUS_DOWNLOAD_PAUSED",
|
||||
"STATUS_DOWNLOAD_COMPLETE",
|
||||
"STATUS_AWAITING_RESTART",
|
||||
"STATUS_STOPPING",
|
||||
"STATUS_STOPPED"
|
||||
};
|
||||
|
||||
if (status >= 0 && status < sizeof(statusStrings) / sizeof(statusStrings[0]))
|
||||
{
|
||||
return statusStrings[status];
|
||||
}
|
||||
else
|
||||
{
|
||||
return "UNKNOWN_STATUS";
|
||||
}
|
||||
}
|
||||
|
||||
typedef struct
|
||||
{
|
||||
GlobalInfo_t *global;
|
||||
CURL *easy;
|
||||
switch_mutex_t* mutex;
|
||||
CircularBuffer_t* buffer;
|
||||
mpg123_handle *mh;
|
||||
char error[CURL_ERROR_SIZE]; // curl error buffer
|
||||
char *err_msg; // http server error message
|
||||
char* url;
|
||||
bool loop;
|
||||
int rate;
|
||||
boost::asio::deadline_timer *timer;
|
||||
Status_t status;
|
||||
downloadId_t id;
|
||||
int response_code;
|
||||
int gain;
|
||||
} ConnInfo_t;
|
||||
|
||||
typedef std::map<int32_t, ConnInfo_t *> Id2ConnMap_t;
|
||||
static Id2ConnMap_t id2ConnMap;
|
||||
|
||||
static boost::object_pool<ConnInfo_t> pool ;
|
||||
static std::map<curl_socket_t, boost::asio::ip::tcp::socket *> socket_map;
|
||||
static boost::asio::io_service io_service;
|
||||
static boost::asio::deadline_timer timer(io_service);
|
||||
static std::string fullDirPath;
|
||||
static std::thread worker_thread;
|
||||
|
||||
/* forward declarations */
|
||||
static ConnInfo_t* createDownloader(const char *url, int rate, int loop, int gain, mpg123_handle *mhm, switch_mutex_t *mutex, CircularBuffer_t *buffer);
|
||||
static CURL* createEasyHandle(void);
|
||||
static void destroyConnection(ConnInfo_t *conn);
|
||||
static void check_multi_info(GlobalInfo_t *g) ;
|
||||
static int mcode_test(const char *where, CURLMcode code);
|
||||
static void event_cb(GlobalInfo_t *g, curl_socket_t s, int action, const boost::system::error_code & error, int *fdp);
|
||||
static void setsock(int *fdp, curl_socket_t s, CURL *e, int act, int oldact, GlobalInfo_t *g);
|
||||
static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo_t *g);
|
||||
static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp);
|
||||
static void threadFunc();
|
||||
static void timer_cb(const boost::system::error_code & error, GlobalInfo_t *g);
|
||||
static int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo_t *g);
|
||||
static std::vector<int16_t> convert_mp3_to_linear(ConnInfo_t *conn, int8_t *data, size_t len);
|
||||
static void throttling_cb(const boost::system::error_code& error, ConnInfo_t* conn) ;
|
||||
static void restart_cb(const boost::system::error_code& error, ConnInfo_t* conn);
|
||||
static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn);
|
||||
static bool parseHeader(const std::string& str, std::string& header, std::string& value) ;
|
||||
static int extract_response_code(const std::string& input) ;
|
||||
static size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo_t *conn);
|
||||
static curl_socket_t opensocket(void *clientp, curlsocktype purpose, struct curl_sockaddr *address);
|
||||
static int close_socket(void *clientp, curl_socket_t item);
|
||||
|
||||
/* apis */
|
||||
extern "C" {
|
||||
|
||||
switch_status_t init_audio_downloader() {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "init_audio_downloader loading..\n");
|
||||
memset(&global, 0, sizeof(GlobalInfo_t));
|
||||
global.multi = curl_multi_init();
|
||||
|
||||
if (!global.multi) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "init_audio_downloader curl_multi_init() failed, exiting!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
curl_multi_setopt(global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_SOCKETDATA, &global);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_TIMERDATA, &global);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
||||
|
||||
if (mpg123_init() != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "init_audio_downloader: failed to initiate MPG123");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
/* start worker thread */
|
||||
std::thread t(threadFunc) ;
|
||||
worker_thread.swap( t ) ;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "init_audio_downloader: loaded\n");
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
|
||||
}
|
||||
|
||||
switch_status_t deinit_audio_downloader() {
|
||||
/* stop the ASIO IO service */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deinit_audio_downloader: stopping io service\n");
|
||||
io_service.stop();
|
||||
|
||||
/* Join the worker thread */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deinit_audio_downloader: wait for worker thread to complete\n");
|
||||
if (worker_thread.joinable()) {
|
||||
worker_thread.join();
|
||||
}
|
||||
|
||||
/* cleanup curl multi handle*/
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deinit_audio_downloader: release curl multi\n");
|
||||
curl_multi_cleanup(global.multi);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deinit_audio_downloader: completed\n");
|
||||
|
||||
mpg123_exit();
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
downloadId_t start_audio_download(const char* url, int rate, int loop, int gain, switch_mutex_t* mutex, CircularBuffer_t* buffer) {
|
||||
int mhError = 0;
|
||||
|
||||
/* allocate handle for mpeg decoding */
|
||||
mpg123_handle *mh = mpg123_new("auto", &mhError);
|
||||
if (!mh) {
|
||||
const char *mhErr = mpg123_plain_strerror(mhError);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error allocating mpg123 handle! %s\n", switch_str_nil(mhErr));
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
if (mpg123_open_feed(mh) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_open_feed!\n");
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
if (mpg123_format_all(mh) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_format_all!\n");
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
if (mpg123_param(mh, MPG123_FORCE_RATE, rate, 0) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error forcing resample to 8k!\n");
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
if (mpg123_param(mh, MPG123_FLAGS, MPG123_MONO_MIX, 0) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error forcing single channel!\n");
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
ConnInfo_t* conn = createDownloader(url, rate, loop, gain, mh, mutex, buffer);
|
||||
if (!conn) {
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,
|
||||
"start_audio_download: starting download %d\n", conn->id);
|
||||
|
||||
|
||||
return conn->id;
|
||||
}
|
||||
|
||||
switch_status_t stop_audio_download(int id) {
|
||||
auto it = id2ConnMap.find(id);
|
||||
if (it == id2ConnMap.end()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "stop_audio_download: id %d has already completed\n", id);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,
|
||||
"stop_audio_download: stopping download %d, status %s\n", id, status2String(it->second->status));
|
||||
|
||||
ConnInfo_t *conn = it->second;
|
||||
auto status = conn->status;
|
||||
|
||||
/* past this point I shall not access either the mutex or the buffer provided */
|
||||
conn->mutex = nullptr;
|
||||
conn->buffer = nullptr;
|
||||
|
||||
/* if download is in progress set status to cancel it during next call back */
|
||||
if (status == Status_t::STATUS_DOWNLOAD_PAUSED) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "stop_audio_download: resuming download %d so we can cancel it\n", id);
|
||||
conn->status = Status_t::STATUS_STOPPING;
|
||||
curl_easy_pause(conn->easy, CURLPAUSE_CONT);
|
||||
}
|
||||
if (status != Status_t::STATUS_DOWNLOAD_IN_PROGRESS) {
|
||||
destroyConnection(conn);
|
||||
}
|
||||
conn->status = Status_t::STATUS_STOPPING;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
/* internal */
|
||||
ConnInfo_t* createDownloader(const char *url, int rate, int loop, int gain, mpg123_handle *mh, switch_mutex_t *mutex, CircularBuffer_t *buffer) {
|
||||
ConnInfo_t *conn = pool.malloc() ;
|
||||
CURL* easy = createEasyHandle();
|
||||
|
||||
if (!easy || !conn) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "createDownloader: failed to allocate memory\n");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
memset(conn, 0, sizeof(ConnInfo_t));
|
||||
conn->easy = easy;
|
||||
conn->mutex = mutex;
|
||||
conn->buffer = buffer;
|
||||
conn->mh = mh;
|
||||
conn->loop = loop;
|
||||
conn->gain = gain;
|
||||
conn->rate = rate;
|
||||
conn->url = strdup(url);
|
||||
conn->global = &global;
|
||||
conn->status = Status_t::STATUS_NONE;
|
||||
conn->timer = new boost::asio::deadline_timer(io_service);
|
||||
|
||||
downloadId_t id = ++currDownloadId;
|
||||
if (id == 0) id++;
|
||||
|
||||
id2ConnMap[id] = conn;
|
||||
conn->id = id;
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_URL, url);
|
||||
curl_easy_setopt(easy, CURLOPT_HTTPGET, 1L);
|
||||
curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, write_cb);
|
||||
curl_easy_setopt(easy, CURLOPT_WRITEDATA, conn);
|
||||
curl_easy_setopt(easy, CURLOPT_ERRORBUFFER, conn->error);
|
||||
curl_easy_setopt(easy, CURLOPT_PRIVATE, conn);
|
||||
curl_easy_setopt(easy, CURLOPT_VERBOSE, 0L);
|
||||
curl_easy_setopt(easy, CURLOPT_NOPROGRESS, 1L);
|
||||
curl_easy_setopt(easy, CURLOPT_HEADERFUNCTION, header_callback);
|
||||
curl_easy_setopt(easy, CURLOPT_HEADERDATA, conn);
|
||||
|
||||
/* call this function to get a socket */
|
||||
curl_easy_setopt(easy, CURLOPT_OPENSOCKETFUNCTION, opensocket);
|
||||
|
||||
/* call this function to close a socket */
|
||||
curl_easy_setopt(easy, CURLOPT_CLOSESOCKETFUNCTION, close_socket);
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
|
||||
|
||||
/* keep the speed down so we don't have to buffer large amounts*/
|
||||
curl_easy_setopt(easy, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)31415);
|
||||
|
||||
auto rc = curl_multi_add_handle(global.multi, conn->easy);
|
||||
if (mcode_test("new_conn: curl_multi_add_handle", rc) < 0) {
|
||||
return nullptr;
|
||||
}
|
||||
conn->status = Status_t::STATUS_DOWNLOAD_IN_PROGRESS;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "createDownloader: launched request, gain %d\n", conn->gain);
|
||||
return conn;
|
||||
}
|
||||
|
||||
void destroyConnection(ConnInfo_t *conn) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "destroyConnection\n");
|
||||
|
||||
/* clean up the curl handle*/
|
||||
curl_multi_remove_handle(conn->global, conn->easy);
|
||||
curl_easy_cleanup(conn->easy);
|
||||
|
||||
/* clear asio resources and free resources */
|
||||
if (conn->timer) {
|
||||
conn->timer->cancel();
|
||||
delete conn->timer;
|
||||
}
|
||||
if (conn->err_msg) {
|
||||
free(conn->err_msg);
|
||||
}
|
||||
|
||||
/* free mp3 decoder */
|
||||
if (conn->mh) {
|
||||
mpg123_close(conn->mh);
|
||||
mpg123_delete(conn->mh);
|
||||
}
|
||||
|
||||
if (conn->url) {
|
||||
free(conn->url);
|
||||
}
|
||||
|
||||
if (conn->mutex) switch_mutex_lock(conn->mutex);
|
||||
id2ConnMap.erase(conn->id);
|
||||
if (conn->mutex) switch_mutex_unlock(conn->mutex);
|
||||
|
||||
memset(conn, 0, sizeof(ConnInfo_t));
|
||||
pool.destroy(conn) ;
|
||||
}
|
||||
|
||||
CURL* createEasyHandle(void) {
|
||||
CURL* easy = curl_easy_init();
|
||||
if(!easy) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "curl_easy_init() failed!\n");
|
||||
return nullptr ;
|
||||
}
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(easy, CURLOPT_USERAGENT, "jambonz/0.8.5");
|
||||
|
||||
// set connect timeout to 3 seconds and no total timeout as files could be large
|
||||
curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
|
||||
curl_easy_setopt(easy, CURLOPT_TIMEOUT, 0L); // no timeout
|
||||
|
||||
return easy ;
|
||||
}
|
||||
|
||||
/* Check for completed transfers, and remove their easy handles */
|
||||
void check_multi_info(GlobalInfo_t *g) {
|
||||
CURLMsg *msg;
|
||||
int msgs_left;
|
||||
ConnInfo_t *conn;
|
||||
CURL *easy;
|
||||
CURLcode res;
|
||||
|
||||
while((msg = curl_multi_info_read(g->multi, &msgs_left))) {
|
||||
if(msg->msg == CURLMSG_DONE) {
|
||||
long response_code;
|
||||
double namelookup=0, connect=0, total=0 ;
|
||||
char *ct = NULL ;
|
||||
|
||||
easy = msg->easy_handle;
|
||||
res = msg->data.result;
|
||||
curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
|
||||
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &response_code);
|
||||
curl_easy_getinfo(easy, CURLINFO_CONTENT_TYPE, &ct);
|
||||
|
||||
curl_easy_getinfo(easy, CURLINFO_NAMELOOKUP_TIME, &namelookup);
|
||||
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connect);
|
||||
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &total);
|
||||
|
||||
downloadId_t id = conn->id;
|
||||
auto mutex = conn->mutex;
|
||||
auto buffer = conn->buffer;
|
||||
auto rate = conn->rate;
|
||||
auto loop = conn->loop;
|
||||
auto gain = conn->gain;
|
||||
auto oldId = conn->id;
|
||||
bool restart = conn->loop && conn->status != Status_t::STATUS_STOPPING && response_code == 200;
|
||||
|
||||
conn->response_code = response_code;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "curl done, response code %d, status %s\n", response_code, status2String(conn->status));
|
||||
conn->status = Status_t::STATUS_DOWNLOAD_COMPLETE;
|
||||
|
||||
curl_multi_remove_handle(g->multi, easy);
|
||||
|
||||
if (restart) {
|
||||
conn->status = Status_t::STATUS_AWAITING_RESTART;
|
||||
conn->timer->expires_from_now(boost::posix_time::millisec(1000));
|
||||
conn->timer->async_wait(boost::bind(&restart_cb, boost::placeholders::_1, conn));
|
||||
|
||||
//TODO: this seems to not be working from this callback; maybe start it from a timer callback?
|
||||
}
|
||||
else {
|
||||
destroyConnection(conn);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int mcode_test(const char *where, CURLMcode code) {
|
||||
if(CURLM_OK != code) {
|
||||
const char *s;
|
||||
switch(code) {
|
||||
case CURLM_CALL_MULTI_PERFORM:
|
||||
s = "CURLM_CALL_MULTI_PERFORM";
|
||||
break;
|
||||
case CURLM_BAD_HANDLE:
|
||||
s = "CURLM_BAD_HANDLE";
|
||||
break;
|
||||
case CURLM_BAD_EASY_HANDLE:
|
||||
s = "CURLM_BAD_EASY_HANDLE";
|
||||
break;
|
||||
case CURLM_OUT_OF_MEMORY:
|
||||
s = "CURLM_OUT_OF_MEMORY";
|
||||
break;
|
||||
case CURLM_INTERNAL_ERROR:
|
||||
s = "CURLM_INTERNAL_ERROR";
|
||||
break;
|
||||
case CURLM_UNKNOWN_OPTION:
|
||||
s = "CURLM_UNKNOWN_OPTION";
|
||||
break;
|
||||
case CURLM_LAST:
|
||||
s = "CURLM_LAST";
|
||||
break;
|
||||
default:
|
||||
s = "CURLM_unknown";
|
||||
break;
|
||||
case CURLM_BAD_SOCKET:
|
||||
s = "CURLM_BAD_SOCKET";
|
||||
break;
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mcode_test ERROR: %s returns %s:%d\n", where, s, code);
|
||||
|
||||
return -1;
|
||||
}
|
||||
return 0 ;
|
||||
}
|
||||
|
||||
void remsock(int *f, GlobalInfo_t *g) {
|
||||
if(f) {
|
||||
free(f);
|
||||
f = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/* Called by asio when there is an action on a socket */
|
||||
void event_cb(GlobalInfo_t *g, curl_socket_t s, int action, const boost::system::error_code & error, int *fdp) {
|
||||
int f = *fdp;
|
||||
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X has action %d\n", s, action) ;
|
||||
|
||||
// Socket already POOL REMOVED.
|
||||
if (f == CURL_POLL_REMOVE) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X removed\n", s);
|
||||
remsock(fdp, g);
|
||||
return;
|
||||
}
|
||||
|
||||
if(socket_map.find(s) == socket_map.end()) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb: socket %#X already closed\n, s");
|
||||
return;
|
||||
}
|
||||
|
||||
/* make sure the event matches what are wanted */
|
||||
if(f == action || f == CURL_POLL_INOUT) {
|
||||
if(error) {
|
||||
action = CURL_CSELECT_ERR;
|
||||
}
|
||||
CURLMcode rc = curl_multi_socket_action(g->multi, s, action, &g->still_running);
|
||||
|
||||
mcode_test("event_cb: curl_multi_socket_action", rc);
|
||||
check_multi_info(g);
|
||||
|
||||
if(g->still_running <= 0) {
|
||||
timer.cancel();
|
||||
}
|
||||
|
||||
/* keep on watching.
|
||||
* the socket may have been closed and/or fdp may have been changed
|
||||
* in curl_multi_socket_action(), so check them both */
|
||||
if(!error && socket_map.find(s) != socket_map.end() &&
|
||||
(f == action || f == CURL_POLL_INOUT)) {
|
||||
boost::asio::ip::tcp::socket *tcp_socket = socket_map.find(s)->second;
|
||||
|
||||
if(action == CURL_POLL_IN) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
action, boost::placeholders::_1, fdp));
|
||||
}
|
||||
if(action == CURL_POLL_OUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
action, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* socket functions */
|
||||
void setsock(int *fdp, curl_socket_t s, CURL *e, int act, int oldact, GlobalInfo_t *g) {
|
||||
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(s);
|
||||
|
||||
if(it == socket_map.end()) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setsock: socket %#X not found\n, s");
|
||||
return;
|
||||
}
|
||||
|
||||
boost::asio::ip::tcp::socket * tcp_socket = it->second;
|
||||
|
||||
*fdp = act;
|
||||
|
||||
if(act == CURL_POLL_IN) {
|
||||
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_IN, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
else if(act == CURL_POLL_OUT) {
|
||||
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_OUT, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
else if(act == CURL_POLL_INOUT) {
|
||||
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_IN, boost::placeholders::_1, fdp));
|
||||
}
|
||||
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_OUT, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo_t *g) {
|
||||
/* fdp is used to store current action */
|
||||
int *fdp = (int *) calloc(sizeof(int), 1);
|
||||
|
||||
setsock(fdp, s, easy, action, 0, g);
|
||||
curl_multi_assign(g->multi, s, fdp);
|
||||
}
|
||||
|
||||
int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) {
|
||||
GlobalInfo_t *g = &global;
|
||||
|
||||
int *actionp = (int *) sockp;
|
||||
static const char *whatstr[] = { "none", "IN", "OUT", "INOUT", "REMOVE"};
|
||||
|
||||
if(what == CURL_POLL_REMOVE) {
|
||||
*actionp = what;
|
||||
}
|
||||
else {
|
||||
if(!actionp) {
|
||||
addsock(s, e, what, g);
|
||||
}
|
||||
else {
|
||||
setsock(actionp, s, e, what, *actionp, g);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void threadFunc() {
|
||||
/* to make sure the event loop doesn't terminate when there is no work to do */
|
||||
io_service.reset() ;
|
||||
boost::asio::io_service::work work(io_service);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_dub threadFunc - starting\n");
|
||||
|
||||
for(;;) {
|
||||
|
||||
try {
|
||||
io_service.run() ;
|
||||
break ;
|
||||
}
|
||||
catch( std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mod_dub threadFunc - Error: %s\n", e.what());
|
||||
}
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_dub threadFunc - ending\n");
|
||||
}
|
||||
|
||||
|
||||
/* Called by asio when our timeout expires */
|
||||
void timer_cb(const boost::system::error_code & error, GlobalInfo_t *g)
|
||||
{
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "timer_cb\n");
|
||||
|
||||
if(!error) {
|
||||
CURLMcode rc = curl_multi_socket_action(g->multi, CURL_SOCKET_TIMEOUT, 0, &g->still_running);
|
||||
mcode_test("timer_cb: curl_multi_socket_action", rc);
|
||||
check_multi_info(g);
|
||||
}
|
||||
}
|
||||
|
||||
int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo_t *g) {
|
||||
|
||||
/* cancel running timer */
|
||||
timer.cancel();
|
||||
|
||||
if(timeout_ms >= 0) {
|
||||
// from libcurl 7.88.1-10+deb12u4 does not allow call curl_multi_socket_action or curl_multi_perform in curl_multi callback directly
|
||||
timer.expires_from_now(boost::posix_time::millisec(timeout_ms ? timeout_ms : 1));
|
||||
timer.async_wait(boost::bind(&timer_cb, boost::placeholders::_1, g));
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::vector<int16_t> convert_mp3_to_linear(ConnInfo_t *conn, int8_t *data, size_t len) {
|
||||
std::vector<int16_t> linear_data;
|
||||
int eof = 0;
|
||||
int mp3err = 0;
|
||||
|
||||
if(mpg123_feed(conn->mh, (const unsigned char*) data, len) == MPG123_OK) {
|
||||
while(!eof) {
|
||||
size_t usedlen = 0;
|
||||
off_t frame_offset;
|
||||
unsigned char* audio;
|
||||
|
||||
int decode_status = mpg123_decode_frame(conn->mh, &frame_offset, &audio, &usedlen);
|
||||
|
||||
switch(decode_status) {
|
||||
case MPG123_NEW_FORMAT:
|
||||
continue;
|
||||
|
||||
case MPG123_OK:
|
||||
{
|
||||
size_t samples = usedlen / sizeof(int16_t);
|
||||
linear_data.insert(linear_data.end(), reinterpret_cast<int16_t*>(audio), reinterpret_cast<int16_t*>(audio) + samples);
|
||||
}
|
||||
break;
|
||||
|
||||
case MPG123_DONE:
|
||||
case MPG123_NEED_MORE:
|
||||
eof = 1;
|
||||
break;
|
||||
|
||||
case MPG123_ERR:
|
||||
default:
|
||||
if(++mp3err >= 5) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Decoder Error!\n");
|
||||
eof = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (eof)
|
||||
break;
|
||||
|
||||
mp3err = 0;
|
||||
}
|
||||
|
||||
if (conn->gain != 0) {
|
||||
switch_change_sln_volume_granular(linear_data.data(), linear_data.size(), conn->gain);
|
||||
}
|
||||
}
|
||||
|
||||
return linear_data;
|
||||
}
|
||||
|
||||
void restart_cb(const boost::system::error_code& error, ConnInfo_t* conn) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "restart_cb status is %s\n", status2String(conn->status));
|
||||
if (conn->status == Status_t::STATUS_AWAITING_RESTART) {
|
||||
auto url = strdup(conn->url);
|
||||
auto rate = conn->rate;
|
||||
auto loop = conn->loop;
|
||||
auto gain = conn->gain;
|
||||
auto mutex = conn->mutex;
|
||||
auto buffer = conn->buffer;
|
||||
auto oldId = conn->id;
|
||||
|
||||
destroyConnection(conn);
|
||||
|
||||
downloadId_t id = start_audio_download(url, rate, loop, gain, mutex, buffer);
|
||||
|
||||
/* re-use id since caller is tracking that id */
|
||||
auto * newConnection = id2ConnMap[id];
|
||||
id2ConnMap[oldId] = newConnection;
|
||||
id2ConnMap.erase(id);
|
||||
|
||||
free(url);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "restarted looped download\n");
|
||||
}
|
||||
}
|
||||
|
||||
void throttling_cb(const boost::system::error_code& error, ConnInfo_t* conn) {
|
||||
if (conn->status == Status_t::STATUS_STOPPING || !conn->mutex || !conn->buffer) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "throttling_cb: session gone, resume download so we can complete\n");
|
||||
curl_easy_pause(conn->easy, CURLPAUSE_CONT);
|
||||
return;
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "throttling_cb: status is %s\n", status2String(conn->status));
|
||||
|
||||
switch_mutex_lock(conn->mutex);
|
||||
if (!error) {
|
||||
auto size = conn->buffer->size();
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "throttling_cb: size is now %ld\n", size);
|
||||
if (size < BUFFER_THROTTLE_LOW) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "throttling_cb: resuming download\n");
|
||||
curl_easy_pause(conn->easy, CURLPAUSE_CONT);
|
||||
switch_mutex_unlock(conn->mutex);
|
||||
return;
|
||||
}
|
||||
|
||||
// check back in 2 seconds
|
||||
conn->timer->expires_from_now(boost::posix_time::millisec(2000));
|
||||
conn->timer->async_wait(boost::bind(&throttling_cb, boost::placeholders::_1, conn));
|
||||
|
||||
} else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "throttling_cb: error (%d): %s\n", error.value(), error.message().c_str());
|
||||
|
||||
// Handle any errors
|
||||
}
|
||||
switch_mutex_unlock(conn->mutex);
|
||||
}
|
||||
|
||||
|
||||
/* CURLOPT_WRITEFUNCTION - here is where we receive the data */
|
||||
size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
|
||||
int8_t *data = (int8_t *) ptr;
|
||||
size_t bytes_received = size * nmemb;
|
||||
std::vector<int16_t> pcm_data;
|
||||
|
||||
if (conn->status == Status_t::STATUS_STOPPING || conn->status == Status_t::STATUS_STOPPED || !conn->mutex || !conn->buffer) {
|
||||
if (conn->timer) conn->timer->cancel();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,
|
||||
"write_cb: aborting transfer, status %s, mutex %p, buffer %p\n", status2String(conn->status), conn->mutex, conn->buffer);
|
||||
/* this will abort the transfer */
|
||||
return 0;
|
||||
}
|
||||
{
|
||||
switch_mutex_lock(conn->mutex);
|
||||
|
||||
if (conn->response_code > 0 && conn->response_code != 200) {
|
||||
std::string body((char *) ptr, bytes_received);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: received body %s\n", body.c_str());
|
||||
conn->err_msg = strdup(body.c_str());
|
||||
conn->status = Status_t::STATUS_FAILED;
|
||||
switch_mutex_unlock(conn->mutex);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* throttle after reaching high water mark */
|
||||
if (conn->buffer->size() > BUFFER_THROTTLE_HIGH) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: throttling download, buffer size is %ld\n", conn->buffer->size());
|
||||
|
||||
// check back in 2 seconds
|
||||
conn->timer->expires_from_now(boost::posix_time::millisec(2000));
|
||||
conn->timer->async_wait(boost::bind(&throttling_cb, boost::placeholders::_1, conn));
|
||||
|
||||
conn->status = Status_t::STATUS_DOWNLOAD_PAUSED;
|
||||
switch_mutex_unlock(conn->mutex);
|
||||
return CURL_WRITEFUNC_PAUSE;
|
||||
}
|
||||
|
||||
pcm_data = convert_mp3_to_linear(conn, data, bytes_received);
|
||||
size_t bytesResampled = pcm_data.size() * sizeof(int16_t);
|
||||
|
||||
// Resize the buffer if necessary
|
||||
if (conn->buffer->capacity() - conn->buffer->size() < (bytesResampled / sizeof(int16_t))) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "write_cb growing buffer, size now %ld\n", conn->buffer->size());
|
||||
|
||||
//TODO: if buffer exceeds some max size, return CURL_WRITEFUNC_ERROR to abort the transfer
|
||||
conn->buffer->set_capacity(conn->buffer->size() + std::max((bytesResampled / sizeof(int16_t)), (size_t)BUFFER_GROW_SIZE));
|
||||
}
|
||||
|
||||
/* Push the data into the buffer */
|
||||
conn->buffer->insert(conn->buffer->end(), pcm_data.data(), pcm_data.data() + pcm_data.size());
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: wrote data, buffer size is now %ld\n", conn->buffer->size());
|
||||
|
||||
switch_mutex_unlock(conn->mutex);
|
||||
}
|
||||
return bytes_received;
|
||||
}
|
||||
|
||||
bool parseHeader(const std::string& str, std::string& header, std::string& value) {
|
||||
std::vector<std::string> parts;
|
||||
boost::split(parts, str, boost::is_any_of(":"), boost::token_compress_on);
|
||||
|
||||
if (parts.size() != 2)
|
||||
return false;
|
||||
|
||||
header = boost::trim_copy(parts[0]);
|
||||
value = boost::trim_copy(parts[1]);
|
||||
return true;
|
||||
}
|
||||
|
||||
int extract_response_code(const std::string& input) {
|
||||
std::size_t space_pos = input.find(' ');
|
||||
if (space_pos == std::string::npos) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid HTTP response format %s\n", input.c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::size_t code_start_pos = space_pos + 1;
|
||||
std::size_t code_end_pos = input.find(' ', code_start_pos);
|
||||
if (code_end_pos == std::string::npos) {
|
||||
code_end_pos = input.length();
|
||||
}
|
||||
|
||||
std::string code_str = input.substr(code_start_pos, code_end_pos - code_start_pos);
|
||||
int response_code = std::stoi(code_str);
|
||||
return response_code;
|
||||
}
|
||||
|
||||
size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo_t *conn) {
|
||||
size_t bytes_received = size * nitems;
|
||||
const std::string prefix = "HTTP/";
|
||||
std::string header, value;
|
||||
std::string input(buffer, bytes_received);
|
||||
if (parseHeader(input, header, value)) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s with value %s\n", header.c_str(), value.c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s\n", input.c_str());
|
||||
if (input.rfind(prefix, 0) == 0) {
|
||||
try {
|
||||
conn->response_code = extract_response_code(input);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "parsed response code: %ld\n", conn->response_code);
|
||||
} catch (const std::invalid_argument& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "header_callback: invalid response code %s\n", input.substr(prefix.length()).c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
return bytes_received;
|
||||
}
|
||||
|
||||
/* CURLOPT_OPENSOCKETFUNCTION */
|
||||
curl_socket_t opensocket(void *clientp, curlsocktype purpose, struct curl_sockaddr *address) {
|
||||
curl_socket_t sockfd = CURL_SOCKET_BAD;
|
||||
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "opensocket: %d\n", purpose);
|
||||
/* restrict to IPv4 */
|
||||
if(purpose == CURLSOCKTYPE_IPCXN && address->family == AF_INET) {
|
||||
/* create a tcp socket object */
|
||||
boost::asio::ip::tcp::socket *tcp_socket = new boost::asio::ip::tcp::socket(io_service);
|
||||
|
||||
/* open it and get the native handle*/
|
||||
boost::system::error_code ec;
|
||||
tcp_socket->open(boost::asio::ip::tcp::v4(), ec);
|
||||
|
||||
if(ec) {
|
||||
/* An error occurred */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't open socket [%ld][%s]\n", ec, ec.message().c_str());
|
||||
}
|
||||
else {
|
||||
sockfd = tcp_socket->native_handle();
|
||||
|
||||
/* save it for monitoring */
|
||||
socket_map.insert(std::pair<curl_socket_t, boost::asio::ip::tcp::socket *>(sockfd, tcp_socket));
|
||||
}
|
||||
}
|
||||
return sockfd;
|
||||
}
|
||||
|
||||
/* CURLOPT_CLOSESOCKETFUNCTION */
|
||||
int close_socket(void *clientp, curl_socket_t item) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "close_socket : %#X\n", item);
|
||||
|
||||
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(item);
|
||||
if(it != socket_map.end()) {
|
||||
delete it->second;
|
||||
socket_map.erase(it);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
#ifndef __AUDIO_DOWNLOADER_H__
|
||||
#define __AUDIO_DOWNLOADER_H__
|
||||
|
||||
#include <switch.h>
|
||||
#include "common.h"
|
||||
|
||||
extern "C" {
|
||||
|
||||
switch_status_t init_audio_downloader();
|
||||
switch_status_t deinit_audio_downloader();
|
||||
|
||||
int start_audio_download(const char* url, int rate, int loop, int gain, switch_mutex_t* mutex, CircularBuffer_t* buffer);
|
||||
switch_status_t stop_audio_download(int id);
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@@ -3,10 +3,7 @@
|
||||
|
||||
#include <boost/circular_buffer.hpp>
|
||||
|
||||
|
||||
typedef boost::circular_buffer<int16_t> CircularBuffer_t;
|
||||
typedef int32_t downloadId_t;
|
||||
|
||||
#define INVALID_DOWNLOAD_ID (-1)
|
||||
|
||||
#endif
|
||||
+117
-130
@@ -1,8 +1,9 @@
|
||||
#include "mod_dub.h"
|
||||
#include "audio_downloader.h"
|
||||
#include "file_loader.h"
|
||||
|
||||
#include "tts_vendor_parser.h"
|
||||
#include "track.h"
|
||||
#include "vector_math.h"
|
||||
#include <string>
|
||||
#include <queue>
|
||||
|
||||
#include <switch.h>
|
||||
|
||||
@@ -15,124 +16,111 @@ typedef boost::circular_buffer<int16_t> CircularBuffer_t;
|
||||
|
||||
extern "C" {
|
||||
|
||||
void init_dub_track(dub_track_t *track, char* trackName, int sampleRate) {
|
||||
track->state = DUB_TRACK_STATE_READY;
|
||||
track->trackName = strdup(trackName);
|
||||
track->sampleRate = sampleRate;
|
||||
track->circularBuffer = new CircularBuffer_t(INIT_BUFFER_SIZE);
|
||||
Track* find_track_by_name(void** tracks, const std::string& trackName) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "find_track_by_name: searching for %s\n", trackName.c_str());
|
||||
|
||||
for (int i = 0; i < MAX_DUB_TRACKS; i++) {
|
||||
Track* track = static_cast<Track*>(tracks[i]);
|
||||
std::string name = track ? track->getTrackName() : "null";
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "find_track_by_name: offset %d: %s\n", i, name.c_str());
|
||||
if (track && 0 == track->getTrackName().compare(trackName)) {
|
||||
return track;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
switch_status_t silence_dub_track(dub_track_t *track) {
|
||||
assert(track);
|
||||
switch (track->generator) {
|
||||
case DUB_GENERATOR_TYPE_HTTP:
|
||||
stop_audio_download(track->generatorId);
|
||||
break;
|
||||
case DUB_GENERATOR_TYPE_FILE:
|
||||
stop_file_load(track->generatorId);
|
||||
break;
|
||||
case DUB_GENERATOR_TYPE_TTS:
|
||||
//TODO
|
||||
break;
|
||||
}
|
||||
CircularBuffer_t* buffer = reinterpret_cast<CircularBuffer_t*>(track->circularBuffer);
|
||||
buffer->clear();
|
||||
track->state = DUB_TRACK_STATE_READY;
|
||||
track->generator = DUB_GENERATOR_TYPE_UNKNOWN;
|
||||
track->generatorId = 0;
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t remove_dub_track(dub_track_t *track) {
|
||||
assert(track);
|
||||
switch (track->generator) {
|
||||
case DUB_GENERATOR_TYPE_HTTP:
|
||||
stop_audio_download(track->generatorId);
|
||||
break;
|
||||
case DUB_GENERATOR_TYPE_FILE:
|
||||
stop_file_load(track->generatorId);
|
||||
break;
|
||||
case DUB_GENERATOR_TYPE_TTS:
|
||||
//TODO
|
||||
break;
|
||||
}
|
||||
CircularBuffer_t* buffer = reinterpret_cast<CircularBuffer_t*>(track->circularBuffer);
|
||||
if (buffer) {
|
||||
delete buffer;
|
||||
}
|
||||
if (track->trackName) {
|
||||
free(track->trackName);
|
||||
}
|
||||
memset(track, 0, sizeof(dub_track_t));
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t play_dub_track(dub_track_t *track, switch_mutex_t *mutex, char* url, int loop, int gain) {
|
||||
bool isHttp = strncmp(url, "http", 4) == 0;
|
||||
if (track->state != DUB_TRACK_STATE_READY) {
|
||||
silence_dub_track(track);
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "play_dub_track: starting %s download: %s\n", (isHttp ? "HTTP" : "file"), url);
|
||||
int id = isHttp ?
|
||||
start_audio_download(url, track->sampleRate, loop, gain, mutex, (CircularBuffer_t*) track->circularBuffer) :
|
||||
start_file_load(url, track->sampleRate, loop, gain, mutex, (CircularBuffer_t*) track->circularBuffer);
|
||||
|
||||
if (id == INVALID_DOWNLOAD_ID) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "play_dub_track: failed to start audio download\n");
|
||||
switch_status_t add_track(struct cap_cb* cb, char* trackName, int sampleRate) {
|
||||
Track* existingTrack = find_track_by_name(cb->tracks, trackName);
|
||||
if (existingTrack) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "add_track: track %s already exists\n", trackName);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
track->state = DUB_TRACK_STATE_ACTIVE;
|
||||
track->generatorId = id;
|
||||
track->generator = isHttp ? DUB_GENERATOR_TYPE_HTTP : DUB_GENERATOR_TYPE_FILE;
|
||||
track->gain = gain;
|
||||
|
||||
for (int i = 0; i < MAX_DUB_TRACKS; i++) {
|
||||
if (!cb->tracks[i]) {
|
||||
cb->tracks[i] = new Track(trackName, sampleRate);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "add_track: added track %s at offset %d\n", trackName, i);
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "add_track: no room for track %s\n", trackName);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
switch_status_t silence_dub_track(struct cap_cb* cb, char* trackName) {
|
||||
Track* track = find_track_by_name(cb->tracks, trackName);
|
||||
if (track) {
|
||||
track->removeAllAudio();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "silence_dub_track: silenced track %s\n", trackName);
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "silence_dub_track: track %s not found\n", trackName);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
switch_status_t remove_dub_track(struct cap_cb* cb, char* trackName) {
|
||||
for (int i = 0; i < MAX_DUB_TRACKS; i++) {
|
||||
Track* track = static_cast<Track*>(cb->tracks[i]);
|
||||
if (track && track->getTrackName() == trackName) {
|
||||
track->removeAllAudio();
|
||||
delete track;
|
||||
cb->tracks[i] = nullptr;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "remove_dub_track: removed track %s\n", trackName);
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "remove_dub_track: track %s not found\n", trackName);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
switch_status_t play_dub_track(struct cap_cb* cb, char* trackName, char* url, int loop, int gain) {
|
||||
bool isHttp = strncmp(url, "http", 4) == 0;
|
||||
Track* track = find_track_by_name(cb->tracks, trackName);
|
||||
|
||||
if (!track) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "play_dub_track: track %s not found\n", trackName);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (isHttp) {
|
||||
track->queueHttpGetAudio(url, gain, loop);
|
||||
}
|
||||
else {
|
||||
track->queueFileAudio(url, gain, loop);
|
||||
}
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t say_dub_track(dub_track_t *track, switch_mutex_t *mutex, char* text, int gain) {
|
||||
if (track->state != DUB_TRACK_STATE_READY) {
|
||||
silence_dub_track(track); // wait...shouldnt we queue says?
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "say_dub_track: starting TTS\n");
|
||||
|
||||
/**
|
||||
* TODO:
|
||||
* This is not implemented yet. We can play TTS using using the playOnSay function
|
||||
* because jambonz can generate local audio files using TTS vendors.
|
||||
* However, we should probably at least implement support for elevenlabs streaming api
|
||||
* here because it is so much faster.
|
||||
*
|
||||
*/
|
||||
switch_status_t say_dub_track(struct cap_cb* cb, char* trackName, char* text, int gain) {
|
||||
std::vector<std::string> headers;
|
||||
std::string url, body, proxy;
|
||||
Track* track = find_track_by_name(cb->tracks, trackName);
|
||||
|
||||
/*
|
||||
track->state = DUB_TRACK_STATE_ACTIVE;
|
||||
track->generatorId = id;
|
||||
track->generator = DUB_GENERATOR_TYPE_TTS;
|
||||
track->gain = gain;
|
||||
*/
|
||||
if (!track) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "play_dub_track: track %s not found\n", trackName);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (tts_vendor_parse_text(text, url, body, headers, proxy) != SWITCH_STATUS_SUCCESS) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "say_dub_track: failed to parse text\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
track->queueHttpPostAudio(url, body, headers, proxy, gain);
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/* module load and unload */
|
||||
switch_status_t dub_init() {
|
||||
switch_status_t status;
|
||||
status = init_audio_downloader();
|
||||
if (status == SWITCH_STATUS_SUCCESS) {
|
||||
status = init_file_loader();
|
||||
}
|
||||
return status;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t dub_cleanup() {
|
||||
switch_status_t status;
|
||||
status = deinit_audio_downloader();
|
||||
if (status == SWITCH_STATUS_SUCCESS) {
|
||||
status = deinit_file_loader();
|
||||
}
|
||||
return status;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t dub_session_cleanup(switch_core_session_t *session, int channelIsClosing, switch_media_bug_t *bug) {
|
||||
@@ -144,17 +132,18 @@ extern "C" {
|
||||
|
||||
if (!switch_channel_get_private(channel, MY_BUG_NAME)) {
|
||||
// race condition
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s Bug is not attached (race).\n", switch_channel_get_name(channel));
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "%s Bug is not attached (race).\n", switch_channel_get_name(channel));
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
switch_channel_set_private(channel, MY_BUG_NAME, NULL);
|
||||
|
||||
for (int i = 0; i < MAX_DUB_TRACKS; i++) {
|
||||
dub_track_t* track = &cb->tracks[i];
|
||||
if (track->state != DUB_TRACK_STATE_INACTIVE) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "dub_session_cleanup: cleared track %d:%s\n", i, track->trackName);
|
||||
remove_dub_track(track);
|
||||
Track* track = static_cast<Track*>(cb->tracks[i]);
|
||||
if (track) {
|
||||
track->removeAllAudio();
|
||||
delete track;
|
||||
cb->tracks[i] = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,7 +151,7 @@ extern "C" {
|
||||
switch_core_media_bug_remove(session, &bug);
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "dub_session_cleanup: removed bug and cleared tracks\n");
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "dub_session_cleanup: removed bug and cleared tracks\n");
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s dub_session_cleanup: Bug is not attached.\n", switch_channel_get_name(channel));
|
||||
@@ -175,12 +164,17 @@ extern "C" {
|
||||
|
||||
if (switch_mutex_trylock(cb->mutex) == SWITCH_STATUS_SUCCESS) {
|
||||
|
||||
/* check if any tracks are actively pushing audio */
|
||||
int trackCount = 0;
|
||||
/* check if any tracks have audio to contribute */
|
||||
std::vector<Track*> activeTracks;
|
||||
activeTracks.reserve(MAX_DUB_TRACKS);
|
||||
for (int i = 0; i < MAX_DUB_TRACKS; i++) {
|
||||
if (cb->tracks[i].state == DUB_TRACK_STATE_ACTIVE) trackCount++;
|
||||
if (cb->tracks[i]) {
|
||||
auto track = static_cast<Track*>(cb->tracks[i]);
|
||||
if (track->hasAudio_NoLock()) activeTracks.push_back(static_cast<Track*>(cb->tracks[i]));
|
||||
}
|
||||
}
|
||||
if (trackCount == 0 && cb->gain == 0) {
|
||||
|
||||
if (activeTracks.size() == 0 && cb->gain == 0) {
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
return SWITCH_TRUE;
|
||||
}
|
||||
@@ -189,31 +183,24 @@ extern "C" {
|
||||
int16_t *fp = reinterpret_cast<int16_t*>(rframe->data);
|
||||
|
||||
rframe->channels = 1;
|
||||
rframe->datalen = rframe->samples * rframe->channels * sizeof(int16_t);
|
||||
rframe->datalen = rframe->samples * sizeof(int16_t);
|
||||
|
||||
/* apply gain to audio in main channel if requested*/
|
||||
if (cb->gain != 0) {
|
||||
switch_change_sln_volume_granular(fp, rframe->samples, cb->gain);
|
||||
vector_change_sln_volume_granular(fp, rframe->samples, cb->gain);
|
||||
}
|
||||
|
||||
/* now mux in the data from tracks */
|
||||
for (int i = 0; i < rframe->samples; i++) {
|
||||
int16_t input = fp[i];
|
||||
int16_t value = input;
|
||||
for (int j = 0; j < MAX_DUB_TRACKS; j++) {
|
||||
dub_track_t* track = &cb->tracks[j];
|
||||
if (track->state == DUB_TRACK_STATE_ACTIVE) {
|
||||
CircularBuffer_t* buffer = reinterpret_cast<CircularBuffer_t*>(track->circularBuffer);
|
||||
if (buffer && !buffer->empty()) {
|
||||
int16_t sample = buffer->front();
|
||||
buffer->pop_front();
|
||||
value += sample;
|
||||
}
|
||||
}
|
||||
for (auto track : activeTracks) {
|
||||
int16_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
|
||||
memset(data, 0, sizeof(data));
|
||||
auto samples = track->retrieveAndClearAudio(data, rframe->samples);
|
||||
if (samples > 0) {
|
||||
vector_add(fp, data, rframe->samples);
|
||||
}
|
||||
switch_normalize_to_16bit(value);
|
||||
fp[i] = (int16_t) value;
|
||||
}
|
||||
vector_normalize(fp, rframe->samples);
|
||||
|
||||
switch_core_media_bug_set_write_replace_frame(bug, rframe);
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
}
|
||||
|
||||
+5
-5
@@ -4,11 +4,11 @@
|
||||
switch_status_t dub_init();
|
||||
switch_status_t dub_cleanup();
|
||||
|
||||
void init_dub_track(dub_track_t *track, char* trackName, int sampleRate);
|
||||
switch_status_t silence_dub_track(dub_track_t *track);
|
||||
switch_status_t remove_dub_track(dub_track_t *track);
|
||||
switch_status_t play_dub_track(dub_track_t *track, switch_mutex_t *mutex, char* url, int loop, int gain);
|
||||
switch_status_t say_dub_track(dub_track_t *track, switch_mutex_t *mutex, char* text, int gain);
|
||||
switch_status_t add_track(struct cap_cb* cb, char* trackName, int sampleRate);
|
||||
switch_status_t silence_dub_track(struct cap_cb* cb, char* trackName);
|
||||
switch_status_t remove_dub_track(struct cap_cb* cb, char* trackName);
|
||||
switch_status_t play_dub_track(struct cap_cb* cb, char* trackName, char* url, int loop, int gain);
|
||||
switch_status_t say_dub_track(struct cap_cb* cb, char* trackName, char* text, int gain);
|
||||
|
||||
switch_status_t dub_session_cleanup(switch_core_session_t *session, int channelIsClosing, switch_media_bug_t *bug);
|
||||
switch_bool_t dub_speech_frame(switch_media_bug_t *bug, void* user_data);
|
||||
|
||||
@@ -1,437 +0,0 @@
|
||||
#include "file_loader.h"
|
||||
|
||||
#include <boost/thread.hpp>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/asio/ssl.hpp>
|
||||
#include <boost/pool/object_pool.hpp>
|
||||
#include <boost/bind/bind.hpp>
|
||||
#include <boost/tokenizer.hpp>
|
||||
#include <boost/foreach.hpp>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/assign/list_of.hpp>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
|
||||
#include <map>
|
||||
|
||||
#include <mpg123.h>
|
||||
|
||||
#define INIT_BUFFER_SIZE (80000)
|
||||
#define BUFFER_GROW_SIZE (80000)
|
||||
#define BUFFER_THROTTLE_LOW (40000)
|
||||
#define BUFFER_THROTTLE_HIGH (160000)
|
||||
|
||||
static uint16_t currDownloadId = 0;
|
||||
|
||||
typedef enum
|
||||
{
|
||||
STATUS_NONE = 0,
|
||||
STATUS_FAILED,
|
||||
STATUS_FILE_IN_PROGRESS,
|
||||
STATUS_FILE_PAUSED,
|
||||
STATUS_FILE_COMPLETE,
|
||||
STATUS_AWAITING_RESTART,
|
||||
STATUS_STOPPING,
|
||||
STATUS_STOPPED
|
||||
} Status_t;
|
||||
|
||||
typedef enum {
|
||||
FILE_TYPE_MP3 = 0,
|
||||
FILE_TYPE_R8
|
||||
} FileType_t;
|
||||
|
||||
static const char* status2String(Status_t status)
|
||||
{
|
||||
static const char* statusStrings[] = {
|
||||
"STATUS_NONE",
|
||||
"STATUS_FAILED",
|
||||
"STATUS_FILE_IN_PROGRESS",
|
||||
"STATUS_FILE_PAUSED",
|
||||
"STATUS_FILE_COMPLETE",
|
||||
"STATUS_AWAITING_RESTART",
|
||||
"STATUS_STOPPING",
|
||||
"STATUS_STOPPED"
|
||||
};
|
||||
|
||||
if (status >= 0 && status < sizeof(statusStrings) / sizeof(statusStrings[0]))
|
||||
{
|
||||
return statusStrings[status];
|
||||
}
|
||||
else
|
||||
{
|
||||
return "UNKNOWN_STATUS";
|
||||
}
|
||||
}
|
||||
|
||||
typedef struct
|
||||
{
|
||||
switch_mutex_t* mutex;
|
||||
CircularBuffer_t* buffer;
|
||||
mpg123_handle *mh;
|
||||
FILE* fp;
|
||||
char* path;
|
||||
bool loop;
|
||||
int rate;
|
||||
boost::asio::deadline_timer *timer;
|
||||
Status_t status;
|
||||
FileType_t type;
|
||||
downloadId_t id;
|
||||
int gain;
|
||||
} FileInfo_t;
|
||||
|
||||
typedef std::map<int32_t, FileInfo_t *> Id2FileMap_t;
|
||||
static Id2FileMap_t id2FileMap;
|
||||
|
||||
static boost::object_pool<FileInfo_t> pool ;
|
||||
static boost::asio::io_service io_service;
|
||||
static std::thread worker_thread;
|
||||
|
||||
|
||||
/* forward declarations */
|
||||
static FileInfo_t* createFileLoader(const char *path, int rate, int loop, int gain, mpg123_handle *mhm, switch_mutex_t *mutex, CircularBuffer_t *buffer);
|
||||
static void destroyFileInfo(FileInfo_t *finfo);
|
||||
static void threadFunc();
|
||||
static std::vector<int16_t> convert_mp3_to_linear(FileInfo_t *file, int8_t *data, size_t len);
|
||||
static void read_cb(const boost::system::error_code& error, FileInfo_t* finfo) ;
|
||||
|
||||
/* apis */
|
||||
extern "C" {
|
||||
|
||||
switch_status_t init_file_loader() {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "init_file_loader loading..\n");
|
||||
|
||||
if (mpg123_init() != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "init_file_loader: failed to initiate MPG123");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
/* start worker thread */
|
||||
std::thread t(threadFunc) ;
|
||||
worker_thread.swap( t ) ;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "init_file_loader: loaded\n");
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
|
||||
}
|
||||
|
||||
switch_status_t deinit_file_loader() {
|
||||
/* stop the ASIO IO service */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deinit_file_loader: stopping io service\n");
|
||||
io_service.stop();
|
||||
|
||||
/* Join the worker thread */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "deinit_file_loader: wait for worker thread to complete\n");
|
||||
if (worker_thread.joinable()) {
|
||||
worker_thread.join();
|
||||
}
|
||||
|
||||
mpg123_exit();
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
downloadId_t start_file_load(const char* path, int rate, int loop, int gain, switch_mutex_t* mutex, CircularBuffer_t* buffer) {
|
||||
int mhError = 0;
|
||||
|
||||
/* we only handle mp3 or r8 files atm */
|
||||
const char *ext = strrchr(path, '.');
|
||||
if (!ext) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "start_file_load: file %s has no extension\n", path);
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
ext++;
|
||||
if (0 != strcmp(ext, "mp3") && 0 != strcmp(ext, "r8")) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "start_file_load: file %s has unsupported extension %s\n", path, ext);
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
/* allocate handle for mpeg decoding */
|
||||
mpg123_handle *mh = mpg123_new("auto", &mhError);
|
||||
if (!mh) {
|
||||
const char *mhErr = mpg123_plain_strerror(mhError);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error allocating mpg123 handle! %s\n", switch_str_nil(mhErr));
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
if (mpg123_open_feed(mh) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_open_feed!\n");
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
if (mpg123_format_all(mh) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_format_all!\n");
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
if (mpg123_param(mh, MPG123_FORCE_RATE, rate, 0) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error forcing resample to 8k!\n");
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
if (mpg123_param(mh, MPG123_FLAGS, MPG123_MONO_MIX, 0) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error forcing single channel!\n");
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
FileInfo_t* finfo = createFileLoader(path, rate, loop, gain, mh, mutex, buffer);
|
||||
if (!finfo) {
|
||||
return INVALID_DOWNLOAD_ID;
|
||||
}
|
||||
|
||||
/* do the initial read in the worker thread so we don't block here */
|
||||
finfo->timer->expires_from_now(boost::posix_time::millisec(1));
|
||||
finfo->timer->async_wait(boost::bind(&read_cb, boost::placeholders::_1, finfo));
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,
|
||||
"start_file_load: starting load %d\n", finfo->id);
|
||||
|
||||
return finfo->id;
|
||||
}
|
||||
|
||||
switch_status_t stop_file_load(int id) {
|
||||
auto it = id2FileMap.find(id);
|
||||
if (it == id2FileMap.end()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "stop_file_load: id %d has already completed\n", id);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,
|
||||
"stop_audio_download: stopping download %d, status %s\n", id, status2String(it->second->status));
|
||||
|
||||
FileInfo_t *finfo = it->second;
|
||||
auto status = finfo->status;
|
||||
|
||||
/* past this point I shall not access either the mutex or the buffer provided */
|
||||
finfo->mutex = nullptr;
|
||||
finfo->buffer = nullptr;
|
||||
|
||||
destroyFileInfo(finfo);
|
||||
|
||||
finfo->status = Status_t::STATUS_STOPPED;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
/* internal */
|
||||
FileInfo_t* createFileLoader(const char *path, int rate, int loop, int gain, mpg123_handle *mh, switch_mutex_t *mutex, CircularBuffer_t *buffer) {
|
||||
FileInfo_t *finfo = pool.malloc() ;
|
||||
const char *ext = strrchr(path, '.');
|
||||
|
||||
memset(finfo, 0, sizeof(FileInfo_t));
|
||||
finfo->mutex = mutex;
|
||||
finfo->buffer = buffer;
|
||||
finfo->mh = mh;
|
||||
finfo->loop = loop;
|
||||
finfo->gain = gain;
|
||||
finfo->rate = rate;
|
||||
finfo->path = strdup(path);
|
||||
finfo->status = Status_t::STATUS_NONE;
|
||||
finfo->timer = new boost::asio::deadline_timer(io_service);
|
||||
|
||||
if (0 == strcmp(ext, "mp3")) finfo->type = FileType_t::FILE_TYPE_MP3;
|
||||
else if (0 == strcmp(ext, "r8")) finfo->type = FileType_t::FILE_TYPE_R8;
|
||||
|
||||
downloadId_t id = ++currDownloadId;
|
||||
if (id == 0) id++;
|
||||
|
||||
id2FileMap[id] = finfo;
|
||||
finfo->id = id;
|
||||
|
||||
finfo->status = Status_t::STATUS_AWAITING_RESTART;
|
||||
|
||||
finfo->fp = fopen(finfo->path, "rb");
|
||||
if (finfo->fp == NULL) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "createFileLoader: failed to open file %s\n", finfo->path);
|
||||
destroyFileInfo(finfo);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,
|
||||
"createFileLoader: launched request, loop %s, gain %d\n", (finfo->loop ? "yes": "no"), finfo->gain);
|
||||
return finfo;
|
||||
}
|
||||
|
||||
void destroyFileInfo(FileInfo_t *finfo) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "destroyFileInfo\n");
|
||||
|
||||
/* clear asio resources and free resources */
|
||||
if (finfo->timer) {
|
||||
finfo->timer->cancel();
|
||||
delete finfo->timer;
|
||||
}
|
||||
|
||||
/* free mp3 decoder */
|
||||
if (finfo->mh) {
|
||||
mpg123_close(finfo->mh);
|
||||
mpg123_delete(finfo->mh);
|
||||
}
|
||||
|
||||
if (finfo->path) {
|
||||
free(finfo->path);
|
||||
}
|
||||
|
||||
if (finfo->mutex) switch_mutex_lock(finfo->mutex);
|
||||
id2FileMap.erase(finfo->id);
|
||||
if (finfo->mutex) switch_mutex_unlock(finfo->mutex);
|
||||
|
||||
memset(finfo, 0, sizeof(FileInfo_t));
|
||||
pool.destroy(finfo) ;
|
||||
}
|
||||
|
||||
void threadFunc() {
|
||||
/* to make sure the event loop doesn't terminate when there is no work to do */
|
||||
io_service.reset() ;
|
||||
boost::asio::io_service::work work(io_service);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "file_loader threadFunc - starting\n");
|
||||
|
||||
for(;;) {
|
||||
|
||||
try {
|
||||
io_service.run() ;
|
||||
break ;
|
||||
}
|
||||
catch( std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "file_loader threadFunc - Error: %s\n", e.what());
|
||||
}
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "file_loader threadFunc - ending\n");
|
||||
}
|
||||
|
||||
std::vector<int16_t> convert_mp3_to_linear(FileInfo_t *finfo, int8_t *data, size_t len) {
|
||||
std::vector<int16_t> linear_data;
|
||||
int eof = 0;
|
||||
int mp3err = 0;
|
||||
|
||||
if(mpg123_feed(finfo->mh, (const unsigned char*) data, len) == MPG123_OK) {
|
||||
while(!eof) {
|
||||
size_t usedlen = 0;
|
||||
off_t frame_offset;
|
||||
unsigned char* audio;
|
||||
|
||||
int decode_status = mpg123_decode_frame(finfo->mh, &frame_offset, &audio, &usedlen);
|
||||
|
||||
switch(decode_status) {
|
||||
case MPG123_NEW_FORMAT:
|
||||
continue;
|
||||
|
||||
case MPG123_OK:
|
||||
{
|
||||
size_t samples = usedlen / sizeof(int16_t);
|
||||
linear_data.insert(linear_data.end(), reinterpret_cast<int16_t*>(audio), reinterpret_cast<int16_t*>(audio) + samples);
|
||||
}
|
||||
break;
|
||||
|
||||
case MPG123_DONE:
|
||||
case MPG123_NEED_MORE:
|
||||
eof = 1;
|
||||
break;
|
||||
|
||||
case MPG123_ERR:
|
||||
default:
|
||||
if(++mp3err >= 5) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Decoder Error!\n");
|
||||
eof = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (eof)
|
||||
break;
|
||||
|
||||
mp3err = 0;
|
||||
}
|
||||
|
||||
if (finfo->gain != 0) {
|
||||
switch_change_sln_volume_granular(linear_data.data(), linear_data.size(), finfo->gain);
|
||||
}
|
||||
}
|
||||
|
||||
return linear_data;
|
||||
}
|
||||
|
||||
void read_cb(const boost::system::error_code& error, FileInfo_t* finfo) {
|
||||
if (finfo->status == Status_t::STATUS_STOPPING || !finfo->mutex || !finfo->buffer) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %u session gone\n", finfo->id);
|
||||
return;
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %u status is %s\n", finfo->id, status2String(finfo->status));
|
||||
if (finfo->status == Status_t::STATUS_AWAITING_RESTART) {
|
||||
finfo->status = Status_t::STATUS_FILE_IN_PROGRESS;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %u starting initial read of file\n", finfo->id);
|
||||
}
|
||||
|
||||
if (!error) {
|
||||
size_t size = 0;
|
||||
|
||||
switch_mutex_lock(finfo->mutex);
|
||||
size = finfo->buffer->size();
|
||||
switch_mutex_unlock(finfo->mutex);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %u size is now %ld\n", finfo->id, size);
|
||||
if (size < BUFFER_THROTTLE_LOW) {
|
||||
std::vector<int16_t> pcm_data;
|
||||
int8_t buf[INIT_BUFFER_SIZE];
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %u reading data\n", finfo->id);
|
||||
|
||||
size_t bytesRead = ::fread(buf, sizeof(int8_t), INIT_BUFFER_SIZE, finfo->fp);
|
||||
if (bytesRead <= 0) {
|
||||
if (::feof(finfo->fp)) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %u eof\n", finfo->id);
|
||||
}
|
||||
else if (::ferror(finfo->fp)) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "read_cb: %u error reading file\n", finfo->id);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "read_cb: %u unknown error reading file\n", finfo->id);
|
||||
}
|
||||
finfo->status = Status_t::STATUS_FILE_COMPLETE;
|
||||
return;
|
||||
}
|
||||
|
||||
if (finfo->type == FileType_t::FILE_TYPE_MP3) {
|
||||
pcm_data = convert_mp3_to_linear(finfo, buf, bytesRead);
|
||||
} else {
|
||||
pcm_data = std::vector<int16_t>(reinterpret_cast<int16_t*>(buf), reinterpret_cast<int16_t*>(buf) + bytesRead / 2);
|
||||
}
|
||||
|
||||
switch_mutex_lock(finfo->mutex);
|
||||
|
||||
// Resize the buffer if necessary
|
||||
if (finfo->buffer->capacity() - finfo->buffer->size() < pcm_data.size()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "write_cb %u growing buffer, size now %ld\n", finfo->id, finfo->buffer->size());
|
||||
finfo->buffer->set_capacity(finfo->buffer->size() + std::max(pcm_data.size(), (size_t)BUFFER_GROW_SIZE));
|
||||
}
|
||||
|
||||
/* Push the data into the buffer */
|
||||
finfo->buffer->insert(finfo->buffer->end(), pcm_data.data(), pcm_data.data() + pcm_data.size());
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %u wrote data, buffer size is now %ld\n", finfo->id, finfo->buffer->size());
|
||||
|
||||
switch_mutex_unlock(finfo->mutex);
|
||||
|
||||
if (bytesRead < INIT_BUFFER_SIZE) {
|
||||
finfo->status = Status_t::STATUS_FILE_COMPLETE;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %u reached end of file, status is %s\n", finfo->id, status2String(finfo->status));
|
||||
}
|
||||
}
|
||||
|
||||
if (finfo->status == Status_t::STATUS_FILE_COMPLETE && finfo->loop) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %u looping\n", finfo->id);
|
||||
::fseek(finfo->fp, 0, SEEK_SET);
|
||||
finfo->status = Status_t::STATUS_AWAITING_RESTART;
|
||||
}
|
||||
|
||||
if (finfo->status != Status_t::STATUS_FILE_COMPLETE) {
|
||||
// read more in 2 seconds
|
||||
finfo->timer->expires_from_now(boost::posix_time::millisec(2000));
|
||||
finfo->timer->async_wait(boost::bind(&read_cb, boost::placeholders::_1, finfo));
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "read_cb: %u file complete, status %s loop %s\n",
|
||||
finfo->id, status2String(finfo->status), (finfo->loop ? "yes" : "no"));
|
||||
}
|
||||
} else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "read_cb: %u error (%d): %s\n", finfo->id, error.value(), error.message().c_str());
|
||||
|
||||
// Handle any errors
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
#ifndef __FILE_LOADER_H__
|
||||
#define __FILE_LOADER_H__
|
||||
|
||||
#include <switch.h>
|
||||
#include "common.h"
|
||||
|
||||
extern "C" {
|
||||
|
||||
switch_status_t init_file_loader();
|
||||
switch_status_t deinit_file_loader();
|
||||
|
||||
int start_file_load(const char* path, int rate, int loop, int gain, switch_mutex_t* mutex, CircularBuffer_t* buffer);
|
||||
switch_status_t stop_file_load(int id);
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
+31
-83
@@ -8,6 +8,7 @@
|
||||
#include <switch.h>
|
||||
#include <switch_curl.h>
|
||||
#include "dub_glue.h"
|
||||
#include <string.h>
|
||||
|
||||
/* Prototypes */
|
||||
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_dub_shutdown);
|
||||
@@ -88,7 +89,6 @@ static switch_status_t dub_add_track(switch_core_session_t *session, char* track
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_media_bug_t *bug = NULL;
|
||||
struct cap_cb *cb = NULL;
|
||||
int offset = 0;
|
||||
int samples_per_second;
|
||||
|
||||
switch_codec_implementation_t write_impl = { 0 };
|
||||
@@ -102,36 +102,32 @@ static switch_status_t dub_add_track(switch_core_session_t *session, char* track
|
||||
cb =(struct cap_cb *) switch_core_session_alloc(session, sizeof(struct cap_cb));
|
||||
memset(cb, 0, sizeof(struct cap_cb));
|
||||
switch_mutex_init(&cb->mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session));
|
||||
offset = 0;
|
||||
}
|
||||
else {
|
||||
/* retrieve the bug and search for an empty track */
|
||||
cb = (struct cap_cb *) switch_core_media_bug_get_user_data(bug);
|
||||
while (offset < MAX_DUB_TRACKS && cb->tracks[offset].state != DUB_TRACK_STATE_INACTIVE) {
|
||||
offset++;
|
||||
}
|
||||
if (offset == MAX_DUB_TRACKS) {
|
||||
/* all tracks are in use */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_add_track: no available tracks\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
switch_core_session_get_write_impl(session, &write_impl);
|
||||
samples_per_second = !strcasecmp(write_impl.iananame, "g722") ? write_impl.actual_samples_per_second : write_impl.samples_per_second;
|
||||
|
||||
init_dub_track(&cb->tracks[offset], trackName, samples_per_second);
|
||||
switch_mutex_lock(cb->mutex);
|
||||
if (add_track(cb, trackName, samples_per_second) != SWITCH_STATUS_SUCCESS) {
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_add_track: error adding track %s\n", trackName);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (!bug) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "dub_add_track: adding bug for track %s\n", trackName);
|
||||
if (switch_core_media_bug_add(session, MY_BUG_NAME, NULL, capture_callback, (void *) cb, 0, SMBF_WRITE_REPLACE, &bug) != SWITCH_STATUS_SUCCESS) {
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_add_track: error adding bug!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
switch_channel_set_private(channel, MY_BUG_NAME, bug);
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "dub_add_track: added track %s at offset %d\n", trackName, offset);
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
@@ -140,39 +136,17 @@ static switch_status_t dub_remove_track(switch_core_session_t *session, char* tr
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, MY_BUG_NAME);
|
||||
switch_status_t status = SWITCH_STATUS_FALSE;
|
||||
dub_track_t *track = NULL;
|
||||
|
||||
if (bug) {
|
||||
struct cap_cb *cb =(struct cap_cb *) switch_core_media_bug_get_user_data(bug);
|
||||
|
||||
switch_mutex_lock(cb->mutex);
|
||||
for (int i = 0; i < MAX_DUB_TRACKS && track == NULL; i++) {
|
||||
if (cb->tracks[i].state != DUB_TRACK_STATE_INACTIVE && strcmp(cb->tracks[i].trackName, trackName) == 0) {
|
||||
track = &cb->tracks[i];
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "dub_remove_track: removing track %s at offset %d\n", trackName, i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (track) {
|
||||
int count = 0;
|
||||
|
||||
remove_dub_track(track);
|
||||
|
||||
/* check if this is the last bug */
|
||||
for (int i = 0; i < MAX_DUB_TRACKS; i++) {
|
||||
if (cb->tracks[i].state != DUB_TRACK_STATE_INACTIVE) count++;
|
||||
}
|
||||
|
||||
if (count == 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "dub_remove_track: removing bug after removing last track\n");
|
||||
dub_session_cleanup(session, 0, bug);
|
||||
}
|
||||
status = SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_remove_track: track %s not found\n", trackName);
|
||||
}
|
||||
status = remove_dub_track(cb, trackName);
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
if (status != SWITCH_STATUS_SUCCESS) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_remove_track: error removing track %s\n", trackName);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_remove_track: bug not found\n");
|
||||
@@ -185,24 +159,14 @@ static switch_status_t dub_silence_track(switch_core_session_t *session, char* t
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, MY_BUG_NAME);
|
||||
switch_status_t status = SWITCH_STATUS_FALSE;
|
||||
dub_track_t *track = NULL;
|
||||
|
||||
if (bug) {
|
||||
struct cap_cb *cb =(struct cap_cb *) switch_core_media_bug_get_user_data(bug);
|
||||
switch_mutex_lock(cb->mutex);
|
||||
for (int i = 0; i < MAX_DUB_TRACKS; i++) {
|
||||
if (cb->tracks[i].state != DUB_TRACK_STATE_INACTIVE && strcmp(cb->tracks[i].trackName, trackName) == 0) {
|
||||
track = &cb->tracks[i];
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "dub_silence_track: silencing track %s at offset %d\n", trackName, i);
|
||||
break;
|
||||
}
|
||||
status = silence_dub_track(cb, trackName);
|
||||
if (status != SWITCH_STATUS_SUCCESS) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_silence_track: error silencing track %s\n", trackName);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (track) {
|
||||
silence_dub_track(track);
|
||||
status = SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_silence_track: bug not found\n");
|
||||
@@ -215,24 +179,14 @@ static switch_status_t dub_play_on_track(switch_core_session_t *session, char* t
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, MY_BUG_NAME);
|
||||
switch_status_t status = SWITCH_STATUS_FALSE;
|
||||
dub_track_t *track = NULL;
|
||||
|
||||
if (bug) {
|
||||
struct cap_cb *cb =(struct cap_cb *) switch_core_media_bug_get_user_data(bug);
|
||||
switch_mutex_lock(cb->mutex);
|
||||
for (int i = 0; i < MAX_DUB_TRACKS; i++) {
|
||||
if (cb->tracks[i].state != DUB_TRACK_STATE_INACTIVE && strcmp(cb->tracks[i].trackName, trackName) == 0) {
|
||||
track = &cb->tracks[i];
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,
|
||||
"dub_play_on_track: playing %s on track %s at offset %d with gain %d\n", url, trackName, i, gain);
|
||||
break;
|
||||
}
|
||||
status = play_dub_track(cb, trackName, url, loop, gain);
|
||||
if (status != SWITCH_STATUS_SUCCESS) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_play_on_track: error playing %s on track %s\n", url, trackName);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (track) {
|
||||
status = play_dub_track(track, cb->mutex, url, loop, gain);
|
||||
}
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
@@ -241,37 +195,30 @@ static switch_status_t dub_say_on_track(switch_core_session_t *session, char* tr
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_media_bug_t *bug = (switch_media_bug_t*) switch_channel_get_private(channel, MY_BUG_NAME);
|
||||
switch_status_t status = SWITCH_STATUS_FALSE;
|
||||
dub_track_t *track = NULL;
|
||||
|
||||
if (bug) {
|
||||
struct cap_cb *cb =(struct cap_cb *) switch_core_media_bug_get_user_data(bug);
|
||||
switch_mutex_lock(cb->mutex);
|
||||
for (int i = 0; i < MAX_DUB_TRACKS; i++) {
|
||||
if (cb->tracks[i].state != DUB_TRACK_STATE_INACTIVE && strcmp(cb->tracks[i].trackName, trackName) == 0) {
|
||||
track = &cb->tracks[i];
|
||||
break;
|
||||
}
|
||||
status = say_dub_track(cb, trackName, text, gain);
|
||||
if (status != SWITCH_STATUS_SUCCESS) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "dub_say_on_track: error saying %s on track %s\n", text, trackName);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (track) {
|
||||
status = say_dub_track(track, cb->mutex, text, gain);
|
||||
}
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
#define DUB_API_SYNTAX "<uuid> [addTrack|removeTrack|silenceTrack|playOnTrack|sayOnTrack|setGain] track [url|text|gain] [gain] [loop]"
|
||||
#define MAX_PARAMS 6
|
||||
SWITCH_STANDARD_API(dub_function)
|
||||
{
|
||||
char *mycmd = NULL, *argv[6] = { 0 };
|
||||
char *mycmd = NULL, *argv[MAX_PARAMS] = { 0 };
|
||||
int argc = 0;
|
||||
int error_written = 0;
|
||||
switch_status_t status = SWITCH_STATUS_FALSE;
|
||||
|
||||
if (!zstr(cmd) && (mycmd = strdup(cmd))) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "dub_function: %s\n", mycmd);
|
||||
argc = switch_separate_string(mycmd, ' ', argv, (sizeof(argv) / sizeof(argv[0])));
|
||||
argc = switch_separate_string(mycmd, ' ', argv, (sizeof(argv) / sizeof(argv[0])));
|
||||
}
|
||||
|
||||
if (zstr(cmd) || argc < 3 || zstr(argv[0]) || zstr(argv[1]) || zstr(argv[2])) {
|
||||
@@ -302,6 +249,7 @@ SWITCH_STANDARD_API(dub_function)
|
||||
}
|
||||
else if (0 == strcmp(action, "playOnTrack")) {
|
||||
if (argc < 4) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "playOnTrack requires a url\n");
|
||||
stream->write_function(stream, "-USAGE: %s\n", DUB_API_SYNTAX);
|
||||
error_written = 1;
|
||||
}
|
||||
|
||||
+1
-32
@@ -17,42 +17,11 @@
|
||||
/* per-channel data */
|
||||
typedef void (*responseHandler_t)(switch_core_session_t* session, const char* json, const char* bugname, const char* details);
|
||||
|
||||
typedef enum {
|
||||
DUB_TRACK_STATE_INACTIVE = 0,
|
||||
DUB_TRACK_STATE_READY,
|
||||
DUB_TRACK_STATE_ACTIVE,
|
||||
DUB_TRACK_STATE_PAUSED
|
||||
} dub_state_t;
|
||||
|
||||
typedef enum {
|
||||
DUB_GENERATOR_TYPE_UNKNOWN = 0,
|
||||
DUB_GENERATOR_TYPE_HTTP,
|
||||
DUB_GENERATOR_TYPE_FILE,
|
||||
DUB_GENERATOR_TYPE_TTS
|
||||
} dub_generator_t;
|
||||
|
||||
typedef enum {
|
||||
DUB_TRACK_EVENT_PLAY = 0,
|
||||
DUB_TRACK_EVENT_STOP,
|
||||
DUB_TRACK_EVENT_PAUSE,
|
||||
DUB_TRACK_EVENT_RESUME
|
||||
} dub_event_t;
|
||||
|
||||
|
||||
typedef struct dub_track {
|
||||
dub_state_t state;
|
||||
dub_generator_t generator;
|
||||
char* trackName;
|
||||
int sampleRate;
|
||||
int gain;
|
||||
void* circularBuffer;
|
||||
int generatorId;
|
||||
} dub_track_t;
|
||||
|
||||
struct cap_cb {
|
||||
switch_mutex_t *mutex;
|
||||
int gain;
|
||||
dub_track_t tracks[MAX_DUB_TRACKS];
|
||||
void *tracks[MAX_DUB_TRACKS];
|
||||
};
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
|
||||
#include "mpg_decode.h"
|
||||
#include "vector_math.h"
|
||||
|
||||
std::vector<int16_t> convert_mp3_to_linear(mpg123_handle *mh, int gain, int8_t *data, size_t len) {
|
||||
std::vector<int16_t> linear_data;
|
||||
int eof = 0;
|
||||
int mp3err = 0;
|
||||
|
||||
if(mpg123_feed(mh, (const unsigned char*) data, len) == MPG123_OK) {
|
||||
while(!eof) {
|
||||
size_t usedlen = 0;
|
||||
off_t frame_offset;
|
||||
unsigned char* audio;
|
||||
|
||||
int decode_status = mpg123_decode_frame(mh, &frame_offset, &audio, &usedlen);
|
||||
|
||||
switch(decode_status) {
|
||||
case MPG123_NEW_FORMAT:
|
||||
continue;
|
||||
|
||||
case MPG123_OK:
|
||||
{
|
||||
size_t samples = usedlen / sizeof(int16_t);
|
||||
linear_data.insert(linear_data.end(), reinterpret_cast<int16_t*>(audio), reinterpret_cast<int16_t*>(audio) + samples);
|
||||
}
|
||||
break;
|
||||
|
||||
case MPG123_DONE:
|
||||
case MPG123_NEED_MORE:
|
||||
eof = 1;
|
||||
break;
|
||||
|
||||
case MPG123_ERR:
|
||||
default:
|
||||
if(++mp3err >= 5) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Decoder Error!\n");
|
||||
eof = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (eof)
|
||||
break;
|
||||
|
||||
mp3err = 0;
|
||||
}
|
||||
|
||||
if (gain != 0) {
|
||||
vector_change_sln_volume_granular(linear_data.data(), linear_data.size(), gain);
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error feeding data to mpg123\n");
|
||||
}
|
||||
|
||||
|
||||
return linear_data;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
|
||||
#ifndef MPG_DECODE_H
|
||||
#define MPG_DECODE_H
|
||||
|
||||
#include <vector>
|
||||
#include <mpg123.h>
|
||||
#include "switch.h"
|
||||
|
||||
std::vector<int16_t> convert_mp3_to_linear(mpg123_handle *mh, int gain, int8_t *data, size_t len);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,157 @@
|
||||
#include "track.h"
|
||||
#include "ap_file.h"
|
||||
#include "ap_http.h"
|
||||
#include "switch.h"
|
||||
|
||||
#define INIT_BUFFER_SIZE (80000)
|
||||
|
||||
Track::Track(const std::string& trackName, int sampleRate) : _trackName(trackName), _sampleRate(sampleRate),
|
||||
_buffer(INIT_BUFFER_SIZE), _stopping(false)
|
||||
{
|
||||
}
|
||||
|
||||
Track::~Track() {
|
||||
removeAllAudio();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Track::~Track: track %s\n", _trackName.c_str());
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief called when an audio producer has finished retrieving the audio.
|
||||
* If we have another producer queued, then start it.
|
||||
*
|
||||
* @param hasError
|
||||
* @param errMsg
|
||||
*/
|
||||
void Track::onPlayDone(bool hasError, const std::string& errMsg) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Track::onPlayDone %s\n", _trackName.c_str());
|
||||
|
||||
if (!_stopping) {
|
||||
std::shared_ptr<AudioProducer> apDone, apNext; // to retain a ref so the ap is not destroyed while under lock below -- avoid deadlock
|
||||
|
||||
if (hasError) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "onPlayDone: error: %s\n", errMsg.c_str());
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
if (!_apQueue.empty()) {
|
||||
apDone = _apQueue.front();
|
||||
_apQueue.pop();
|
||||
}
|
||||
if (!_apQueue.empty()) apNext = _apQueue.front();
|
||||
}
|
||||
if (apNext) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "onPlayDone: starting queued audio on track %s\n", _trackName.c_str());
|
||||
apNext->start(std::bind(&Track::onPlayDone, this, std::placeholders::_1, std::placeholders::_2));
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "onPlayDone: track %s stopping\n", _trackName.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
void Track::queueFileAudio(const std::string& path, int gain, bool loop) {
|
||||
bool startIt = false;
|
||||
if (_stopping) return;
|
||||
|
||||
auto ap = std::make_shared<AudioProducerFile>(_mutex, _buffer, _sampleRate);
|
||||
ap->queueFileAudio(path, gain, loop);
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
_apQueue.push(ap);
|
||||
startIt = _apQueue.size() == 1;
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,
|
||||
"queueFileAudio: queue file %s on track %s\n", _trackName.c_str(), path.c_str());
|
||||
|
||||
if (startIt) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,
|
||||
"queueFileAudio: track %s starting queued file as none are in progress\n", _trackName.c_str());
|
||||
try {
|
||||
ap->start(std::bind(&Track::onPlayDone, this, std::placeholders::_1, std::placeholders::_2));
|
||||
} catch (std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,
|
||||
"queueFileAudio: track %s error starting %s: %s\n", _trackName.c_str(), path.c_str(), e.what());
|
||||
onPlayDone(true, e.what());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Track::queueHttpGetAudio(const std::string& url, int gain, bool loop) {
|
||||
bool startIt = false;
|
||||
if (_stopping) return;
|
||||
auto ap = std::make_shared<AudioProducerHttp>(_mutex, _buffer, _sampleRate);
|
||||
ap->queueHttpGetAudio(url, gain, loop);
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
_apQueue.push(ap);
|
||||
startIt = _apQueue.size() == 1;
|
||||
}
|
||||
|
||||
if (startIt) {
|
||||
try {
|
||||
ap->start(std::bind(&Track::onPlayDone, this, std::placeholders::_1, std::placeholders::_2));
|
||||
} catch (std::exception& e) {
|
||||
onPlayDone(true, e.what());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Track::queueHttpPostAudio(const std::string& url, int gain, bool loop) {
|
||||
bool startIt = false;
|
||||
if (_stopping) return;
|
||||
auto ap = std::make_shared<AudioProducerHttp>(_mutex, _buffer, _sampleRate);
|
||||
ap->queueHttpPostAudio(url, gain, loop);
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
_apQueue.push(ap);
|
||||
startIt = _apQueue.size() == 1;
|
||||
}
|
||||
|
||||
if (startIt) {
|
||||
try {
|
||||
ap->start(std::bind(&Track::onPlayDone, this, std::placeholders::_1, std::placeholders::_2));
|
||||
} catch (std::exception& e) {
|
||||
onPlayDone(true, e.what());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Track::queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, const std::string& proxy, int gain, bool loop) {
|
||||
bool startIt = false;
|
||||
if (_stopping) return;
|
||||
auto ap = std::make_shared<AudioProducerHttp>(_mutex, _buffer, _sampleRate);
|
||||
ap->queueHttpPostAudio(url, body, headers, proxy, gain, loop);
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
_apQueue.push(ap);
|
||||
startIt = _apQueue.size() == 1;
|
||||
}
|
||||
|
||||
if (startIt) {
|
||||
try {
|
||||
ap->start(std::bind(&Track::onPlayDone, this, std::placeholders::_1, std::placeholders::_2));
|
||||
} catch (std::exception& e) {
|
||||
onPlayDone(true, e.what());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool Track::hasAudio() {
|
||||
if (_stopping) return false;
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
return hasAudio_NoLock();
|
||||
}
|
||||
|
||||
void Track::removeAllAudio() {
|
||||
_stopping = true;
|
||||
std::queue<std::shared_ptr<AudioProducer>> apQueueCopy;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
apQueueCopy = _apQueue;
|
||||
_apQueue = std::queue<std::shared_ptr<AudioProducer>>();
|
||||
}
|
||||
|
||||
while (!apQueueCopy.empty()) {
|
||||
auto ap = apQueueCopy.front();
|
||||
apQueueCopy.pop();
|
||||
ap->stop();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
#ifndef __TRACK_H__
|
||||
#define __TRACK_H__
|
||||
|
||||
#include <mutex>
|
||||
#include <queue>
|
||||
#include "common.h"
|
||||
#include "ap.h"
|
||||
|
||||
class Track {
|
||||
public:
|
||||
Track(const std::string& trackName, int sampleRate);
|
||||
~Track();
|
||||
|
||||
/* audio production methods */
|
||||
void queueHttpGetAudio(const std::string& url, int gain = 0, bool loop = false);
|
||||
void queueHttpPostAudio(const std::string& url, int gain = 0, bool loop = false);
|
||||
void queueHttpPostAudio(const std::string& url, const std::string& body, std::vector<std::string>& headers, const std::string& proxy, int gain = 0, bool loop = false);
|
||||
void queueFileAudio(const std::string& path, int gain = 0, bool loop = false);
|
||||
void removeAllAudio();
|
||||
|
||||
void onPlayDone(bool hasError, const std::string& errMsg);
|
||||
|
||||
std::string& getTrackName() { return _trackName; }
|
||||
|
||||
/* audio playout methods */
|
||||
bool hasAudio();
|
||||
inline bool hasAudio_NoLock() const {
|
||||
return !_stopping && !_buffer.empty();
|
||||
}
|
||||
|
||||
|
||||
int retrieveAndClearAudio(int16_t* buf, int desiredSamples) {
|
||||
std::lock_guard<std::mutex> lock(_mutex);
|
||||
int samplesToCopy = std::min(static_cast<int>(_buffer.size()), desiredSamples);
|
||||
std::copy_n(_buffer.begin(), samplesToCopy, buf);
|
||||
_buffer.erase(_buffer.begin(), _buffer.begin() + samplesToCopy);
|
||||
return samplesToCopy;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string _trackName;
|
||||
int _sampleRate;
|
||||
std::mutex _mutex;
|
||||
CircularBuffer_t _buffer;
|
||||
std::queue<std::shared_ptr<AudioProducer>> _apQueue;
|
||||
bool _stopping;
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,483 @@
|
||||
#include "tts_vendor_parser.h"
|
||||
|
||||
#include <sstream>
|
||||
#include <switch_json.h>
|
||||
#include <map>
|
||||
|
||||
switch_status_t rimelabs_parse_text(const std::map<std::string, std::string>& params, const std::string& text,
|
||||
std::string& url, std::string& body, std::vector<std::string>& headers) {
|
||||
std::string api_key;
|
||||
std::string voice_name;
|
||||
std::string model_id;
|
||||
std::string speed_alpha;
|
||||
std::string reduce_latency;
|
||||
|
||||
for (const auto& pair : params) {
|
||||
if (pair.first == "api_key") {
|
||||
api_key = pair.second;
|
||||
} else if (pair.first == "voice") {
|
||||
voice_name = pair.second;
|
||||
} else if (pair.first == "model_id") {
|
||||
model_id = pair.second;
|
||||
} else if (pair.first == "speed_alpha") {
|
||||
speed_alpha = pair.second;
|
||||
} else if (pair.first == "reduce_latency") {
|
||||
reduce_latency = pair.second;
|
||||
}
|
||||
}
|
||||
|
||||
if (api_key.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "rimelabs_parse_text: no api_key provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (voice_name.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "rimelabs_parse_text: no voice_name provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (model_id.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "rimelabs_parse_text: no model_id provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
url = "https://users.rime.ai/v1/rime-tts";
|
||||
|
||||
/* create the JSON body */
|
||||
cJSON * jResult = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(jResult, "text", text.c_str());
|
||||
cJSON_AddNumberToObject(jResult, "samplingRate", 8000);
|
||||
if (!voice_name.empty()) {
|
||||
cJSON_AddStringToObject(jResult, "speaker", voice_name.c_str());
|
||||
}
|
||||
if (!model_id.empty()) {
|
||||
cJSON_AddStringToObject(jResult, "modelId", model_id.c_str());
|
||||
}
|
||||
if (!speed_alpha.empty()) {
|
||||
cJSON_AddNumberToObject(jResult, "speedAlpha", std::strtof(speed_alpha.c_str(), nullptr));
|
||||
}
|
||||
if (!reduce_latency.empty()) {
|
||||
cJSON_AddBoolToObject(jResult, "reduceLatency", !strcmp(reduce_latency.c_str(), "true") ? 1 : 0);
|
||||
}
|
||||
char* _body = cJSON_PrintUnformatted(jResult);
|
||||
body = _body;
|
||||
|
||||
cJSON_Delete(jResult);
|
||||
free(_body);
|
||||
|
||||
// Create headers
|
||||
headers.push_back("Authorization: Bearer " + api_key);
|
||||
headers.push_back("Accept: audio/mp3");
|
||||
headers.push_back("Content-Type: application/json");
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t whisper_parse_text(const std::map<std::string, std::string>& params, const std::string& text,
|
||||
std::string& url, std::string& body, std::vector<std::string>& headers) {
|
||||
std::string api_key;
|
||||
std::string voice_name;
|
||||
std::string model_id;
|
||||
std::string speed;
|
||||
|
||||
for (const auto& pair : params) {
|
||||
if (pair.first == "api_key") {
|
||||
api_key = pair.second;
|
||||
} else if (pair.first == "voice") {
|
||||
voice_name = pair.second;
|
||||
} else if (pair.first == "model_id") {
|
||||
model_id = pair.second;
|
||||
} else if (pair.first == "speed") {
|
||||
speed = pair.second;
|
||||
}
|
||||
}
|
||||
|
||||
if (api_key.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "whisper_parse_text: no api_key provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (model_id.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "whisper_parse_text: no model_id provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
url = "https://api.openai.com/v1/audio/speech";
|
||||
|
||||
/* create the JSON body */
|
||||
cJSON * jResult = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(jResult, "model", model_id.c_str());
|
||||
cJSON_AddStringToObject(jResult, "input", text.c_str());
|
||||
cJSON_AddStringToObject(jResult, "voice", voice_name.c_str());
|
||||
cJSON_AddStringToObject(jResult, "response_format", "mp3");
|
||||
if (!speed.empty()) {
|
||||
cJSON_AddStringToObject(jResult, "speed", speed.c_str());
|
||||
}
|
||||
char* _body = cJSON_PrintUnformatted(jResult);
|
||||
body = _body;
|
||||
|
||||
cJSON_Delete(jResult);
|
||||
free(_body);
|
||||
|
||||
// Create headers
|
||||
headers.push_back("Authorization: Bearer " + api_key);
|
||||
headers.push_back("Content-Type: application/json");
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t azure_parse_text(const std::map<std::string, std::string>& params, const std::string& text,
|
||||
std::string& url, std::string& body, std::vector<std::string>& headers, std::string& proxy) {
|
||||
|
||||
std::string api_key;
|
||||
std::string voice_name;
|
||||
std::string language;
|
||||
std::string region;
|
||||
std::string endpoint;
|
||||
std::string endpointId;
|
||||
std::string http_proxy_ip;
|
||||
std::string http_proxy_port;
|
||||
|
||||
for (const auto& pair : params) {
|
||||
if (pair.first == "api_key") {
|
||||
api_key = pair.second;
|
||||
} else if (pair.first == "voice") {
|
||||
voice_name = pair.second;
|
||||
} else if (pair.first == "language") {
|
||||
language = pair.second;
|
||||
} else if (pair.first == "region") {
|
||||
region = pair.second;
|
||||
} else if (pair.first == "endpoint") {
|
||||
endpoint = pair.second;
|
||||
} else if (pair.first == "endpointId") {
|
||||
endpointId = pair.second;
|
||||
} else if (pair.first == "http_proxy_ip") {
|
||||
http_proxy_ip = pair.second;
|
||||
} else if (pair.first == "http_proxy_port") {
|
||||
http_proxy_port = pair.second;
|
||||
}
|
||||
}
|
||||
|
||||
if (language.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "azure_parse_text: no language provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (voice_name.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "azure_parse_text: no voice_name provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (region.empty()) {
|
||||
region = "westus";
|
||||
}
|
||||
/* format url*/
|
||||
url = !endpoint.empty() ? endpoint : "https://" + region + ".tts.speech.microsoft.com/cognitiveservices/v1";
|
||||
|
||||
// Body
|
||||
if (strncmp(text.c_str(), "<speak", 6) == 0) {
|
||||
body = text;
|
||||
} else {
|
||||
std::ostringstream body_stream;
|
||||
body_stream << "<speak version=\"1.0\" xmlns=\"http://www.w3.org/2001/10/synthesis\" xmlns:mstts=\"https://www.w3.org/2001/mstts\" xml:lang=\"" << language << "\">";
|
||||
body_stream << "<voice name=\"" << voice_name << "\">";
|
||||
body_stream << text;
|
||||
body_stream << "</voice>";
|
||||
body_stream << "</speak>";
|
||||
body = body_stream.str();
|
||||
}
|
||||
|
||||
// Create headers
|
||||
if (!api_key.empty()) {
|
||||
headers.push_back("Ocp-Apim-Subscription-Key: " + api_key);
|
||||
}
|
||||
if (!endpointId.empty()) {
|
||||
headers.push_back("X-Microsoft-EndpointId: " + endpointId);
|
||||
}
|
||||
headers.push_back("Content-Type: application/ssml+xml");
|
||||
headers.push_back("X-Microsoft-OutputFormat: audio-16khz-32kbitrate-mono-mp3");
|
||||
|
||||
// Proxy
|
||||
std::ostringstream proxy_stream;
|
||||
if (!http_proxy_ip.empty()) {
|
||||
proxy_stream << "http://" << http_proxy_ip;
|
||||
if (!http_proxy_port.empty()) {
|
||||
proxy_stream << ":" << http_proxy_port;
|
||||
}
|
||||
}
|
||||
proxy = proxy_stream.str();
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t deepgram_parse_text(const std::map<std::string, std::string>& params, const std::string& text,
|
||||
std::string& url, std::string& body, std::vector<std::string>& headers) {
|
||||
|
||||
std::string api_key;
|
||||
std::string voice_name;
|
||||
|
||||
for (const auto& pair : params) {
|
||||
if (pair.first == "api_key") {
|
||||
api_key = pair.second;
|
||||
} else if (pair.first == "voice") {
|
||||
voice_name = pair.second;
|
||||
}
|
||||
}
|
||||
|
||||
if (api_key.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "deepgram_parse_text: no api_key provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (voice_name.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "deepgram_parse_text: no voice_name provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
/* format url*/
|
||||
std::ostringstream url_stream;
|
||||
url_stream << "https://api.deepgram.com/v1/speak?model=" << voice_name << "&encoding=mp3";
|
||||
url = url_stream.str();
|
||||
|
||||
/* create the JSON body */
|
||||
cJSON * jResult = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(jResult, "text", text.c_str());
|
||||
|
||||
char* _body = cJSON_PrintUnformatted(jResult);
|
||||
body = _body;
|
||||
|
||||
cJSON_Delete(jResult);
|
||||
free(_body);
|
||||
|
||||
// Create headers
|
||||
headers.push_back("Authorization: Token " + api_key);
|
||||
headers.push_back("Content-Type: application/json");
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t playht_parse_text(const std::map<std::string, std::string>& params, const std::string& text,
|
||||
std::string& url, std::string& body, std::vector<std::string>& headers) {
|
||||
|
||||
std::string api_key;
|
||||
std::string voice_name;
|
||||
std::string user_id;
|
||||
std::string quality;
|
||||
std::string speed;
|
||||
std::string seed;
|
||||
std::string temperature;
|
||||
std::string voice_engine;
|
||||
std::string emotion;
|
||||
std::string voice_guidance;
|
||||
std::string style_guidance;
|
||||
std::string text_guidance;
|
||||
|
||||
for (const auto& pair : params) {
|
||||
if (pair.first == "api_key") {
|
||||
api_key = pair.second;
|
||||
} else if (pair.first == "voice") {
|
||||
voice_name = pair.second;
|
||||
} else if (pair.first == "user_id") {
|
||||
user_id = pair.second;
|
||||
} else if (pair.first == "quality") {
|
||||
quality = pair.second;
|
||||
} else if (pair.first == "speed") {
|
||||
speed = pair.second;
|
||||
} else if (pair.first == "seed") {
|
||||
seed = pair.second;
|
||||
} else if (pair.first == "temperature") {
|
||||
temperature = pair.second;
|
||||
} else if (pair.first == "voice_engine") {
|
||||
voice_engine = pair.second;
|
||||
} else if (pair.first == "emotion") {
|
||||
emotion = pair.second;
|
||||
} else if (pair.first == "voice_guidance") {
|
||||
voice_guidance = pair.second;
|
||||
} else if (pair.first == "style_guidance") {
|
||||
style_guidance = pair.second;
|
||||
} else if (pair.first == "text_guidance") {
|
||||
text_guidance = pair.second;
|
||||
}
|
||||
}
|
||||
|
||||
if (api_key.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "playht_parse_text: no api_key provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (user_id.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "playht_parse_text: no user_id provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (voice_name.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "playht_parse_text: no voice_name provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
// URL
|
||||
url = "https://api.play.ht/api/v2/tts/stream";
|
||||
|
||||
/* create the JSON body */
|
||||
cJSON * jResult = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(jResult, "text", text.c_str());
|
||||
cJSON_AddStringToObject(jResult, "voice", voice_name.c_str());
|
||||
cJSON_AddStringToObject(jResult, "output_format", "mp3");
|
||||
cJSON_AddNumberToObject(jResult, "sample_rate", 8000);
|
||||
if (!voice_engine.empty()) {
|
||||
cJSON_AddStringToObject(jResult, "voice_engine", voice_engine.c_str());
|
||||
}
|
||||
if (!quality.empty()) {
|
||||
cJSON_AddStringToObject(jResult, "quality", quality.c_str());
|
||||
}
|
||||
if (!speed.empty()) {
|
||||
cJSON_AddNumberToObject(jResult, "speed", atoi(speed.c_str()));
|
||||
}
|
||||
if (!seed.empty()) {
|
||||
cJSON_AddNumberToObject(jResult, "seed", atoi(seed.c_str()));
|
||||
}
|
||||
if (!temperature.empty()) {
|
||||
cJSON_AddNumberToObject(jResult, "temperature", std::strtof(temperature.c_str(), nullptr));
|
||||
}
|
||||
if (!emotion.empty()) {
|
||||
cJSON_AddStringToObject(jResult, "emotion", emotion.c_str());
|
||||
}
|
||||
if (!voice_guidance.empty()) {
|
||||
cJSON_AddNumberToObject(jResult, "voice_guidance", atoi(voice_guidance.c_str()));
|
||||
}
|
||||
if (!style_guidance.empty()) {
|
||||
cJSON_AddNumberToObject(jResult, "style_guidance", atoi(style_guidance.c_str()));
|
||||
}
|
||||
if (!text_guidance.empty()) {
|
||||
cJSON_AddNumberToObject(jResult, "text_guidance", atoi(text_guidance.c_str()));
|
||||
}
|
||||
char* _body = cJSON_PrintUnformatted(jResult);
|
||||
body = _body;
|
||||
|
||||
cJSON_Delete(jResult);
|
||||
free(_body);
|
||||
|
||||
// Create headers
|
||||
headers.push_back("AUTHORIZATION: " + api_key);
|
||||
headers.push_back("X-USER-ID: " + user_id);
|
||||
headers.push_back("Accept: audio/mpeg");
|
||||
headers.push_back("Content-Type: application/json");
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t elevenlabs_parse_text(const std::map<std::string, std::string>& params, const std::string& text,
|
||||
std::string& url, std::string& body, std::vector<std::string>& headers) {
|
||||
|
||||
std::string api_key;
|
||||
std::string voice_name;
|
||||
std::string model_id;
|
||||
std::string similarity_boost;
|
||||
std::string stability;
|
||||
std::string style;
|
||||
std::string use_speaker_boost;
|
||||
std::string optimize_streaming_latency;
|
||||
|
||||
for (const auto& pair : params) {
|
||||
if (pair.first == "api_key") {
|
||||
api_key = pair.second;
|
||||
} else if (pair.first == "voice") {
|
||||
voice_name = pair.second;
|
||||
} else if (pair.first == "model_id") {
|
||||
model_id = pair.second;
|
||||
} else if (pair.first == "similarity_boost") {
|
||||
similarity_boost = pair.second;
|
||||
} else if (pair.first == "stability") {
|
||||
stability = pair.second;
|
||||
} else if (pair.first == "style") {
|
||||
style = pair.second;
|
||||
} else if (pair.first == "use_speaker_boost") {
|
||||
use_speaker_boost = pair.second;
|
||||
} else if (pair.first == "modeloptimize_streaming_latency_id") {
|
||||
optimize_streaming_latency = pair.second;
|
||||
}
|
||||
}
|
||||
|
||||
if (api_key.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "elevenlabs_parse_text: no api_key provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (model_id.empty()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "elevenlabs_parse_text: no model_id provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (optimize_streaming_latency.empty()) {
|
||||
optimize_streaming_latency = "2";
|
||||
}
|
||||
|
||||
// URL
|
||||
std::ostringstream url_stream;
|
||||
url_stream << "https://api.elevenlabs.io/v1/text-to-speech/" << voice_name << "/stream?";
|
||||
url_stream << "optimize_streaming_latency=" << optimize_streaming_latency << "&output_format=mp3_44100_128";
|
||||
url = url_stream.str();
|
||||
|
||||
/* create the JSON body */
|
||||
cJSON * jResult = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(jResult, "model_id", model_id.c_str());
|
||||
cJSON_AddStringToObject(jResult, "text", text.c_str());
|
||||
if (!similarity_boost.empty() || !style.empty() || !use_speaker_boost.empty() || !stability.empty()) {
|
||||
cJSON * jVoiceSettings = cJSON_CreateObject();
|
||||
cJSON_AddItemToObject(jResult, "voice_settings", jVoiceSettings);
|
||||
if (!similarity_boost.empty()) {
|
||||
cJSON_AddStringToObject(jVoiceSettings, "similarity_boost", similarity_boost.c_str());
|
||||
}
|
||||
if (!style.empty()) {
|
||||
cJSON_AddStringToObject(jVoiceSettings, "style", style.c_str());
|
||||
}
|
||||
if (!use_speaker_boost.empty()) {
|
||||
cJSON_AddStringToObject(jVoiceSettings, "use_speaker_boost", use_speaker_boost.c_str());
|
||||
}
|
||||
if (!stability.empty()) {
|
||||
cJSON_AddStringToObject(jVoiceSettings, "stability", stability.c_str());
|
||||
}
|
||||
}
|
||||
char* _body = cJSON_PrintUnformatted(jResult);
|
||||
body = _body;
|
||||
|
||||
cJSON_Delete(jResult);
|
||||
free(_body);
|
||||
|
||||
// Create headers
|
||||
headers.push_back("xi-api-key: " + api_key);
|
||||
headers.push_back("Content-Type: application/json");
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t tts_vendor_parse_text(const std::string& say, std::string& url, std::string& body, std::vector<std::string>& headers, std::string& proxy) {
|
||||
size_t start = say.find("{") + 1;
|
||||
size_t end = say.find("}");
|
||||
|
||||
std::string text = say.substr(end + 1);
|
||||
|
||||
std::string params_string = say.substr(start, end - start);
|
||||
std::istringstream ss(params_string);
|
||||
std::map<std::string, std::string> params;
|
||||
|
||||
while (ss.good()) {
|
||||
std::string substr;
|
||||
getline(ss, substr, ',');
|
||||
substr.erase(0, substr.find_first_not_of(' '));
|
||||
|
||||
size_t equal_pos = substr.find("=");
|
||||
std::string key = substr.substr(0, equal_pos);
|
||||
std::string value = substr.substr(equal_pos + 1, substr.size());
|
||||
|
||||
params[key] = value;
|
||||
}
|
||||
|
||||
if (params["vendor"] == "elevenlabs") {
|
||||
return elevenlabs_parse_text(params, text, url, body, headers);
|
||||
} else if (params["vendor"] == "deepgram") {
|
||||
return deepgram_parse_text(params, text, url, body, headers);
|
||||
} else if (params["vendor"] == "microsoft") {
|
||||
return azure_parse_text(params, text, url, body, headers, proxy);
|
||||
} else if (params["vendor"] == "whisper") {
|
||||
return whisper_parse_text(params, text, url, body, headers);
|
||||
} else if (params["vendor"] == "playht") {
|
||||
return playht_parse_text(params, text, url, body, headers);
|
||||
} else if (params["vendor"] == "rimelabs") {
|
||||
return rimelabs_parse_text(params, text, url, body, headers);
|
||||
} else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "tts_vendor_parse_text: There is no available parser for vendor %s\n", params["vendor"]);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#ifndef __MOD_DUB_TTS_VENDOR_PARSER_H__
|
||||
#define __MOD_DUB_TTS_VENDOR_PARSER_H__
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <switch.h>
|
||||
#include "common.h"
|
||||
|
||||
|
||||
switch_status_t tts_vendor_parse_text(const std::string& say, std::string& url, std::string& body, std::vector<std::string>& headers, std::string& proxy);
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,214 @@
|
||||
#include "vector_math.h"
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
#include <cstdlib>
|
||||
|
||||
|
||||
#define GRANULAR_VOLUME_MAX (50)
|
||||
#define SMAX 32767
|
||||
#define SMIN (-32768)
|
||||
#define normalize_to_16bit_basic(n) if (n > SMAX) n = SMAX; else if (n < SMIN) n = SMIN;
|
||||
#define normalize_volume_granular(x) if (x > GRANULAR_VOLUME_MAX) x = GRANULAR_VOLUME_MAX; if (x < -GRANULAR_VOLUME_MAX) x = -GRANULAR_VOLUME_MAX;
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#if defined(USE_AVX2)
|
||||
#include <immintrin.h>
|
||||
#pragma message("Using AVX2 SIMD.")
|
||||
void vector_add(int16_t* a, int16_t* b, size_t len) {
|
||||
size_t i = 0;
|
||||
for (; i + 15 < len; i += 16) {
|
||||
__m256i va = _mm256_loadu_si256((const __m256i*)(a + i));
|
||||
__m256i vb = _mm256_loadu_si256((const __m256i*)(b + i));
|
||||
__m256i vc = _mm256_add_epi16(va, vb);
|
||||
_mm256_storeu_si256((__m256i*)(a + i), vc);
|
||||
}
|
||||
for (; i < len; ++i) {
|
||||
a[i] += b[i];
|
||||
}
|
||||
}
|
||||
void vector_normalize(int16_t* a, size_t len) {
|
||||
__m256i max_val = _mm256_set1_epi16(SMAX);
|
||||
__m256i min_val = _mm256_set1_epi16(SMIN);
|
||||
|
||||
size_t i = 0;
|
||||
for (; i + 15 < len; i += 16) {
|
||||
__m256i values = _mm256_loadu_si256((__m256i*)(a + i));
|
||||
__m256i gt_max = _mm256_cmpgt_epi16(values, max_val);
|
||||
__m256i lt_min = _mm256_cmpgt_epi16(min_val, values);
|
||||
values = _mm256_blendv_epi8(values, max_val, gt_max);
|
||||
values = _mm256_blendv_epi8(values, min_val, lt_min);
|
||||
_mm256_storeu_si256((__m256i*)(a + i), values);
|
||||
}
|
||||
|
||||
// Process remaining elements
|
||||
for (; i < len; ++i) {
|
||||
if (a[i] > SMAX) a[i] = SMAX;
|
||||
else if (a[i] < SMIN) a[i] = SMIN;
|
||||
}
|
||||
}
|
||||
typedef union {
|
||||
int16_t* data;
|
||||
__m256i* fp_avx2;
|
||||
} vector_data_t;
|
||||
|
||||
void vector_change_sln_volume_granular(int16_t* data, uint32_t samples, int32_t vol) {
|
||||
float newrate = 0;
|
||||
static const float pos[GRANULAR_VOLUME_MAX] = {
|
||||
1.122018, 1.258925, 1.412538, 1.584893, 1.778279, 1.995262, 2.238721, 2.511887, 2.818383, 3.162278,
|
||||
3.548134, 3.981072, 4.466835, 5.011872, 5.623413, 6.309574, 7.079458, 7.943282, 8.912509, 10.000000,
|
||||
11.220183, 12.589254, 14.125375, 15.848933, 17.782795, 19.952621, 22.387213, 25.118862, 28.183832, 31.622776,
|
||||
35.481335, 39.810719, 44.668358, 50.118729, 56.234131, 63.095726, 70.794586, 79.432816, 89.125107, 100.000000,
|
||||
112.201836, 125.892517, 141.253784, 158.489334, 177.827942, 199.526215, 223.872070, 251.188705, 281.838318, 316.227753
|
||||
};
|
||||
static const float neg[GRANULAR_VOLUME_MAX] = {
|
||||
0.891251, 0.794328, 0.707946, 0.630957, 0.562341, 0.501187, 0.446684, 0.398107, 0.354813, 0.316228,
|
||||
0.281838, 0.251189, 0.223872, 0.199526, 0.177828, 0.158489, 0.141254, 0.125893, 0.112202, 0.100000,
|
||||
0.089125, 0.079433, 0.070795, 0.063096, 0.056234, 0.050119, 0.044668, 0.039811, 0.035481, 0.031623,
|
||||
0.028184, 0.025119, 0.022387, 0.019953, 0.017783, 0.015849, 0.014125, 0.012589, 0.011220, 0.010000,
|
||||
0.008913, 0.007943, 0.007079, 0.006310, 0.005623, 0.005012, 0.004467, 0.003981, 0.003548, 0.000000 // NOTE mapped -50 dB ratio to total silence instead of 0.003162
|
||||
};
|
||||
const float* chart;
|
||||
uint32_t i = abs(vol) - 1;
|
||||
|
||||
if (vol == 0) return;
|
||||
normalize_volume_granular(vol);
|
||||
|
||||
chart = vol > 0 ? pos : neg;
|
||||
|
||||
newrate = chart[i];
|
||||
|
||||
if (newrate) {
|
||||
__m256 scale_factor_reg = _mm256_set1_ps(newrate);
|
||||
uint32_t processed_samples = samples - (samples % 8); // Ensure we process only multiples of 8
|
||||
for (uint32_t i = 0; i < processed_samples; i += 8) {
|
||||
__m128i data_ = _mm_loadu_si128((__m128i*)(data + i));
|
||||
__m256i data_32 = _mm256_cvtepi16_epi32(data_);
|
||||
|
||||
__m256 data_float = _mm256_cvtepi32_ps(data_32);
|
||||
__m256 result = _mm256_mul_ps(data_float, scale_factor_reg);
|
||||
|
||||
__m256i result_32 = _mm256_cvtps_epi32(result);
|
||||
|
||||
// Handle saturation
|
||||
__m256i min_val = _mm256_set1_epi32(SMIN);
|
||||
__m256i max_val = _mm256_set1_epi32(SMAX);
|
||||
result_32 = _mm256_min_epi32(result_32, max_val);
|
||||
result_32 = _mm256_max_epi32(result_32, min_val);
|
||||
|
||||
__m128i result_16 = _mm_packs_epi32(_mm256_castsi256_si128(result_32), _mm256_extractf128_si256(result_32, 1));
|
||||
|
||||
_mm_storeu_si128((__m128i*)(data + i), result_16);
|
||||
}
|
||||
|
||||
// Process any remaining samples
|
||||
for (uint32_t i = processed_samples; i < samples; i++) {
|
||||
int32_t tmp = (int32_t)(data[i] * newrate);
|
||||
tmp = tmp > SMAX ? SMAX : (tmp < SMIN ? SMIN : tmp);
|
||||
data[i] = (int16_t)tmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
#elif defined(USE_SSE2)
|
||||
#include <emmintrin.h>
|
||||
#pragma message("Using SSE2 SIMD.")
|
||||
void vector_add(int16_t* a, int16_t* b, size_t len) {
|
||||
size_t i = 0;
|
||||
for (; i + 7 < len; i += 8) {
|
||||
__m128i va = _mm_loadu_si128((const __m128i*)(a + i));
|
||||
__m128i vb = _mm_loadu_si128((const __m128i*)(b + i));
|
||||
__m128i vc = _mm_add_epi16(va, vb);
|
||||
_mm_storeu_si128((__m128i*)(a + i), vc);
|
||||
}
|
||||
for (; i < len; ++i) {
|
||||
a[i] += b[i];
|
||||
}
|
||||
}
|
||||
void vector_normalize(int16_t* a, size_t len) {
|
||||
__m128i max_val = _mm_set1_epi16(SMAX);
|
||||
__m128i min_val = _mm_set1_epi16(SMIN);
|
||||
|
||||
size_t i = 0;
|
||||
for (; i + 7 < len; i += 8) {
|
||||
__m128i values = _mm_loadu_si128((__m128i*)(a + i));
|
||||
__m128i gt_max = _mm_cmpgt_epi16(values, max_val);
|
||||
__m128i lt_min = _mm_cmpgt_epi16(min_val, values);
|
||||
__m128i max_masked = _mm_and_si128(gt_max, max_val);
|
||||
__m128i min_masked = _mm_and_si128(lt_min, min_val);
|
||||
__m128i other_masked = _mm_andnot_si128(_mm_or_si128(gt_max, lt_min), values);
|
||||
values = _mm_or_si128(_mm_or_si128(max_masked, min_masked), other_masked);
|
||||
_mm_storeu_si128((__m128i*)(a + i), values);
|
||||
}
|
||||
|
||||
// Process remaining elements
|
||||
for (; i < len; ++i) {
|
||||
if (a[i] > SMAX) a[i] = SMAX;
|
||||
else if (a[i] < SMIN) a[i] = SMIN;
|
||||
}
|
||||
}
|
||||
|
||||
typedef union {
|
||||
int16_t* data;
|
||||
__m128i* fp_sse2;
|
||||
} vector_data_t;
|
||||
|
||||
#else
|
||||
#pragma message("Building without vector math support")
|
||||
void vector_add(int16_t* a, int16_t* b, size_t len) {
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
a[i] += b[i];
|
||||
}
|
||||
}
|
||||
void vector_normalize(int16_t* a, size_t len) {
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
normalize_to_16bit_basic(a[i]);
|
||||
}
|
||||
}
|
||||
void vector_change_sln_volume_granular(int16_t* data, uint32_t samples, int32_t vol) {
|
||||
float newrate = 0;
|
||||
static const float pos[GRANULAR_VOLUME_MAX] = {
|
||||
1.122018, 1.258925, 1.412538, 1.584893, 1.778279, 1.995262, 2.238721, 2.511887, 2.818383, 3.162278,
|
||||
3.548134, 3.981072, 4.466835, 5.011872, 5.623413, 6.309574, 7.079458, 7.943282, 8.912509, 10.000000,
|
||||
11.220183, 12.589254, 14.125375, 15.848933, 17.782795, 19.952621, 22.387213, 25.118862, 28.183832, 31.622776,
|
||||
35.481335, 39.810719, 44.668358, 50.118729, 56.234131, 63.095726, 70.794586, 79.432816, 89.125107, 100.000000,
|
||||
112.201836, 125.892517, 141.253784, 158.489334, 177.827942, 199.526215, 223.872070, 251.188705, 281.838318, 316.227753
|
||||
};
|
||||
static const float neg[GRANULAR_VOLUME_MAX] = {
|
||||
0.891251, 0.794328, 0.707946, 0.630957, 0.562341, 0.501187, 0.446684, 0.398107, 0.354813, 0.316228,
|
||||
0.281838, 0.251189, 0.223872, 0.199526, 0.177828, 0.158489, 0.141254, 0.125893, 0.112202, 0.100000,
|
||||
0.089125, 0.079433, 0.070795, 0.063096, 0.056234, 0.050119, 0.044668, 0.039811, 0.035481, 0.031623,
|
||||
0.028184, 0.025119, 0.022387, 0.019953, 0.017783, 0.015849, 0.014125, 0.012589, 0.011220, 0.010000,
|
||||
0.008913, 0.007943, 0.007079, 0.006310, 0.005623, 0.005012, 0.004467, 0.003981, 0.003548, 0.000000 // NOTE mapped -50 dB ratio to total silence instead of 0.003162
|
||||
};
|
||||
const float* chart;
|
||||
uint32_t i;
|
||||
|
||||
if (vol == 0) return;
|
||||
|
||||
normalize_volume_granular(vol);
|
||||
|
||||
chart = vol > 0 ? pos : neg;
|
||||
|
||||
i = abs(vol) - 1;
|
||||
assert(i < GRANULAR_VOLUME_MAX);
|
||||
newrate = chart[i];
|
||||
if (newrate) {
|
||||
int32_t tmp;
|
||||
uint32_t x;
|
||||
int16_t *fp = data;
|
||||
|
||||
for (x = 0; x < samples; x++) {
|
||||
tmp = (int32_t) (fp[x] * newrate);
|
||||
normalize_to_16bit_basic(tmp);
|
||||
fp[x] = (int16_t) tmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,20 @@
|
||||
#ifndef VECTOR_MATH_H
|
||||
#define VECTOR_MATH_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
void vector_add(int16_t* a, int16_t* b, size_t len);
|
||||
void vector_normalize(int16_t* a, size_t len);
|
||||
void vector_change_sln_volume_granular(int16_t* data, uint32_t samples, int32_t vol);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -445,15 +445,12 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) el->circularBuffer;
|
||||
std::vector<uint16_t> pcm_data;
|
||||
|
||||
if (conn->flushed) {
|
||||
if (conn->flushed || cBuffer == nullptr) {
|
||||
/* this will abort the transfer */
|
||||
return 0;
|
||||
}
|
||||
{
|
||||
switch_mutex_lock(el->mutex);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: received data, response %ld\n",
|
||||
el->response_code);
|
||||
|
||||
if (el->response_code > 0 && el->response_code != 200) {
|
||||
std::string body((char *) ptr, bytes_received);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: received body %s\n", body.c_str());
|
||||
@@ -552,9 +549,28 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
|
||||
return bytes_received;
|
||||
}
|
||||
|
||||
static int extract_response_code(const std::string& input) {
|
||||
std::size_t space_pos = input.find(' ');
|
||||
if (space_pos == std::string::npos) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid HTTP response format %s\n", input.c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::size_t code_start_pos = space_pos + 1;
|
||||
std::size_t code_end_pos = input.find(' ', code_start_pos);
|
||||
if (code_end_pos == std::string::npos) {
|
||||
code_end_pos = input.length();
|
||||
}
|
||||
|
||||
std::string code_str = input.substr(code_start_pos, code_end_pos - code_start_pos);
|
||||
int response_code = std::stoi(code_str);
|
||||
return response_code;
|
||||
}
|
||||
|
||||
|
||||
static size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo_t *conn) {
|
||||
size_t bytes_received = size * nitems;
|
||||
const std::string prefix = "HTTP/2 ";
|
||||
const std::string prefix = "HTTP/ ";
|
||||
elevenlabs_t* el = conn->elevenlabs;
|
||||
std::string header, value;
|
||||
std::string input(buffer, bytes_received);
|
||||
@@ -565,11 +581,11 @@ static size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo
|
||||
else if (0 == header.compare("history-item-id")) el->history_item_id = strdup(value.c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s\n", input.c_str());
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: %s\n", input.c_str());
|
||||
if (input.rfind(prefix, 0) == 0) {
|
||||
try {
|
||||
el->response_code = std::stoi(input.substr(prefix.length()));
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "parsed response code: %ld\n", el->response_code);
|
||||
el->response_code = extract_response_code(input);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: parsed response code: %ld\n", el->response_code);
|
||||
} catch (const std::invalid_argument& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "header_callback: invalid response code %s\n", input.substr(prefix.length()).c_str());
|
||||
}
|
||||
@@ -662,14 +678,14 @@ extern "C" {
|
||||
/* create temp folder for cache files */
|
||||
const char* baseDir = std::getenv("JAMBONZ_TMP_CACHE_FOLDER");
|
||||
if (!baseDir) {
|
||||
baseDir = "/var/";
|
||||
baseDir = "/tmp/";
|
||||
}
|
||||
if (strcmp(baseDir, "/") == 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", baseDir);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
fullDirPath = std::string(baseDir) + "jambonz-tts-cache-files";
|
||||
fullDirPath = std::string(baseDir) + "tts-cache-files";
|
||||
|
||||
// Create the directory with read, write, and execute permissions for everyone
|
||||
mode_t oldMask = umask(0);
|
||||
@@ -822,7 +838,6 @@ extern "C" {
|
||||
CURL* easy = createEasyHandle();
|
||||
|
||||
el->conn = (void *) conn ;
|
||||
el->sample_rate = 0;
|
||||
conn->elevenlabs = el;
|
||||
conn->easy = easy;
|
||||
conn->global = &global;
|
||||
@@ -833,20 +848,12 @@ extern "C" {
|
||||
|
||||
el->circularBuffer = (void *) new CircularBuffer_t(8192);
|
||||
|
||||
if (el->session_id) {
|
||||
if (el->rate != 8000 /*Hz*/) {
|
||||
int err;
|
||||
switch_codec_implementation_t read_impl;
|
||||
switch_core_session_t *psession = switch_core_session_locate(el->session_id);
|
||||
switch_core_session_get_read_impl(psession, &read_impl);
|
||||
uint32_t samples_per_second = !strcasecmp(read_impl.iananame, "g722") ? read_impl.actual_samples_per_second : read_impl.samples_per_second;
|
||||
el->sample_rate = samples_per_second;
|
||||
// elevenlabs output is PCMU 8000
|
||||
if (samples_per_second != 8000 /*Hz*/) {
|
||||
el->resampler = speex_resampler_init(1, 8000, samples_per_second, SWITCH_RESAMPLE_QUALITY, &err);
|
||||
if (0 != err) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing resampler: %s.\n", speex_resampler_strerror(err));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
el->resampler = speex_resampler_init(1, 8000, el->rate, SWITCH_RESAMPLE_QUALITY, &err);
|
||||
if (0 != err) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing resampler: %s.\n", speex_resampler_strerror(err));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -876,7 +883,10 @@ extern "C" {
|
||||
curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
|
||||
//curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
|
||||
// libcurl adding random byte to the response body that creates white noise to audio file
|
||||
// https://github.com/curl/curl/issues/10525
|
||||
const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
|
||||
curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
|
||||
|
||||
rc = curl_multi_add_handle(global.multi, conn->easy);
|
||||
mcode_test("new_conn: curl_multi_add_handle", rc);
|
||||
@@ -920,9 +930,7 @@ extern "C" {
|
||||
switch_mutex_unlock(el->mutex);
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
size_t size = el->sample_rate ?
|
||||
std::min((*datalen/(2 * el->sample_rate / 8000)), cBuffer->size()) :
|
||||
std::min((*datalen/2), cBuffer->size());
|
||||
size_t size = std::min((*datalen/(2 * el->rate / 8000)), cBuffer->size());
|
||||
pcm_data.insert(pcm_data.end(), cBuffer->begin(), cBuffer->begin() + size);
|
||||
cBuffer->erase(cBuffer->begin(), cBuffer->begin() + size);
|
||||
switch_mutex_unlock(el->mutex);
|
||||
|
||||
@@ -31,7 +31,6 @@ struct elevenlabs_data {
|
||||
char *cache_filename;
|
||||
|
||||
int rate;
|
||||
uint32_t sample_rate;
|
||||
|
||||
void *conn;
|
||||
FILE *file;
|
||||
|
||||
@@ -2,7 +2,7 @@ include $(top_srcdir)/build/modmake.rulesam
|
||||
MODNAME=mod_google_transcribe
|
||||
|
||||
mod_LTLIBRARIES = mod_google_transcribe.la
|
||||
mod_google_transcribe_la_SOURCES = mod_google_transcribe.c google_glue.cpp
|
||||
mod_google_transcribe_la_SOURCES = mod_google_transcribe.c google_glue.cpp google_glue_v1.cpp google_glue_v2.cpp
|
||||
mod_google_transcribe_la_CFLAGS = $(AM_CFLAGS)
|
||||
mod_google_transcribe_la_CXXFLAGS = -I $(top_srcdir)/libs/googleapis/gens $(AM_CXXFLAGS) -std=c++17
|
||||
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
#ifndef __GENERIC_GOOGLE_GLUE_H__
|
||||
#define __GENERIC_GOOGLE_GLUE_H__
|
||||
|
||||
#include <switch_json.h>
|
||||
|
||||
template<typename Streamer>
|
||||
switch_bool_t google_speech_frame(switch_media_bug_t *bug, void* user_data) {
|
||||
switch_core_session_t *session = switch_core_media_bug_get_session(bug);
|
||||
struct cap_cb *cb = (struct cap_cb *) user_data;
|
||||
if (cb->streamer && (!cb->wants_single_utterance || !cb->got_end_of_utterance)) {
|
||||
Streamer* streamer = (Streamer *) cb->streamer;
|
||||
uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
|
||||
switch_frame_t frame = {};
|
||||
frame.data = data;
|
||||
frame.buflen = SWITCH_RECOMMENDED_BUFFER_SIZE;
|
||||
|
||||
if (switch_mutex_trylock(cb->mutex) == SWITCH_STATUS_SUCCESS) {
|
||||
while (switch_core_media_bug_read(bug, &frame, SWITCH_TRUE) == SWITCH_STATUS_SUCCESS && !switch_test_flag((&frame), SFF_CNG)) {
|
||||
if (frame.datalen) {
|
||||
if (cb->vad && !streamer->isConnected()) {
|
||||
switch_vad_state_t state = switch_vad_process(cb->vad, (int16_t*) frame.data, frame.samples);
|
||||
if (state == SWITCH_VAD_STATE_START_TALKING) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "detected speech, connect to google speech now\n");
|
||||
streamer->connect();
|
||||
cb->responseHandler(session, "vad_detected", cb->bugname);
|
||||
}
|
||||
}
|
||||
|
||||
if (cb->resampler) {
|
||||
spx_int16_t out[SWITCH_RECOMMENDED_BUFFER_SIZE];
|
||||
spx_uint32_t out_len = SWITCH_RECOMMENDED_BUFFER_SIZE;
|
||||
spx_uint32_t in_len = frame.samples;
|
||||
size_t written;
|
||||
|
||||
speex_resampler_process_interleaved_int(cb->resampler,
|
||||
(const spx_int16_t *) frame.data,
|
||||
(spx_uint32_t *) &in_len,
|
||||
&out[0],
|
||||
&out_len);
|
||||
streamer->write( &out[0], sizeof(spx_int16_t) * out_len);
|
||||
}
|
||||
else {
|
||||
streamer->write( frame.data, sizeof(spx_int16_t) * frame.samples);
|
||||
}
|
||||
}
|
||||
}
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
}
|
||||
}
|
||||
return SWITCH_TRUE;
|
||||
}
|
||||
|
||||
template<typename Streamer>
|
||||
switch_status_t google_speech_session_init(switch_core_session_t *session, responseHandler_t responseHandler,
|
||||
switch_thread_start_t func, uint32_t to_rate, uint32_t samples_per_second, uint32_t channels, char* lang,
|
||||
int interim, char *bugname, int single_utterance, int separate_recognition, int max_alternatives,
|
||||
int profanity_filter, int word_time_offset, int punctuation, const char* model, int enhanced,
|
||||
const char* hints, char* play_file, void **ppUserData) {
|
||||
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
auto read_codec = switch_core_session_get_read_codec(session);
|
||||
uint32_t sampleRate = read_codec->implementation->actual_samples_per_second;
|
||||
struct cap_cb *cb;
|
||||
int err;
|
||||
|
||||
cb =(struct cap_cb *) switch_core_session_alloc(session, sizeof(*cb));
|
||||
strncpy(cb->sessionId, switch_core_session_get_uuid(session), MAX_SESSION_ID);
|
||||
strncpy(cb->bugname, bugname, MAX_BUG_LEN);
|
||||
cb->got_end_of_utterance = 0;
|
||||
cb->wants_single_utterance = single_utterance;
|
||||
if (play_file != NULL){
|
||||
cb->play_file = 1;
|
||||
}
|
||||
|
||||
switch_mutex_init(&cb->mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session));
|
||||
if (sampleRate != to_rate) {
|
||||
cb->resampler = speex_resampler_init(channels, sampleRate, to_rate, SWITCH_RESAMPLE_QUALITY, &err);
|
||||
if (0 != err) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing resampler: %s.\n",
|
||||
switch_channel_get_name(channel), speex_resampler_strerror(err));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
} else {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "%s: no resampling needed for this call\n", switch_channel_get_name(channel));
|
||||
}
|
||||
cb->responseHandler = responseHandler;
|
||||
|
||||
// allocate vad if we are delaying connecting to the recognizer until we detect speech
|
||||
if (switch_channel_var_true(channel, "START_RECOGNIZING_ON_VAD")) {
|
||||
cb->vad = switch_vad_init(sampleRate, channels);
|
||||
if (cb->vad) {
|
||||
const char* var;
|
||||
int mode = 2;
|
||||
int silence_ms = 150;
|
||||
int voice_ms = 250;
|
||||
int debug = 0;
|
||||
|
||||
if (var = switch_channel_get_variable(channel, "RECOGNIZER_VAD_MODE")) {
|
||||
mode = atoi(var);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "RECOGNIZER_VAD_SILENCE_MS")) {
|
||||
silence_ms = atoi(var);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "RECOGNIZER_VAD_VOICE_MS")) {
|
||||
voice_ms = atoi(var);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "RECOGNIZER_VAD_VOICE_MS")) {
|
||||
voice_ms = atoi(var);
|
||||
}
|
||||
switch_vad_set_mode(cb->vad, mode);
|
||||
switch_vad_set_param(cb->vad, "silence_ms", silence_ms);
|
||||
switch_vad_set_param(cb->vad, "voice_ms", voice_ms);
|
||||
switch_vad_set_param(cb->vad, "debug", debug);
|
||||
}
|
||||
}
|
||||
|
||||
Streamer *streamer = NULL;
|
||||
try {
|
||||
streamer = new Streamer(session, channels, lang, interim, to_rate, sampleRate, single_utterance, separate_recognition, max_alternatives,
|
||||
profanity_filter, word_time_offset, punctuation, model, enhanced, hints);
|
||||
cb->streamer = streamer;
|
||||
} catch (std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing gstreamer: %s.\n",
|
||||
switch_channel_get_name(channel), e.what());
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (!cb->vad) streamer->connect();
|
||||
|
||||
// create the read thread
|
||||
switch_threadattr_t *thd_attr = NULL;
|
||||
switch_memory_pool_t *pool = switch_core_session_get_pool(session);
|
||||
|
||||
switch_threadattr_create(&thd_attr, pool);
|
||||
switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE);
|
||||
switch_thread_create(&cb->thread, thd_attr, func, cb, pool);
|
||||
|
||||
*ppUserData = cb;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
template<typename Streamer>
|
||||
switch_status_t google_speech_session_cleanup(switch_core_session_t *session, int channelIsClosing, switch_media_bug_t *bug) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
|
||||
if (bug) {
|
||||
struct cap_cb *cb = (struct cap_cb *) switch_core_media_bug_get_user_data(bug);
|
||||
switch_mutex_lock(cb->mutex);
|
||||
|
||||
if (!switch_channel_get_private(channel, cb->bugname)) {
|
||||
// race condition
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s Bug is not attached (race).\n", switch_channel_get_name(channel));
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
switch_channel_set_private(channel, cb->bugname, NULL);
|
||||
|
||||
// stop playback if available
|
||||
if (cb->play_file == 1){
|
||||
if (switch_channel_test_flag(channel, CF_BROADCAST)) {
|
||||
switch_channel_stop_broadcast(channel);
|
||||
} else {
|
||||
switch_channel_set_flag_value(channel, CF_BREAK, 1);
|
||||
}
|
||||
}
|
||||
|
||||
// close connection and get final responses
|
||||
Streamer* streamer = (Streamer *) cb->streamer;
|
||||
|
||||
if (streamer) {
|
||||
streamer->writesDone();
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "google_speech_session_cleanup: GStreamer (%p) waiting for read thread to complete\n", (void*)streamer);
|
||||
switch_status_t st;
|
||||
switch_thread_join(&st, cb->thread);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "google_speech_session_cleanup: GStreamer (%p) read thread completed\n", (void*)streamer);
|
||||
|
||||
delete streamer;
|
||||
cb->streamer = NULL;
|
||||
}
|
||||
|
||||
if (cb->resampler) {
|
||||
speex_resampler_destroy(cb->resampler);
|
||||
}
|
||||
if (cb->vad) {
|
||||
switch_vad_destroy(&cb->vad);
|
||||
cb->vad = nullptr;
|
||||
}
|
||||
if (!channelIsClosing) {
|
||||
switch_core_media_bug_remove(session, &bug);
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "google_speech_session_cleanup: Closed stream\n");
|
||||
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s Bug is not attached.\n", switch_channel_get_name(channel));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
template<typename PhraseSet>
|
||||
void google_speech_configure_grammar_hints(switch_core_session_t *session, switch_channel_t *channel, const char* hints, PhraseSet* phrase_set) {
|
||||
float boost = -1;
|
||||
|
||||
// get boost setting for the phrase set in its entirety
|
||||
if (switch_true(switch_channel_get_variable(channel, "GOOGLE_SPEECH_HINTS_BOOST"))) {
|
||||
boost = (float) atof(switch_channel_get_variable(channel, "GOOGLE_SPEECH_HINTS_BOOST"));
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "boost value: %f\n", boost);
|
||||
phrase_set->set_boost(boost);
|
||||
}
|
||||
|
||||
// hints are either a simple comma-separated list of phrases, or a json array of objects
|
||||
// containing a phrase and a boost value
|
||||
auto *jHint = cJSON_Parse((char *) hints);
|
||||
if (jHint) {
|
||||
int i = 0;
|
||||
cJSON *jPhrase = NULL;
|
||||
cJSON_ArrayForEach(jPhrase, jHint) {
|
||||
cJSON *jItem = cJSON_GetObjectItem(jPhrase, "phrase");
|
||||
if (jItem) {
|
||||
auto* phrase = phrase_set->add_phrases();
|
||||
phrase->set_value(cJSON_GetStringValue(jItem));
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "phrase: %s\n", phrase->value().c_str());
|
||||
if (cJSON_GetObjectItem(jPhrase, "boost")) {
|
||||
phrase->set_boost((float) cJSON_GetObjectItem(jPhrase, "boost")->valuedouble);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "boost value: %f\n", phrase->boost());
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
cJSON_Delete(jHint);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "added %d hints\n", i);
|
||||
}
|
||||
else {
|
||||
char *phrases[500] = { 0 };
|
||||
int argc = switch_separate_string((char *) hints, ',', phrases, 500);
|
||||
for (int i = 0; i < argc; i++) {
|
||||
auto* phrase = phrase_set->add_phrases();
|
||||
phrase->set_value(phrases[i]);
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "added %d hints\n", argc);
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,727 +1,27 @@
|
||||
#include <cstdlib>
|
||||
#include <algorithm>
|
||||
#include <future>
|
||||
|
||||
#include <switch.h>
|
||||
#include <switch_json.h>
|
||||
#include <grpc++/grpc++.h>
|
||||
|
||||
#include "google/cloud/speech/v1p1beta1/cloud_speech.grpc.pb.h"
|
||||
|
||||
#include <switch_json.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "mod_google_transcribe.h"
|
||||
#include "simple_buffer.h"
|
||||
|
||||
using google::cloud::speech::v1p1beta1::RecognitionConfig;
|
||||
using google::cloud::speech::v1p1beta1::Speech;
|
||||
using google::cloud::speech::v1p1beta1::SpeechContext;
|
||||
using google::cloud::speech::v1p1beta1::StreamingRecognizeRequest;
|
||||
using google::cloud::speech::v1p1beta1::StreamingRecognizeResponse;
|
||||
using google::cloud::speech::v1p1beta1::SpeakerDiarizationConfig;
|
||||
using google::cloud::speech::v1p1beta1::SpeechAdaptation;
|
||||
using google::cloud::speech::v1p1beta1::PhraseSet;
|
||||
using google::cloud::speech::v1p1beta1::PhraseSet_Phrase;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_DISCUSSION;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_PRESENTATION;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_PHONE_CALL;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_VOICEMAIL;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_PROFESSIONALLY_PRODUCED;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_VOICE_SEARCH;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_VOICE_COMMAND;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_DICTATION;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_MicrophoneDistance_NEARFIELD;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_MicrophoneDistance_MIDFIELD;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_MicrophoneDistance_FARFIELD;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_OriginalMediaType_AUDIO;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_OriginalMediaType_VIDEO;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_SMARTPHONE;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_PC;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_PHONE_LINE;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_VEHICLE;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_OTHER_OUTDOOR_DEVICE;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_OTHER_INDOOR_DEVICE;
|
||||
using google::cloud::speech::v1p1beta1::StreamingRecognizeResponse_SpeechEventType_END_OF_SINGLE_UTTERANCE;
|
||||
using google::rpc::Status;
|
||||
|
||||
#define CHUNKSIZE (320)
|
||||
|
||||
namespace {
|
||||
int case_insensitive_match(std::string s1, std::string s2) {
|
||||
std::transform(s1.begin(), s1.end(), s1.begin(), ::tolower);
|
||||
std::transform(s2.begin(), s2.end(), s2.begin(), ::tolower);
|
||||
if(s1.compare(s2) == 0)
|
||||
return 1; //The strings are same
|
||||
return 0; //not matched
|
||||
}
|
||||
}
|
||||
class GStreamer;
|
||||
|
||||
class GStreamer {
|
||||
public:
|
||||
GStreamer(
|
||||
switch_core_session_t *session,
|
||||
uint32_t channels,
|
||||
char* lang,
|
||||
int interim,
|
||||
uint32_t config_sample_rate,
|
||||
uint32_t samples_per_second,
|
||||
int single_utterance,
|
||||
int separate_recognition,
|
||||
int max_alternatives,
|
||||
int profanity_filter,
|
||||
int word_time_offset,
|
||||
int punctuation,
|
||||
const char* model,
|
||||
int enhanced,
|
||||
const char* hints) : m_session(session), m_writesDone(false), m_connected(false),
|
||||
m_audioBuffer(CHUNKSIZE, 15) {
|
||||
|
||||
const char* var;
|
||||
const char* google_uri;
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
|
||||
if (!(google_uri = switch_channel_get_variable(channel, "GOOGLE_SPEECH_TO_TEXT_URI"))) {
|
||||
google_uri = "speech.googleapis.com";
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_APPLICATION_CREDENTIALS")) {
|
||||
auto channelCreds = grpc::SslCredentials(grpc::SslCredentialsOptions());
|
||||
auto callCreds = grpc::ServiceAccountJWTAccessCredentials(var);
|
||||
auto creds = grpc::CompositeChannelCredentials(channelCreds, callCreds);
|
||||
m_channel = grpc::CreateChannel(google_uri, creds);
|
||||
}
|
||||
else {
|
||||
auto creds = grpc::GoogleDefaultCredentials();
|
||||
m_channel = grpc::CreateChannel(google_uri, creds);
|
||||
}
|
||||
|
||||
m_stub = Speech::NewStub(m_channel);
|
||||
|
||||
auto* streaming_config = m_request.mutable_streaming_config();
|
||||
RecognitionConfig* config = streaming_config->mutable_config();
|
||||
|
||||
streaming_config->set_interim_results(interim);
|
||||
if (single_utterance == 1) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_single_utterance\n");
|
||||
streaming_config->set_single_utterance(true);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_single_utterance is FALSE\n");
|
||||
streaming_config->set_single_utterance(false);
|
||||
}
|
||||
|
||||
config->set_language_code(lang);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "transcribe language %s \n", lang);
|
||||
|
||||
config->set_sample_rate_hertz(config_sample_rate);
|
||||
|
||||
config->set_encoding(RecognitionConfig::LINEAR16);
|
||||
|
||||
// the rest of config comes from channel vars
|
||||
|
||||
// number of channels in the audio stream (default: 1)
|
||||
if (channels > 1) {
|
||||
config->set_audio_channel_count(channels);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "audio_channel_count %d\n", channels);
|
||||
|
||||
// transcribe each separately?
|
||||
if (separate_recognition == 1) {
|
||||
config->set_enable_separate_recognition_per_channel(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_separate_recognition_per_channel on\n");
|
||||
}
|
||||
}
|
||||
|
||||
// max alternatives
|
||||
if (max_alternatives > 1) {
|
||||
config->set_max_alternatives(max_alternatives);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "max_alternatives %d\n", max_alternatives);
|
||||
}
|
||||
|
||||
// profanity filter
|
||||
if (profanity_filter == 1) {
|
||||
config->set_profanity_filter(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "profanity_filter\n");
|
||||
}
|
||||
|
||||
// enable word offsets
|
||||
if (word_time_offset == 1) {
|
||||
config->set_enable_word_time_offsets(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_word_time_offsets\n");
|
||||
}
|
||||
|
||||
// enable automatic punctuation
|
||||
if (punctuation == 1) {
|
||||
config->set_enable_automatic_punctuation(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_automatic_punctuation\n");
|
||||
}
|
||||
else {
|
||||
config->set_enable_automatic_punctuation(false);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "disable_automatic_punctuation\n");
|
||||
}
|
||||
|
||||
// speech model
|
||||
if (model != NULL) {
|
||||
config->set_model(model);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "speech model %s\n", model);
|
||||
}
|
||||
|
||||
// use enhanced model
|
||||
if (enhanced == 1) {
|
||||
config->set_use_enhanced(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "use_enhanced\n");
|
||||
}
|
||||
|
||||
// hints
|
||||
if (hints != NULL) {
|
||||
auto* adaptation = config->mutable_adaptation();
|
||||
auto* phrase_set = adaptation->add_phrase_sets();
|
||||
auto *context = config->add_speech_contexts();
|
||||
float boost = -1;
|
||||
|
||||
// get boost setting for the phrase set in its entirety
|
||||
if (switch_true(switch_channel_get_variable(channel, "GOOGLE_SPEECH_HINTS_BOOST"))) {
|
||||
boost = (float) atof(switch_channel_get_variable(channel, "GOOGLE_SPEECH_HINTS_BOOST"));
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "boost value: %f\n", boost);
|
||||
phrase_set->set_boost(boost);
|
||||
}
|
||||
|
||||
// hints are either a simple comma-separated list of phrases, or a json array of objects
|
||||
// containing a phrase and a boost value
|
||||
auto *jHint = cJSON_Parse((char *) hints);
|
||||
if (jHint) {
|
||||
int i = 0;
|
||||
cJSON *jPhrase = NULL;
|
||||
cJSON_ArrayForEach(jPhrase, jHint) {
|
||||
auto* phrase = phrase_set->add_phrases();
|
||||
cJSON *jItem = cJSON_GetObjectItem(jPhrase, "phrase");
|
||||
if (jItem) {
|
||||
phrase->set_value(cJSON_GetStringValue(jItem));
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "phrase: %s\n", phrase->value().c_str());
|
||||
if (cJSON_GetObjectItem(jPhrase, "boost")) {
|
||||
phrase->set_boost((float) cJSON_GetObjectItem(jPhrase, "boost")->valuedouble);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "boost value: %f\n", phrase->boost());
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
cJSON_Delete(jHint);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "added %d hints\n", i);
|
||||
}
|
||||
else {
|
||||
char *phrases[500] = { 0 };
|
||||
int argc = switch_separate_string((char *) hints, ',', phrases, 500);
|
||||
for (int i = 0; i < argc; i++) {
|
||||
auto* phrase = phrase_set->add_phrases();
|
||||
phrase->set_value(phrases[i]);
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "added %d hints\n", argc);
|
||||
}
|
||||
}
|
||||
|
||||
// alternative language
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES")) {
|
||||
char *alt_langs[3] = { 0 };
|
||||
int argc = switch_separate_string((char *) var, ',', alt_langs, 3);
|
||||
for (int i = 0; i < argc; i++) {
|
||||
config->add_alternative_language_codes(alt_langs[i]);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "added alternative lang %s\n", alt_langs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// speaker diarization
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_SPEAKER_DIARIZATION")) {
|
||||
auto* diarization_config = config->mutable_diarization_config();
|
||||
diarization_config->set_enable_speaker_diarization(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enabling speaker diarization\n", var);
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_SPEAKER_DIARIZATION_MIN_SPEAKER_COUNT")) {
|
||||
int count = std::max(atoi(var), 1);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "setting min speaker count to %d\n", count);
|
||||
diarization_config->set_min_speaker_count(count);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_SPEAKER_DIARIZATION_MAX_SPEAKER_COUNT")) {
|
||||
int count = std::max(atoi(var), 2);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "setting max speaker count to %d\n", count);
|
||||
diarization_config->set_max_speaker_count(count);
|
||||
}
|
||||
}
|
||||
|
||||
// recognition metadata
|
||||
auto* metadata = config->mutable_metadata();
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_METADATA_INTERACTION_TYPE")) {
|
||||
if (case_insensitive_match("discussion", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_DISCUSSION);
|
||||
if (case_insensitive_match("presentation", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_PRESENTATION);
|
||||
if (case_insensitive_match("phone_call", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_PHONE_CALL);
|
||||
if (case_insensitive_match("voicemail", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_VOICEMAIL);
|
||||
if (case_insensitive_match("professionally_produced", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_PROFESSIONALLY_PRODUCED);
|
||||
if (case_insensitive_match("voice_search", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_VOICE_SEARCH);
|
||||
if (case_insensitive_match("voice_command", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_VOICE_COMMAND);
|
||||
if (case_insensitive_match("dictation", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_DICTATION);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_METADATA_INDUSTRY_NAICS_CODE")) {
|
||||
metadata->set_industry_naics_code_of_audio(atoi(var));
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_METADATA_MICROPHONE_DISTANCE")) {
|
||||
if (case_insensitive_match("nearfield", var)) metadata->set_microphone_distance(RecognitionMetadata_MicrophoneDistance_NEARFIELD);
|
||||
if (case_insensitive_match("midfield", var)) metadata->set_microphone_distance(RecognitionMetadata_MicrophoneDistance_MIDFIELD);
|
||||
if (case_insensitive_match("farfield", var)) metadata->set_microphone_distance(RecognitionMetadata_MicrophoneDistance_FARFIELD);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_METADATA_ORIGINAL_MEDIA_TYPE")) {
|
||||
if (case_insensitive_match("audio", var)) metadata->set_original_media_type(RecognitionMetadata_OriginalMediaType_AUDIO);
|
||||
if (case_insensitive_match("video", var)) metadata->set_original_media_type(RecognitionMetadata_OriginalMediaType_VIDEO);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_METADATA_RECORDING_DEVICE_TYPE")) {
|
||||
if (case_insensitive_match("smartphone", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_SMARTPHONE);
|
||||
if (case_insensitive_match("pc", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_PC);
|
||||
if (case_insensitive_match("phone_line", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_PHONE_LINE);
|
||||
if (case_insensitive_match("vehicle", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_VEHICLE);
|
||||
if (case_insensitive_match("other_outdoor_device", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_OTHER_OUTDOOR_DEVICE);
|
||||
if (case_insensitive_match("other_indoor_device", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_OTHER_INDOOR_DEVICE);
|
||||
}
|
||||
}
|
||||
|
||||
~GStreamer() {
|
||||
//switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_INFO, "GStreamer::~GStreamer - deleting channel and stub: %p\n", (void*)this);
|
||||
}
|
||||
|
||||
void connect() {
|
||||
assert(!m_connected);
|
||||
// Begin a stream.
|
||||
m_streamer = m_stub->StreamingRecognize(&m_context);
|
||||
m_connected = true;
|
||||
|
||||
// read thread is waiting on this
|
||||
m_promise.set_value();
|
||||
|
||||
// Write the first request, containing the config only.
|
||||
m_streamer->Write(m_request);
|
||||
|
||||
// send any buffered audio
|
||||
int nFrames = m_audioBuffer.getNumItems();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p got stream ready, %d buffered frames\n", this, nFrames);
|
||||
if (nFrames) {
|
||||
char *p;
|
||||
do {
|
||||
p = m_audioBuffer.getNextChunk();
|
||||
if (p) {
|
||||
write(p, CHUNKSIZE);
|
||||
}
|
||||
} while (p);
|
||||
}
|
||||
}
|
||||
|
||||
bool write(void* data, uint32_t datalen) {
|
||||
if (!m_connected) {
|
||||
if (datalen % CHUNKSIZE == 0) {
|
||||
m_audioBuffer.add(data, datalen);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
m_request.set_audio_content(data, datalen);
|
||||
bool ok = m_streamer->Write(m_request);
|
||||
return ok;
|
||||
}
|
||||
|
||||
uint32_t nextMessageSize(void) {
|
||||
uint32_t size = 0;
|
||||
m_streamer->NextMessageSize(&size);
|
||||
return size;
|
||||
}
|
||||
|
||||
bool read(StreamingRecognizeResponse* response) {
|
||||
return m_streamer->Read(response);
|
||||
}
|
||||
|
||||
grpc::Status finish() {
|
||||
return m_streamer->Finish();
|
||||
}
|
||||
|
||||
void writesDone() {
|
||||
// grpc crashes if we call this twice on a stream
|
||||
if (!m_connected) {
|
||||
cancelConnect();
|
||||
}
|
||||
else if (!m_writesDone) {
|
||||
m_streamer->WritesDone();
|
||||
m_writesDone = true;
|
||||
}
|
||||
}
|
||||
|
||||
bool waitForConnect() {
|
||||
std::shared_future<void> sf(m_promise.get_future());
|
||||
sf.wait();
|
||||
return m_connected;
|
||||
}
|
||||
|
||||
void cancelConnect() {
|
||||
assert(!m_connected);
|
||||
m_promise.set_value();
|
||||
}
|
||||
|
||||
bool isConnected() {
|
||||
return m_connected;
|
||||
}
|
||||
|
||||
private:
|
||||
switch_core_session_t* m_session;
|
||||
grpc::ClientContext m_context;
|
||||
std::shared_ptr<grpc::Channel> m_channel;
|
||||
std::unique_ptr<Speech::Stub> m_stub;
|
||||
std::unique_ptr< grpc::ClientReaderWriterInterface<StreamingRecognizeRequest, StreamingRecognizeResponse> > m_streamer;
|
||||
StreamingRecognizeRequest m_request;
|
||||
bool m_writesDone;
|
||||
bool m_connected;
|
||||
std::promise<void> m_promise;
|
||||
SimpleBuffer m_audioBuffer;
|
||||
};
|
||||
|
||||
static void *SWITCH_THREAD_FUNC grpc_read_thread(switch_thread_t *thread, void *obj) {
|
||||
static int count;
|
||||
struct cap_cb *cb = (struct cap_cb *) obj;
|
||||
GStreamer* streamer = (GStreamer *) cb->streamer;
|
||||
|
||||
bool connected = streamer->waitForConnect();
|
||||
if (!connected) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "google transcribe grpc read thread exiting since we didnt connect\n") ;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Read responses.
|
||||
StreamingRecognizeResponse response;
|
||||
while (streamer->read(&response)) { // Returns false when no more to read.
|
||||
switch_core_session_t* session = switch_core_session_locate(cb->sessionId);
|
||||
if (!session) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "grpc_read_thread: session %s is gone!\n", cb->sessionId) ;
|
||||
return nullptr;
|
||||
}
|
||||
count++;
|
||||
auto speech_event_type = response.speech_event_type();
|
||||
if (response.has_error()) {
|
||||
Status status = response.error();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "grpc_read_thread: error %s (%d)\n", status.message().c_str(), status.code()) ;
|
||||
cJSON* json = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(json, "type", "error");
|
||||
cJSON_AddStringToObject(json, "error", status.message().c_str());
|
||||
char* jsonString = cJSON_PrintUnformatted(json);
|
||||
cb->responseHandler(session, jsonString, cb->bugname);
|
||||
free(jsonString);
|
||||
cJSON_Delete(json);
|
||||
}
|
||||
|
||||
if (cb->play_file == 1){
|
||||
cb->responseHandler(session, "play_interrupt", cb->bugname);
|
||||
}
|
||||
|
||||
for (int r = 0; r < response.results_size(); ++r) {
|
||||
auto result = response.results(r);
|
||||
cJSON * jResult = cJSON_CreateObject();
|
||||
cJSON * jAlternatives = cJSON_CreateArray();
|
||||
cJSON * jStability = cJSON_CreateNumber(result.stability());
|
||||
cJSON * jIsFinal = cJSON_CreateBool(result.is_final());
|
||||
cJSON * jLanguageCode = cJSON_CreateString(result.language_code().c_str());
|
||||
cJSON * jChannelTag = cJSON_CreateNumber(result.channel_tag());
|
||||
|
||||
auto duration = result.result_end_time();
|
||||
int32_t seconds = duration.seconds();
|
||||
int64_t nanos = duration.nanos();
|
||||
int span = (int) trunc(seconds * 1000. + ((float) nanos / 1000000.));
|
||||
cJSON * jResultEndTime = cJSON_CreateNumber(span);
|
||||
|
||||
cJSON_AddItemToObject(jResult, "stability", jStability);
|
||||
cJSON_AddItemToObject(jResult, "is_final", jIsFinal);
|
||||
cJSON_AddItemToObject(jResult, "alternatives", jAlternatives);
|
||||
cJSON_AddItemToObject(jResult, "language_code", jLanguageCode);
|
||||
cJSON_AddItemToObject(jResult, "channel_tag", jChannelTag);
|
||||
cJSON_AddItemToObject(jResult, "result_end_time", jResultEndTime);
|
||||
|
||||
for (int a = 0; a < result.alternatives_size(); ++a) {
|
||||
auto alternative = result.alternatives(a);
|
||||
cJSON* jAlt = cJSON_CreateObject();
|
||||
cJSON* jConfidence = cJSON_CreateNumber(alternative.confidence());
|
||||
cJSON* jTranscript = cJSON_CreateString(alternative.transcript().c_str());
|
||||
cJSON_AddItemToObject(jAlt, "confidence", jConfidence);
|
||||
cJSON_AddItemToObject(jAlt, "transcript", jTranscript);
|
||||
|
||||
if (alternative.words_size() > 0) {
|
||||
cJSON * jWords = cJSON_CreateArray();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: %d words\n", alternative.words_size()) ;
|
||||
for (int b = 0; b < alternative.words_size(); b++) {
|
||||
auto words = alternative.words(b);
|
||||
cJSON* jWord = cJSON_CreateObject();
|
||||
cJSON_AddItemToObject(jWord, "word", cJSON_CreateString(words.word().c_str()));
|
||||
if (words.has_start_time()) {
|
||||
cJSON_AddItemToObject(jWord, "start_time", cJSON_CreateNumber(words.start_time().seconds()));
|
||||
}
|
||||
if (words.has_end_time()) {
|
||||
cJSON_AddItemToObject(jWord, "end_time", cJSON_CreateNumber(words.end_time().seconds()));
|
||||
}
|
||||
int speaker_tag = words.speaker_tag();
|
||||
if (speaker_tag > 0) {
|
||||
cJSON_AddItemToObject(jWord, "speaker_tag", cJSON_CreateNumber(speaker_tag));
|
||||
}
|
||||
float confidence = words.confidence();
|
||||
if (confidence > 0.0) {
|
||||
cJSON_AddItemToObject(jWord, "confidence", cJSON_CreateNumber(confidence));
|
||||
}
|
||||
|
||||
cJSON_AddItemToArray(jWords, jWord);
|
||||
}
|
||||
cJSON_AddItemToObject(jAlt, "words", jWords);
|
||||
}
|
||||
cJSON_AddItemToArray(jAlternatives, jAlt);
|
||||
}
|
||||
|
||||
char* json = cJSON_PrintUnformatted(jResult);
|
||||
cb->responseHandler(session, (const char *) json, cb->bugname);
|
||||
free(json);
|
||||
|
||||
cJSON_Delete(jResult);
|
||||
}
|
||||
|
||||
if (speech_event_type == StreamingRecognizeResponse_SpeechEventType_END_OF_SINGLE_UTTERANCE) {
|
||||
// we only get this when we have requested it, and recognition stops after we get this
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: got end_of_utterance\n") ;
|
||||
cb->got_end_of_utterance = 1;
|
||||
cb->responseHandler(session, "end_of_utterance", cb->bugname);
|
||||
if (cb->wants_single_utterance) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: sending writesDone because we want only a single utterance\n") ;
|
||||
streamer->writesDone();
|
||||
}
|
||||
}
|
||||
switch_core_session_rwunlock(session);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: got %d responses\n", response.results_size());
|
||||
}
|
||||
|
||||
{
|
||||
switch_core_session_t* session = switch_core_session_locate(cb->sessionId);
|
||||
if (session) {
|
||||
grpc::Status status = streamer->finish();
|
||||
if (11 == status.error_code()) {
|
||||
if (std::string::npos != status.error_message().find("Exceeded maximum allowed stream duration")) {
|
||||
cb->responseHandler(session, "max_duration_exceeded", cb->bugname);
|
||||
}
|
||||
else {
|
||||
cb->responseHandler(session, "no_audio", cb->bugname);
|
||||
}
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: finish() status %s (%d)\n", status.error_message().c_str(), status.error_code()) ;
|
||||
switch_core_session_rwunlock(session);
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
#include "google_glue.h"
|
||||
#include "generic_google_glue.h"
|
||||
|
||||
extern "C" {
|
||||
switch_status_t google_speech_init() {
|
||||
const char* gcsServiceKeyFile = std::getenv("GOOGLE_APPLICATION_CREDENTIALS");
|
||||
if (gcsServiceKeyFile) {
|
||||
try {
|
||||
auto creds = grpc::GoogleDefaultCredentials();
|
||||
} catch (const std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT,
|
||||
"Error initializing google api with provided credentials in %s: %s\n", gcsServiceKeyFile, e.what());
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
}
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t google_speech_init() {
|
||||
const char* gcsServiceKeyFile = std::getenv("GOOGLE_APPLICATION_CREDENTIALS");
|
||||
if (gcsServiceKeyFile) {
|
||||
try {
|
||||
auto creds = grpc::GoogleDefaultCredentials();
|
||||
} catch (const std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT,
|
||||
"Error initializing google api with provided credentials in %s: %s\n", gcsServiceKeyFile, e.what());
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
}
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t google_speech_cleanup() {
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
switch_status_t google_speech_session_init(switch_core_session_t *session, responseHandler_t responseHandler,
|
||||
uint32_t to_rate, uint32_t samples_per_second, uint32_t channels, char* lang, int interim, char *bugname,
|
||||
int single_utterance, int separate_recognition, int max_alternatives, int profanity_filter, int word_time_offset,
|
||||
int punctuation, const char* model, int enhanced, const char* hints, char* play_file, void **ppUserData) {
|
||||
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
auto read_codec = switch_core_session_get_read_codec(session);
|
||||
uint32_t sampleRate = read_codec->implementation->actual_samples_per_second;
|
||||
struct cap_cb *cb;
|
||||
int err;
|
||||
|
||||
cb =(struct cap_cb *) switch_core_session_alloc(session, sizeof(*cb));
|
||||
strncpy(cb->sessionId, switch_core_session_get_uuid(session), MAX_SESSION_ID);
|
||||
strncpy(cb->bugname, bugname, MAX_BUG_LEN);
|
||||
cb->got_end_of_utterance = 0;
|
||||
cb->wants_single_utterance = single_utterance;
|
||||
if (play_file != NULL){
|
||||
cb->play_file = 1;
|
||||
}
|
||||
|
||||
switch_mutex_init(&cb->mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session));
|
||||
if (sampleRate != to_rate) {
|
||||
cb->resampler = speex_resampler_init(channels, sampleRate, to_rate, SWITCH_RESAMPLE_QUALITY, &err);
|
||||
if (0 != err) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing resampler: %s.\n",
|
||||
switch_channel_get_name(channel), speex_resampler_strerror(err));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
} else {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "%s: no resampling needed for this call\n", switch_channel_get_name(channel));
|
||||
}
|
||||
cb->responseHandler = responseHandler;
|
||||
|
||||
// allocate vad if we are delaying connecting to the recognizer until we detect speech
|
||||
if (switch_channel_var_true(channel, "START_RECOGNIZING_ON_VAD")) {
|
||||
cb->vad = switch_vad_init(sampleRate, channels);
|
||||
if (cb->vad) {
|
||||
const char* var;
|
||||
int mode = 2;
|
||||
int silence_ms = 150;
|
||||
int voice_ms = 250;
|
||||
int debug = 0;
|
||||
|
||||
if (var = switch_channel_get_variable(channel, "RECOGNIZER_VAD_MODE")) {
|
||||
mode = atoi(var);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "RECOGNIZER_VAD_SILENCE_MS")) {
|
||||
silence_ms = atoi(var);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "RECOGNIZER_VAD_VOICE_MS")) {
|
||||
voice_ms = atoi(var);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "RECOGNIZER_VAD_VOICE_MS")) {
|
||||
voice_ms = atoi(var);
|
||||
}
|
||||
switch_vad_set_mode(cb->vad, mode);
|
||||
switch_vad_set_param(cb->vad, "silence_ms", silence_ms);
|
||||
switch_vad_set_param(cb->vad, "voice_ms", voice_ms);
|
||||
switch_vad_set_param(cb->vad, "debug", debug);
|
||||
}
|
||||
}
|
||||
|
||||
GStreamer *streamer = NULL;
|
||||
try {
|
||||
streamer = new GStreamer(session, channels, lang, interim, to_rate, sampleRate, single_utterance, separate_recognition, max_alternatives,
|
||||
profanity_filter, word_time_offset, punctuation, model, enhanced, hints);
|
||||
cb->streamer = streamer;
|
||||
} catch (std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing gstreamer: %s.\n",
|
||||
switch_channel_get_name(channel), e.what());
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (!cb->vad) streamer->connect();
|
||||
|
||||
// create the read thread
|
||||
switch_threadattr_t *thd_attr = NULL;
|
||||
switch_memory_pool_t *pool = switch_core_session_get_pool(session);
|
||||
|
||||
switch_threadattr_create(&thd_attr, pool);
|
||||
switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE);
|
||||
switch_thread_create(&cb->thread, thd_attr, grpc_read_thread, cb, pool);
|
||||
|
||||
*ppUserData = cb;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t google_speech_session_cleanup(switch_core_session_t *session, int channelIsClosing, switch_media_bug_t *bug) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
|
||||
if (bug) {
|
||||
struct cap_cb *cb = (struct cap_cb *) switch_core_media_bug_get_user_data(bug);
|
||||
switch_mutex_lock(cb->mutex);
|
||||
|
||||
if (!switch_channel_get_private(channel, cb->bugname)) {
|
||||
// race condition
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s Bug is not attached (race).\n", switch_channel_get_name(channel));
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
switch_channel_set_private(channel, cb->bugname, NULL);
|
||||
|
||||
// stop playback if available
|
||||
if (cb->play_file == 1){
|
||||
if (switch_channel_test_flag(channel, CF_BROADCAST)) {
|
||||
switch_channel_stop_broadcast(channel);
|
||||
} else {
|
||||
switch_channel_set_flag_value(channel, CF_BREAK, 1);
|
||||
}
|
||||
}
|
||||
|
||||
// close connection and get final responses
|
||||
GStreamer* streamer = (GStreamer *) cb->streamer;
|
||||
|
||||
if (streamer) {
|
||||
streamer->writesDone();
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "google_speech_session_cleanup: GStreamer (%p) waiting for read thread to complete\n", (void*)streamer);
|
||||
switch_status_t st;
|
||||
switch_thread_join(&st, cb->thread);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "google_speech_session_cleanup: GStreamer (%p) read thread completed\n", (void*)streamer);
|
||||
|
||||
delete streamer;
|
||||
cb->streamer = NULL;
|
||||
}
|
||||
|
||||
if (cb->resampler) {
|
||||
speex_resampler_destroy(cb->resampler);
|
||||
}
|
||||
if (cb->vad) {
|
||||
switch_vad_destroy(&cb->vad);
|
||||
cb->vad = nullptr;
|
||||
}
|
||||
if (!channelIsClosing) {
|
||||
switch_core_media_bug_remove(session, &bug);
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "google_speech_session_cleanup: Closed stream\n");
|
||||
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s Bug is not attached.\n", switch_channel_get_name(channel));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
switch_bool_t google_speech_frame(switch_media_bug_t *bug, void* user_data) {
|
||||
switch_core_session_t *session = switch_core_media_bug_get_session(bug);
|
||||
struct cap_cb *cb = (struct cap_cb *) user_data;
|
||||
if (cb->streamer && (!cb->wants_single_utterance || !cb->got_end_of_utterance)) {
|
||||
GStreamer* streamer = (GStreamer *) cb->streamer;
|
||||
uint8_t data[SWITCH_RECOMMENDED_BUFFER_SIZE];
|
||||
switch_frame_t frame = {};
|
||||
frame.data = data;
|
||||
frame.buflen = SWITCH_RECOMMENDED_BUFFER_SIZE;
|
||||
|
||||
if (switch_mutex_trylock(cb->mutex) == SWITCH_STATUS_SUCCESS) {
|
||||
while (switch_core_media_bug_read(bug, &frame, SWITCH_TRUE) == SWITCH_STATUS_SUCCESS && !switch_test_flag((&frame), SFF_CNG)) {
|
||||
if (frame.datalen) {
|
||||
if (cb->vad && !streamer->isConnected()) {
|
||||
switch_vad_state_t state = switch_vad_process(cb->vad, (int16_t*) frame.data, frame.samples);
|
||||
if (state == SWITCH_VAD_STATE_START_TALKING) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "detected speech, connect to google speech now\n");
|
||||
streamer->connect();
|
||||
cb->responseHandler(session, "vad_detected", cb->bugname);
|
||||
}
|
||||
}
|
||||
|
||||
if (cb->resampler) {
|
||||
spx_int16_t out[SWITCH_RECOMMENDED_BUFFER_SIZE];
|
||||
spx_uint32_t out_len = SWITCH_RECOMMENDED_BUFFER_SIZE;
|
||||
spx_uint32_t in_len = frame.samples;
|
||||
size_t written;
|
||||
|
||||
speex_resampler_process_interleaved_int(cb->resampler,
|
||||
(const spx_int16_t *) frame.data,
|
||||
(spx_uint32_t *) &in_len,
|
||||
&out[0],
|
||||
&out_len);
|
||||
streamer->write( &out[0], sizeof(spx_int16_t) * out_len);
|
||||
}
|
||||
else {
|
||||
streamer->write( frame.data, sizeof(spx_int16_t) * frame.samples);
|
||||
}
|
||||
}
|
||||
}
|
||||
switch_mutex_unlock(cb->mutex);
|
||||
}
|
||||
}
|
||||
return SWITCH_TRUE;
|
||||
}
|
||||
switch_status_t google_speech_cleanup() {
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,28 @@
|
||||
#ifndef __GOOGLE_GLUE_H__
|
||||
#define __GOOGLE_GLUE_H__
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
switch_status_t google_speech_init();
|
||||
switch_status_t google_speech_cleanup();
|
||||
switch_status_t google_speech_session_init(switch_core_session_t *session, responseHandler_t responseHandler,
|
||||
switch_status_t google_speech_session_init_v1(switch_core_session_t *session, responseHandler_t responseHandler,
|
||||
uint32_t to_rate, uint32_t samples_per_second, uint32_t channels, char* lang, int interim, char *bugname, int single_utterence,
|
||||
int separate_recognition, int max_alternatives, int profinity_filter, int word_time_offset, int punctuation, const char* model, int enhanced,
|
||||
int separate_recognition, int max_alternatives, int profanity_filter, int word_time_offset, int punctuation, const char* model, int enhanced,
|
||||
const char* hints, char* play_file, void **ppUserData);
|
||||
switch_status_t google_speech_session_cleanup(switch_core_session_t *session, int channelIsClosing, switch_media_bug_t *bug);
|
||||
switch_bool_t google_speech_frame(switch_media_bug_t *bug, void* user_data);
|
||||
switch_status_t google_speech_session_cleanup_v1(switch_core_session_t *session, int channelIsClosing, switch_media_bug_t *bug);
|
||||
switch_status_t google_speech_session_init_v2(switch_core_session_t *session, responseHandler_t responseHandler,
|
||||
uint32_t to_rate, uint32_t samples_per_second, uint32_t channels, char* lang, int interim, char *bugname, int single_utterence,
|
||||
int separate_recognition, int max_alternatives, int profanity_filter, int word_time_offset, int punctuation, const char* model, int enhanced,
|
||||
const char* hints, char* play_file, void **ppUserData);
|
||||
switch_status_t google_speech_session_cleanup_v2(switch_core_session_t *session, int channelIsClosing, switch_media_bug_t *bug);
|
||||
|
||||
switch_bool_t google_speech_frame_v1(switch_media_bug_t *bug, void* user_data);
|
||||
switch_bool_t google_speech_frame_v2(switch_media_bug_t *bug, void* user_data);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,393 @@
|
||||
#include <switch.h>
|
||||
#include <switch_json.h>
|
||||
#include <grpc++/grpc++.h>
|
||||
|
||||
#include "mod_google_transcribe.h"
|
||||
|
||||
#include "gstreamer.h"
|
||||
#include "generic_google_glue.h"
|
||||
|
||||
#include "google/cloud/speech/v1p1beta1/cloud_speech.grpc.pb.h"
|
||||
|
||||
using google::cloud::speech::v1p1beta1::RecognitionConfig;
|
||||
using google::cloud::speech::v1p1beta1::Speech;
|
||||
using google::cloud::speech::v1p1beta1::SpeechContext;
|
||||
using google::cloud::speech::v1p1beta1::StreamingRecognizeRequest;
|
||||
using google::cloud::speech::v1p1beta1::StreamingRecognizeResponse;
|
||||
using google::cloud::speech::v1p1beta1::SpeakerDiarizationConfig;
|
||||
using google::cloud::speech::v1p1beta1::SpeechAdaptation;
|
||||
using google::cloud::speech::v1p1beta1::PhraseSet;
|
||||
using google::cloud::speech::v1p1beta1::PhraseSet_Phrase;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_DISCUSSION;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_PRESENTATION;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_PHONE_CALL;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_VOICEMAIL;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_PROFESSIONALLY_PRODUCED;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_VOICE_SEARCH;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_VOICE_COMMAND;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_InteractionType_DICTATION;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_MicrophoneDistance_NEARFIELD;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_MicrophoneDistance_MIDFIELD;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_MicrophoneDistance_FARFIELD;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_OriginalMediaType_AUDIO;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_OriginalMediaType_VIDEO;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_SMARTPHONE;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_PC;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_PHONE_LINE;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_VEHICLE;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_OTHER_OUTDOOR_DEVICE;
|
||||
using google::cloud::speech::v1p1beta1::RecognitionMetadata_RecordingDeviceType_OTHER_INDOOR_DEVICE;
|
||||
using google::cloud::speech::v1p1beta1::StreamingRecognizeResponse_SpeechEventType_END_OF_SINGLE_UTTERANCE;
|
||||
using google::rpc::Status;
|
||||
|
||||
typedef GStreamer<StreamingRecognizeRequest, StreamingRecognizeResponse, Speech::Stub> GStreamer_V1;
|
||||
|
||||
template<>
|
||||
GStreamer<StreamingRecognizeRequest, StreamingRecognizeResponse, Speech::Stub>::GStreamer(
|
||||
switch_core_session_t *session,
|
||||
uint32_t channels,
|
||||
char* lang,
|
||||
int interim,
|
||||
uint32_t config_sample_rate,
|
||||
uint32_t samples_per_second,
|
||||
int single_utterance,
|
||||
int separate_recognition,
|
||||
int max_alternatives,
|
||||
int profanity_filter,
|
||||
int word_time_offset,
|
||||
int punctuation,
|
||||
const char* model,
|
||||
int enhanced,
|
||||
const char* hints) : m_session(session), m_writesDone(false), m_connected(false),
|
||||
m_audioBuffer(CHUNKSIZE, 15) {
|
||||
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
m_channel = create_grpc_channel(channel);
|
||||
m_stub = Speech::NewStub(m_channel);
|
||||
|
||||
auto* streaming_config = m_request.mutable_streaming_config();
|
||||
RecognitionConfig* config = streaming_config->mutable_config();
|
||||
|
||||
streaming_config->set_interim_results(interim);
|
||||
if (single_utterance == 1) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_single_utterance\n");
|
||||
streaming_config->set_single_utterance(true);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_single_utterance is FALSE\n");
|
||||
streaming_config->set_single_utterance(false);
|
||||
}
|
||||
|
||||
config->set_language_code(lang);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "transcribe language %s \n", lang);
|
||||
|
||||
config->set_sample_rate_hertz(config_sample_rate);
|
||||
|
||||
config->set_encoding(RecognitionConfig::LINEAR16);
|
||||
|
||||
// the rest of config comes from channel vars
|
||||
|
||||
// number of channels in the audio stream (default: 1)
|
||||
if (channels > 1) {
|
||||
config->set_audio_channel_count(channels);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "audio_channel_count %d\n", channels);
|
||||
|
||||
// transcribe each separately?
|
||||
if (separate_recognition == 1) {
|
||||
config->set_enable_separate_recognition_per_channel(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_separate_recognition_per_channel on\n");
|
||||
}
|
||||
}
|
||||
|
||||
// max alternatives
|
||||
if (max_alternatives > 1) {
|
||||
config->set_max_alternatives(max_alternatives);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "max_alternatives %d\n", max_alternatives);
|
||||
}
|
||||
|
||||
// profanity filter
|
||||
if (profanity_filter == 1) {
|
||||
config->set_profanity_filter(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "profanity_filter\n");
|
||||
}
|
||||
|
||||
// enable word offsets
|
||||
if (word_time_offset == 1) {
|
||||
config->set_enable_word_time_offsets(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_word_time_offsets\n");
|
||||
}
|
||||
|
||||
// enable automatic punctuation
|
||||
if (punctuation == 1) {
|
||||
config->set_enable_automatic_punctuation(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_automatic_punctuation\n");
|
||||
}
|
||||
else {
|
||||
config->set_enable_automatic_punctuation(false);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "disable_automatic_punctuation\n");
|
||||
}
|
||||
|
||||
// speech model
|
||||
if (model != NULL) {
|
||||
config->set_model(model);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "speech model %s\n", model);
|
||||
}
|
||||
|
||||
// use enhanced model
|
||||
if (enhanced == 1) {
|
||||
config->set_use_enhanced(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "use_enhanced\n");
|
||||
}
|
||||
|
||||
// hints
|
||||
if (hints != NULL) {
|
||||
auto* adaptation = config->mutable_adaptation();
|
||||
auto* phrase_set = adaptation->add_phrase_sets();
|
||||
google_speech_configure_grammar_hints(m_session, channel, hints, phrase_set);
|
||||
}
|
||||
|
||||
const char* var;
|
||||
|
||||
// alternative language
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES")) {
|
||||
char *alt_langs[3] = { 0 };
|
||||
int argc = switch_separate_string((char *) var, ',', alt_langs, 3);
|
||||
for (int i = 0; i < argc; i++) {
|
||||
config->add_alternative_language_codes(alt_langs[i]);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "added alternative lang %s\n", alt_langs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// speaker diarization
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_SPEAKER_DIARIZATION")) {
|
||||
auto* diarization_config = config->mutable_diarization_config();
|
||||
diarization_config->set_enable_speaker_diarization(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enabling speaker diarization\n", var);
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_SPEAKER_DIARIZATION_MIN_SPEAKER_COUNT")) {
|
||||
int count = std::max(atoi(var), 1);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "setting min speaker count to %d\n", count);
|
||||
diarization_config->set_min_speaker_count(count);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_SPEAKER_DIARIZATION_MAX_SPEAKER_COUNT")) {
|
||||
int count = std::max(atoi(var), 2);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "setting max speaker count to %d\n", count);
|
||||
diarization_config->set_max_speaker_count(count);
|
||||
}
|
||||
}
|
||||
|
||||
// recognition metadata
|
||||
auto* metadata = config->mutable_metadata();
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_METADATA_INTERACTION_TYPE")) {
|
||||
if (case_insensitive_match("discussion", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_DISCUSSION);
|
||||
if (case_insensitive_match("presentation", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_PRESENTATION);
|
||||
if (case_insensitive_match("phone_call", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_PHONE_CALL);
|
||||
if (case_insensitive_match("voicemail", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_VOICEMAIL);
|
||||
if (case_insensitive_match("professionally_produced", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_PROFESSIONALLY_PRODUCED);
|
||||
if (case_insensitive_match("voice_search", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_VOICE_SEARCH);
|
||||
if (case_insensitive_match("voice_command", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_VOICE_COMMAND);
|
||||
if (case_insensitive_match("dictation", var)) metadata->set_interaction_type(RecognitionMetadata_InteractionType_DICTATION);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_METADATA_INDUSTRY_NAICS_CODE")) {
|
||||
metadata->set_industry_naics_code_of_audio(atoi(var));
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_METADATA_MICROPHONE_DISTANCE")) {
|
||||
if (case_insensitive_match("nearfield", var)) metadata->set_microphone_distance(RecognitionMetadata_MicrophoneDistance_NEARFIELD);
|
||||
if (case_insensitive_match("midfield", var)) metadata->set_microphone_distance(RecognitionMetadata_MicrophoneDistance_MIDFIELD);
|
||||
if (case_insensitive_match("farfield", var)) metadata->set_microphone_distance(RecognitionMetadata_MicrophoneDistance_FARFIELD);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_METADATA_ORIGINAL_MEDIA_TYPE")) {
|
||||
if (case_insensitive_match("audio", var)) metadata->set_original_media_type(RecognitionMetadata_OriginalMediaType_AUDIO);
|
||||
if (case_insensitive_match("video", var)) metadata->set_original_media_type(RecognitionMetadata_OriginalMediaType_VIDEO);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_METADATA_RECORDING_DEVICE_TYPE")) {
|
||||
if (case_insensitive_match("smartphone", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_SMARTPHONE);
|
||||
if (case_insensitive_match("pc", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_PC);
|
||||
if (case_insensitive_match("phone_line", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_PHONE_LINE);
|
||||
if (case_insensitive_match("vehicle", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_VEHICLE);
|
||||
if (case_insensitive_match("other_outdoor_device", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_OTHER_OUTDOOR_DEVICE);
|
||||
if (case_insensitive_match("other_indoor_device", var)) metadata->set_recording_device_type(RecognitionMetadata_RecordingDeviceType_OTHER_INDOOR_DEVICE);
|
||||
}
|
||||
}
|
||||
|
||||
static void *SWITCH_THREAD_FUNC grpc_read_thread(switch_thread_t *thread, void *obj) {
|
||||
static int count;
|
||||
struct cap_cb *cb = (struct cap_cb *) obj;
|
||||
GStreamer_V1* streamer = (GStreamer_V1 *) cb->streamer;
|
||||
|
||||
bool connected = streamer->waitForConnect();
|
||||
if (!connected) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "google transcribe grpc read thread exiting since we didn't connect\n") ;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Read responses.
|
||||
StreamingRecognizeResponse response;
|
||||
while (streamer->read(&response)) { // Returns false when no more to read.
|
||||
switch_core_session_t* session = switch_core_session_locate(cb->sessionId);
|
||||
if (!session) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "grpc_read_thread: session %s is gone!\n", cb->sessionId) ;
|
||||
return nullptr;
|
||||
}
|
||||
count++;
|
||||
auto speech_event_type = response.speech_event_type();
|
||||
if (response.has_error()) {
|
||||
Status status = response.error();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "grpc_read_thread: error %s (%d)\n", status.message().c_str(), status.code()) ;
|
||||
cJSON* json = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(json, "type", "error");
|
||||
cJSON_AddStringToObject(json, "error_cause", "stream_response");
|
||||
cJSON_AddStringToObject(json, "error", status.message().c_str());
|
||||
char* jsonString = cJSON_PrintUnformatted(json);
|
||||
cb->responseHandler(session, jsonString, cb->bugname);
|
||||
free(jsonString);
|
||||
cJSON_Delete(json);
|
||||
}
|
||||
|
||||
if (cb->play_file == 1) {
|
||||
cb->responseHandler(session, "play_interrupt", cb->bugname);
|
||||
}
|
||||
|
||||
for (int r = 0; r < response.results_size(); ++r) {
|
||||
auto result = response.results(r);
|
||||
cJSON * jResult = cJSON_CreateObject();
|
||||
cJSON * jAlternatives = cJSON_CreateArray();
|
||||
cJSON * jStability = cJSON_CreateNumber(result.stability());
|
||||
cJSON * jIsFinal = cJSON_CreateBool(result.is_final());
|
||||
cJSON * jLanguageCode = cJSON_CreateString(result.language_code().c_str());
|
||||
cJSON * jChannelTag = cJSON_CreateNumber(result.channel_tag());
|
||||
|
||||
auto duration = result.result_end_time();
|
||||
int32_t seconds = duration.seconds();
|
||||
int64_t nanos = duration.nanos();
|
||||
int span = (int) trunc(seconds * 1000. + ((float) nanos / 1000000.));
|
||||
cJSON * jResultEndTime = cJSON_CreateNumber(span);
|
||||
|
||||
cJSON_AddItemToObject(jResult, "stability", jStability);
|
||||
cJSON_AddItemToObject(jResult, "is_final", jIsFinal);
|
||||
cJSON_AddItemToObject(jResult, "alternatives", jAlternatives);
|
||||
cJSON_AddItemToObject(jResult, "language_code", jLanguageCode);
|
||||
cJSON_AddItemToObject(jResult, "channel_tag", jChannelTag);
|
||||
cJSON_AddItemToObject(jResult, "result_end_time", jResultEndTime);
|
||||
|
||||
for (int a = 0; a < result.alternatives_size(); ++a) {
|
||||
auto alternative = result.alternatives(a);
|
||||
cJSON* jAlt = cJSON_CreateObject();
|
||||
cJSON* jConfidence = cJSON_CreateNumber(alternative.confidence());
|
||||
cJSON* jTranscript = cJSON_CreateString(alternative.transcript().c_str());
|
||||
cJSON_AddItemToObject(jAlt, "confidence", jConfidence);
|
||||
cJSON_AddItemToObject(jAlt, "transcript", jTranscript);
|
||||
|
||||
if (alternative.words_size() > 0) {
|
||||
cJSON * jWords = cJSON_CreateArray();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: %d words\n", alternative.words_size()) ;
|
||||
for (int b = 0; b < alternative.words_size(); b++) {
|
||||
auto words = alternative.words(b);
|
||||
cJSON* jWord = cJSON_CreateObject();
|
||||
cJSON_AddItemToObject(jWord, "word", cJSON_CreateString(words.word().c_str()));
|
||||
if (words.has_start_time()) {
|
||||
cJSON_AddItemToObject(jWord, "start_time", cJSON_CreateNumber(words.start_time().seconds()));
|
||||
}
|
||||
if (words.has_end_time()) {
|
||||
cJSON_AddItemToObject(jWord, "end_time", cJSON_CreateNumber(words.end_time().seconds()));
|
||||
}
|
||||
int speaker_tag = words.speaker_tag();
|
||||
if (speaker_tag > 0) {
|
||||
cJSON_AddItemToObject(jWord, "speaker_tag", cJSON_CreateNumber(speaker_tag));
|
||||
}
|
||||
float confidence = words.confidence();
|
||||
if (confidence > 0.0) {
|
||||
cJSON_AddItemToObject(jWord, "confidence", cJSON_CreateNumber(confidence));
|
||||
}
|
||||
|
||||
cJSON_AddItemToArray(jWords, jWord);
|
||||
}
|
||||
cJSON_AddItemToObject(jAlt, "words", jWords);
|
||||
}
|
||||
cJSON_AddItemToArray(jAlternatives, jAlt);
|
||||
}
|
||||
|
||||
char* json = cJSON_PrintUnformatted(jResult);
|
||||
cb->responseHandler(session, (const char *) json, cb->bugname);
|
||||
free(json);
|
||||
|
||||
cJSON_Delete(jResult);
|
||||
}
|
||||
|
||||
if (speech_event_type == StreamingRecognizeResponse_SpeechEventType_END_OF_SINGLE_UTTERANCE) {
|
||||
// we only get this when we have requested it, and recognition stops after we get this
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: got end_of_utterance\n") ;
|
||||
cb->got_end_of_utterance = 1;
|
||||
cb->responseHandler(session, "end_of_utterance", cb->bugname);
|
||||
if (cb->wants_single_utterance) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: sending writesDone because we want only a single utterance\n") ;
|
||||
streamer->writesDone();
|
||||
}
|
||||
}
|
||||
switch_core_session_rwunlock(session);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: got %d responses\n", response.results_size());
|
||||
}
|
||||
|
||||
{
|
||||
switch_core_session_t* session = switch_core_session_locate(cb->sessionId);
|
||||
if (session) {
|
||||
grpc::Status status = streamer->finish();
|
||||
if (11 == status.error_code()) {
|
||||
if (std::string::npos != status.error_message().find("Exceeded maximum allowed stream duration")) {
|
||||
cb->responseHandler(session, "max_duration_exceeded", cb->bugname);
|
||||
}
|
||||
else {
|
||||
cb->responseHandler(session, "no_audio", cb->bugname);
|
||||
}
|
||||
}
|
||||
else if (status.error_code() != 0) {
|
||||
cJSON* json = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(json, "type", "error");
|
||||
cJSON_AddStringToObject(json, "error_cause", "stream_close");
|
||||
cJSON_AddItemToObject(json, "error_code", cJSON_CreateNumber(status.error_code()));
|
||||
cJSON_AddStringToObject(json, "error_message", status.error_message().c_str());
|
||||
char* jsonString = cJSON_PrintUnformatted(json);
|
||||
cb->responseHandler(session, jsonString, cb->bugname);
|
||||
free(jsonString);
|
||||
cJSON_Delete(json);
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: finish() status %s (%d)\n", status.error_message().c_str(), status.error_code()) ;
|
||||
switch_core_session_rwunlock(session);
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template<>
|
||||
bool GStreamer<StreamingRecognizeRequest, StreamingRecognizeResponse, Speech::Stub>::write(void* data, uint32_t datalen) {
|
||||
if (!m_connected) {
|
||||
if (datalen % CHUNKSIZE == 0) {
|
||||
m_audioBuffer.add(data, datalen);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
m_request.set_audio_content(data, datalen);
|
||||
bool ok = m_streamer->Write(m_request);
|
||||
return ok;
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
|
||||
switch_status_t google_speech_session_cleanup_v1(switch_core_session_t *session, int channelIsClosing, switch_media_bug_t *bug) {
|
||||
return google_speech_session_cleanup<GStreamer_V1>(session, channelIsClosing, bug);
|
||||
}
|
||||
|
||||
switch_bool_t google_speech_frame_v1(switch_media_bug_t *bug, void* user_data) {
|
||||
return google_speech_frame<GStreamer_V1>(bug, user_data);
|
||||
}
|
||||
|
||||
switch_status_t google_speech_session_init_v1(switch_core_session_t *session, responseHandler_t responseHandler,
|
||||
uint32_t to_rate, uint32_t samples_per_second, uint32_t channels, char* lang, int interim, char *bugname, int single_utterance,
|
||||
int separate_recognition, int max_alternatives, int profanity_filter, int word_time_offset, int punctuation, const char* model, int enhanced,
|
||||
const char* hints, char* play_file, void **ppUserData) {
|
||||
return google_speech_session_init<GStreamer_V1>(session, responseHandler, grpc_read_thread, to_rate, samples_per_second, channels,
|
||||
lang, interim, bugname, single_utterance, separate_recognition, max_alternatives, profanity_filter,
|
||||
word_time_offset, punctuation, model, enhanced, hints, play_file, ppUserData);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
#include <switch.h>
|
||||
#include <switch_json.h>
|
||||
#include <grpc++/grpc++.h>
|
||||
|
||||
#include "mod_google_transcribe.h"
|
||||
#include "gstreamer.h"
|
||||
#include "generic_google_glue.h"
|
||||
|
||||
#include "google/cloud/speech/v2/cloud_speech.grpc.pb.h"
|
||||
|
||||
using google::cloud::speech::v2::RecognitionConfig;
|
||||
using google::cloud::speech::v2::Speech;
|
||||
using google::cloud::speech::v2::StreamingRecognizeRequest;
|
||||
using google::cloud::speech::v2::StreamingRecognizeResponse;
|
||||
using google::cloud::speech::v2::SpeakerDiarizationConfig;
|
||||
using google::cloud::speech::v2::SpeechAdaptation;
|
||||
using google::cloud::speech::v2::SpeechRecognitionAlternative;
|
||||
using google::cloud::speech::v2::PhraseSet;
|
||||
using google::cloud::speech::v2::PhraseSet_Phrase;
|
||||
using google::cloud::speech::v2::StreamingRecognizeResponse_SpeechEventType_END_OF_SINGLE_UTTERANCE;
|
||||
using google::cloud::speech::v2::StreamingRecognizeResponse_SpeechEventType_SPEECH_ACTIVITY_BEGIN;
|
||||
using google::cloud::speech::v2::StreamingRecognizeResponse_SpeechEventType_SPEECH_ACTIVITY_END;
|
||||
using google::cloud::speech::v2::ExplicitDecodingConfig_AudioEncoding_LINEAR16;
|
||||
using google::cloud::speech::v2::RecognitionFeatures_MultiChannelMode_SEPARATE_RECOGNITION_PER_CHANNEL;
|
||||
using google::cloud::speech::v2::SpeechAdaptation_AdaptationPhraseSet;
|
||||
using google::rpc::Status;
|
||||
|
||||
typedef GStreamer<StreamingRecognizeRequest, StreamingRecognizeResponse, Speech::Stub> GStreamer_V2;
|
||||
|
||||
template<>
|
||||
GStreamer<StreamingRecognizeRequest, StreamingRecognizeResponse, Speech::Stub>::GStreamer(
|
||||
switch_core_session_t *session,
|
||||
uint32_t channels,
|
||||
char* lang,
|
||||
int interim,
|
||||
uint32_t config_sample_rate,
|
||||
uint32_t samples_per_second,
|
||||
int single_utterance,
|
||||
int separate_recognition,
|
||||
int max_alternatives,
|
||||
int profanity_filter,
|
||||
int word_time_offset,
|
||||
int punctuation,
|
||||
const char* model,
|
||||
int enhanced,
|
||||
const char* hints) : m_session(session), m_writesDone(false), m_connected(false),
|
||||
m_audioBuffer(CHUNKSIZE, 15) {
|
||||
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
m_channel = create_grpc_channel(channel);
|
||||
m_stub = Speech::NewStub(m_channel);
|
||||
|
||||
auto streaming_config = m_request.mutable_streaming_config();
|
||||
const char* var;
|
||||
|
||||
// The parent of the recognizer must still be provided even if the wildcard
|
||||
// recognizer is used rather than a pre-prepared recognizer.
|
||||
std::string recognizer;
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_RECOGNIZER_PARENT")) {
|
||||
recognizer = var;
|
||||
recognizer += "/recognizers/";
|
||||
} else {
|
||||
throw std::runtime_error("The v2 Speech-To-Text library requires GOOGLE_SPEECH_RECOGNIZER_PARENT to be set");
|
||||
}
|
||||
|
||||
// Use the recognizer specified in the variable or just use the wildcard if this is not set.
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_RECOGNIZER_ID")) {
|
||||
recognizer += var;
|
||||
} else {
|
||||
recognizer += "_";
|
||||
|
||||
RecognitionConfig* config = streaming_config->mutable_config();
|
||||
config->add_language_codes(lang);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "transcribe language %s\n", lang);
|
||||
|
||||
// alternative language
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_ALTERNATIVE_LANGUAGE_CODES")) {
|
||||
char *alt_langs[3] = { 0 };
|
||||
int argc = switch_separate_string((char *) var, ',', alt_langs, 3);
|
||||
for (int i = 0; i < argc; i++) {
|
||||
config->add_language_codes(alt_langs[i]);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "added alternative lang %s\n", alt_langs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
config->mutable_explicit_decoding_config()->set_sample_rate_hertz(config_sample_rate);
|
||||
config->mutable_explicit_decoding_config()->set_encoding(ExplicitDecodingConfig_AudioEncoding_LINEAR16);
|
||||
|
||||
// number of channels in the audio stream (default: 1)
|
||||
// N.B. It is essential to set this configuration value in v2 even if it doesn't deviate from the default.
|
||||
config->mutable_explicit_decoding_config()->set_audio_channel_count(channels);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "audio_channel_count %d\n", channels);
|
||||
if (channels > 1) {
|
||||
// transcribe each separately?
|
||||
if (separate_recognition == 1) {
|
||||
config->mutable_features()->set_multi_channel_mode(RecognitionFeatures_MultiChannelMode_SEPARATE_RECOGNITION_PER_CHANNEL);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_separate_recognition_per_channel on\n");
|
||||
}
|
||||
}
|
||||
|
||||
// max alternatives
|
||||
if (max_alternatives > 1) {
|
||||
config->mutable_features()->set_max_alternatives(max_alternatives);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "max_alternatives %d\n", max_alternatives);
|
||||
}
|
||||
|
||||
// profanity filter
|
||||
if (profanity_filter == 1) {
|
||||
config->mutable_features()->set_profanity_filter(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "profanity_filter\n");
|
||||
}
|
||||
|
||||
// enable word offsets
|
||||
if (word_time_offset == 1) {
|
||||
config->mutable_features()->set_enable_word_time_offsets(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_word_time_offsets\n");
|
||||
}
|
||||
|
||||
// enable automatic punctuation
|
||||
if (punctuation == 1) {
|
||||
config->mutable_features()->set_enable_automatic_punctuation(true);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enable_automatic_punctuation\n");
|
||||
}
|
||||
else {
|
||||
config->mutable_features()->set_enable_automatic_punctuation(false);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "disable_automatic_punctuation\n");
|
||||
}
|
||||
|
||||
// speech model
|
||||
if (model != NULL) {
|
||||
config->set_model(model);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "speech model %s\n", model);
|
||||
}
|
||||
|
||||
// hints
|
||||
if (hints != NULL) {
|
||||
auto* adaptation = config->mutable_adaptation();
|
||||
auto* phrase_set = adaptation->add_phrase_sets()->mutable_inline_phrase_set();
|
||||
google_speech_configure_grammar_hints(m_session, channel, hints, phrase_set);
|
||||
}
|
||||
|
||||
// the rest of config comes from channel vars
|
||||
|
||||
// speaker diarization
|
||||
// N.B. At the moment there does not seem to be any combination of model, language and location which supports diarization for STT v2.
|
||||
// See https://stackoverflow.com/questions/76779418/speaker-diarization-is-disabled-even-for-supported-languages-in-google-speech-to
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_SPEAKER_DIARIZATION")) {
|
||||
auto* diarization_config = config->mutable_features()->mutable_diarization_config();
|
||||
// There is no enable function in v2
|
||||
// diarization_config->set_enable_speaker_diarization(true);
|
||||
// switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "enabling speaker diarization\n", var);
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_SPEAKER_DIARIZATION_MIN_SPEAKER_COUNT")) {
|
||||
int count = std::max(atoi(var), 1);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "setting min speaker count to %d\n", count);
|
||||
diarization_config->set_min_speaker_count(count);
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_SPEAKER_DIARIZATION_MAX_SPEAKER_COUNT")) {
|
||||
int count = std::max(atoi(var), 2);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "setting max speaker count to %d\n", count);
|
||||
diarization_config->set_max_speaker_count(count);
|
||||
}
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_TRANSCRIPTION_NORMALIZATION")) {
|
||||
// parse JSON string
|
||||
cJSON *json_array = cJSON_Parse(var);
|
||||
|
||||
int array_size = cJSON_GetArraySize(json_array);
|
||||
|
||||
for(int i=0; i<array_size; i++) {
|
||||
cJSON* json_item = cJSON_GetArrayItem(json_array, i);
|
||||
|
||||
auto entry = config->mutable_transcript_normalization()->add_entries();
|
||||
|
||||
std::string search_string = cJSON_GetObjectItem(json_item, "search")->valuestring;
|
||||
std::string replacement_string = cJSON_GetObjectItem(json_item, "replace")->valuestring;
|
||||
bool case_sensitive = cJSON_GetObjectItem(json_item, "case_sensitive")->valueint != 0;
|
||||
|
||||
entry->set_search(search_string);
|
||||
entry->set_replace(replacement_string);
|
||||
entry->set_case_sensitive(case_sensitive);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG,
|
||||
"TRANSCRIPTION_NORMALIZATION search %s, replace %s, set_case_sensitive %d\n", search_string.c_str(), replacement_string.c_str(), case_sensitive);
|
||||
}
|
||||
// clean json
|
||||
cJSON_Delete(json_array);
|
||||
}
|
||||
}
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_START_TIMEOUT_MS")) {
|
||||
auto ms = atoi(var);
|
||||
streaming_config->mutable_streaming_features()->mutable_voice_activity_timeout()->mutable_speech_start_timeout()->set_nanos(ms * 1000000);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "setting speech_start_timeout to %d milliseconds\n", ms);
|
||||
}
|
||||
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_END_TIMEOUT_MS")) {
|
||||
auto ms = atoi(var);
|
||||
streaming_config->mutable_streaming_features()->mutable_voice_activity_timeout()->mutable_speech_end_timeout()->set_nanos(ms * 1000000);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "setting speech_end_timeout to %d milliseconds\n", ms);
|
||||
}
|
||||
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_ENABLE_VOICE_ACTIVITY_EVENTS")) {
|
||||
bool enabled = !strcmp(var, "true") ? 1 : 0;
|
||||
streaming_config->mutable_streaming_features()->set_enable_voice_activity_events(enabled);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "setting enable_voice_activity_events to %d \n", enabled);
|
||||
}
|
||||
|
||||
m_request.set_recognizer(recognizer);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_DEBUG, "using recognizer: %s\n", recognizer.c_str());
|
||||
|
||||
// This must be set whether a recognizer id is provided or not, because it cannot be configured as part of a recognizer.
|
||||
if (interim > 0) {
|
||||
streaming_config->mutable_streaming_features()->set_interim_results(interim > 0);
|
||||
}
|
||||
}
|
||||
|
||||
static void *SWITCH_THREAD_FUNC grpc_read_thread(switch_thread_t *thread, void *obj) {
|
||||
static int count;
|
||||
struct cap_cb *cb = (struct cap_cb *) obj;
|
||||
GStreamer_V2* streamer = (GStreamer_V2 *) cb->streamer;
|
||||
|
||||
bool connected = streamer->waitForConnect();
|
||||
if (!connected) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "google transcribe grpc read thread exiting since we didn't connect\n") ;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Read responses.
|
||||
StreamingRecognizeResponse response;
|
||||
while (streamer->read(&response)) { // Returns false when no more to read.
|
||||
switch_core_session_t* session = switch_core_session_locate(cb->sessionId);
|
||||
if (!session) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "grpc_read_thread: session %s is gone!\n", cb->sessionId) ;
|
||||
return nullptr;
|
||||
}
|
||||
count++;
|
||||
|
||||
if (cb->play_file == 1){
|
||||
cb->responseHandler(session, "play_interrupt", cb->bugname);
|
||||
}
|
||||
|
||||
for (int r = 0; r < response.results_size(); ++r) {
|
||||
auto result = response.results(r);
|
||||
cJSON * jResult = cJSON_CreateObject();
|
||||
cJSON * jAlternatives = cJSON_CreateArray();
|
||||
cJSON * jStability = cJSON_CreateNumber(result.stability());
|
||||
cJSON * jIsFinal = cJSON_CreateBool(result.is_final());
|
||||
cJSON * jLanguageCode = cJSON_CreateString(result.language_code().c_str());
|
||||
cJSON * jChannelTag = cJSON_CreateNumber(result.channel_tag());
|
||||
|
||||
auto duration = result.result_end_offset();
|
||||
int32_t seconds = duration.seconds();
|
||||
int64_t nanos = duration.nanos();
|
||||
int span = (int) trunc(seconds * 1000. + ((float) nanos / 1000000.));
|
||||
cJSON * jResultEndTime = cJSON_CreateNumber(span);
|
||||
|
||||
cJSON_AddItemToObject(jResult, "stability", jStability);
|
||||
cJSON_AddItemToObject(jResult, "is_final", jIsFinal);
|
||||
cJSON_AddItemToObject(jResult, "alternatives", jAlternatives);
|
||||
cJSON_AddItemToObject(jResult, "language_code", jLanguageCode);
|
||||
cJSON_AddItemToObject(jResult, "channel_tag", jChannelTag);
|
||||
cJSON_AddItemToObject(jResult, "result_end_time", jResultEndTime);
|
||||
|
||||
if (result.alternatives_size() == 0) {
|
||||
SpeechRecognitionAlternative alternative;
|
||||
alternative.set_confidence(0.0);
|
||||
alternative.set_transcript("");
|
||||
*result.add_alternatives() = alternative;
|
||||
}
|
||||
for (int a = 0; a < result.alternatives_size(); ++a) {
|
||||
auto alternative = result.alternatives(a);
|
||||
cJSON* jAlt = cJSON_CreateObject();
|
||||
cJSON* jConfidence = cJSON_CreateNumber(alternative.confidence());
|
||||
cJSON* jTranscript = cJSON_CreateString(alternative.transcript().c_str());
|
||||
cJSON_AddItemToObject(jAlt, "confidence", jConfidence);
|
||||
cJSON_AddItemToObject(jAlt, "transcript", jTranscript);
|
||||
|
||||
if (alternative.words_size() > 0) {
|
||||
cJSON * jWords = cJSON_CreateArray();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: %d words\n", alternative.words_size()) ;
|
||||
for (int b = 0; b < alternative.words_size(); b++) {
|
||||
auto words = alternative.words(b);
|
||||
cJSON* jWord = cJSON_CreateObject();
|
||||
cJSON_AddItemToObject(jWord, "word", cJSON_CreateString(words.word().c_str()));
|
||||
if (words.has_start_offset()) {
|
||||
cJSON_AddItemToObject(jWord, "start_offset", cJSON_CreateNumber(words.start_offset().seconds()));
|
||||
}
|
||||
if (words.has_end_offset()) {
|
||||
cJSON_AddItemToObject(jWord, "end_offset", cJSON_CreateNumber(words.end_offset().seconds()));
|
||||
}
|
||||
auto speaker_label = words.speaker_label();
|
||||
if (speaker_label.size() > 0) {
|
||||
cJSON_AddItemToObject(jWord, "speaker_label", cJSON_CreateString(speaker_label.c_str()));
|
||||
}
|
||||
float confidence = words.confidence();
|
||||
if (confidence > 0.0) {
|
||||
cJSON_AddItemToObject(jWord, "confidence", cJSON_CreateNumber(confidence));
|
||||
}
|
||||
|
||||
cJSON_AddItemToArray(jWords, jWord);
|
||||
}
|
||||
cJSON_AddItemToObject(jAlt, "words", jWords);
|
||||
}
|
||||
cJSON_AddItemToArray(jAlternatives, jAlt);
|
||||
}
|
||||
|
||||
char* json = cJSON_PrintUnformatted(jResult);
|
||||
cb->responseHandler(session, (const char *) json, cb->bugname);
|
||||
free(json);
|
||||
|
||||
cJSON_Delete(jResult);
|
||||
}
|
||||
|
||||
auto speech_event_type = response.speech_event_type();
|
||||
if (speech_event_type == StreamingRecognizeResponse_SpeechEventType_END_OF_SINGLE_UTTERANCE) {
|
||||
// we only get this when we have requested it, and recognition stops after we get this
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: got end_of_utterance\n") ;
|
||||
cb->got_end_of_utterance = 1;
|
||||
cb->responseHandler(session, "end_of_utterance", cb->bugname);
|
||||
if (cb->wants_single_utterance) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: sending writesDone because we want only a single utterance\n") ;
|
||||
streamer->writesDone();
|
||||
}
|
||||
}
|
||||
else if (speech_event_type == StreamingRecognizeResponse_SpeechEventType_SPEECH_ACTIVITY_BEGIN) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: got SPEECH_ACTIVITY_BEGIN\n") ;
|
||||
}
|
||||
else if (speech_event_type == StreamingRecognizeResponse_SpeechEventType_SPEECH_ACTIVITY_END) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: got SPEECH_ACTIVITY_END\n") ;
|
||||
}
|
||||
switch_core_session_rwunlock(session);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: got %d responses\n", response.results_size());
|
||||
}
|
||||
|
||||
{
|
||||
switch_core_session_t* session = switch_core_session_locate(cb->sessionId);
|
||||
if (session) {
|
||||
grpc::Status status = streamer->finish();
|
||||
// TODO: This works on the same principle as that used in the v1 equivalent, in that we search for the textual
|
||||
// error message to determine whether the cause of the problem is the expiration of the session.
|
||||
// It would be better if we could find a more reliable way of detecting this.
|
||||
if (10 == status.error_code()) {
|
||||
if (std::string::npos != status.error_message().find("Max duration of 5 minutes reached")) {
|
||||
cb->responseHandler(session, "max_duration_exceeded", cb->bugname);
|
||||
}
|
||||
else {
|
||||
cb->responseHandler(session, "no_audio", cb->bugname);
|
||||
}
|
||||
}
|
||||
else if (status.error_code() != 0) {
|
||||
cJSON* json = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(json, "type", "error");
|
||||
cJSON_AddStringToObject(json, "error_cause", "stream_close");
|
||||
cJSON_AddItemToObject(json, "error_code", cJSON_CreateNumber(status.error_code()));
|
||||
cJSON_AddStringToObject(json, "error_message", status.error_message().c_str());
|
||||
char* jsonString = cJSON_PrintUnformatted(json);
|
||||
cb->responseHandler(session, jsonString, cb->bugname);
|
||||
free(jsonString);
|
||||
cJSON_Delete(json);
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "grpc_read_thread: finish() status %s (%d)\n", status.error_message().c_str(), status.error_code()) ;
|
||||
switch_core_session_rwunlock(session);
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template <>
|
||||
bool GStreamer<StreamingRecognizeRequest, StreamingRecognizeResponse, Speech::Stub>::write(void* data, uint32_t datalen) {
|
||||
if (!m_connected) {
|
||||
if (datalen % CHUNKSIZE == 0) {
|
||||
m_audioBuffer.add(data, datalen);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
m_request.clear_streaming_config();
|
||||
m_request.set_audio(data, datalen);
|
||||
bool ok = m_streamer->Write(m_request);
|
||||
return ok;
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
|
||||
switch_status_t google_speech_session_cleanup_v2(switch_core_session_t *session, int channelIsClosing, switch_media_bug_t *bug) {
|
||||
return google_speech_session_cleanup<GStreamer_V2>(session, channelIsClosing, bug);
|
||||
}
|
||||
|
||||
switch_bool_t google_speech_frame_v2(switch_media_bug_t *bug, void* user_data) {
|
||||
return google_speech_frame<GStreamer_V2>(bug, user_data);
|
||||
}
|
||||
|
||||
switch_status_t google_speech_session_init_v2(switch_core_session_t *session, responseHandler_t responseHandler,
|
||||
uint32_t to_rate, uint32_t samples_per_second, uint32_t channels, char* lang, int interim, char *bugname, int single_utterance,
|
||||
int separate_recognition, int max_alternatives, int profanity_filter, int word_time_offset, int punctuation, const char* model, int enhanced,
|
||||
const char* hints, char* play_file, void **ppUserData) {
|
||||
return google_speech_session_init<GStreamer_V2>(session, responseHandler, grpc_read_thread, to_rate, samples_per_second, channels,
|
||||
lang, interim, bugname, single_utterance, separate_recognition, max_alternatives, profanity_filter,
|
||||
word_time_offset, punctuation, model, enhanced, hints, play_file, ppUserData);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
#include <cstdlib>
|
||||
#include <algorithm>
|
||||
#include <future>
|
||||
|
||||
#include <switch.h>
|
||||
#include <switch_json.h>
|
||||
#include <grpc++/grpc++.h>
|
||||
#include <grpcpp/impl/codegen/sync_stream.h>
|
||||
|
||||
#include "mod_google_transcribe.h"
|
||||
#include "simple_buffer.h"
|
||||
|
||||
#define CHUNKSIZE (320)
|
||||
|
||||
namespace {
|
||||
int case_insensitive_match(std::string s1, std::string s2) {
|
||||
std::transform(s1.begin(), s1.end(), s1.begin(), ::tolower);
|
||||
std::transform(s2.begin(), s2.end(), s2.begin(), ::tolower);
|
||||
if(s1.compare(s2) == 0)
|
||||
return 1; //The strings are same
|
||||
return 0; //not matched
|
||||
}
|
||||
}
|
||||
|
||||
template <typename Request, typename Response, typename Stub>
|
||||
class GStreamer {
|
||||
public:
|
||||
GStreamer(
|
||||
switch_core_session_t *session,
|
||||
uint32_t channels,
|
||||
char* lang,
|
||||
int interim,
|
||||
uint32_t config_sample_rate,
|
||||
uint32_t samples_per_second,
|
||||
int single_utterance,
|
||||
int separate_recognition,
|
||||
int max_alternatives,
|
||||
int profanity_filter,
|
||||
int word_time_offset,
|
||||
int punctuation,
|
||||
const char* model,
|
||||
int enhanced,
|
||||
const char* hints);
|
||||
|
||||
~GStreamer() {
|
||||
//switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(m_session), SWITCH_LOG_INFO, "GStreamer::~GStreamer - deleting channel and stub: %p\n", (void*)this);
|
||||
}
|
||||
|
||||
bool write(void* data, uint32_t datalen);
|
||||
|
||||
void connect() {
|
||||
assert(!m_connected);
|
||||
// Begin a stream.
|
||||
m_streamer = m_stub->StreamingRecognize(&m_context);
|
||||
m_connected = true;
|
||||
|
||||
// read thread is waiting on this
|
||||
m_promise.set_value();
|
||||
|
||||
// Write the first request, containing the config only.
|
||||
m_streamer->Write(m_request);
|
||||
|
||||
// send any buffered audio
|
||||
int nFrames = m_audioBuffer.getNumItems();
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer %p got stream ready, %d buffered frames\n", this, nFrames);
|
||||
if (nFrames) {
|
||||
char *p;
|
||||
do {
|
||||
p = m_audioBuffer.getNextChunk();
|
||||
if (p) {
|
||||
write(p, CHUNKSIZE);
|
||||
}
|
||||
} while (p);
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t nextMessageSize(void) {
|
||||
uint32_t size = 0;
|
||||
m_streamer->NextMessageSize(&size);
|
||||
return size;
|
||||
}
|
||||
|
||||
bool read(Response* response) {
|
||||
return m_streamer->Read(response);
|
||||
}
|
||||
|
||||
grpc::Status finish() {
|
||||
return m_streamer->Finish();
|
||||
}
|
||||
|
||||
void writesDone() {
|
||||
// grpc crashes if we call this twice on a stream
|
||||
if (!m_connected) {
|
||||
cancelConnect();
|
||||
}
|
||||
else if (!m_writesDone) {
|
||||
m_streamer->WritesDone();
|
||||
m_writesDone = true;
|
||||
}
|
||||
}
|
||||
|
||||
bool waitForConnect() {
|
||||
std::shared_future<void> sf(m_promise.get_future());
|
||||
sf.wait();
|
||||
return m_connected;
|
||||
}
|
||||
|
||||
void cancelConnect() {
|
||||
assert(!m_connected);
|
||||
m_promise.set_value();
|
||||
}
|
||||
|
||||
bool isConnected() {
|
||||
return m_connected;
|
||||
}
|
||||
|
||||
private:
|
||||
std::shared_ptr<grpc::Channel> create_grpc_channel(switch_channel_t *channel) {
|
||||
const char* google_uri;
|
||||
if (!(google_uri = switch_channel_get_variable(channel, "GOOGLE_SPEECH_TO_TEXT_URI"))) {
|
||||
google_uri = "speech.googleapis.com";
|
||||
}
|
||||
|
||||
const char* var;
|
||||
if (var = switch_channel_get_variable(channel, "GOOGLE_APPLICATION_CREDENTIALS")) {
|
||||
auto channelCreds = grpc::SslCredentials(grpc::SslCredentialsOptions());
|
||||
auto callCreds = grpc::ServiceAccountJWTAccessCredentials(var);
|
||||
auto creds = grpc::CompositeChannelCredentials(channelCreds, callCreds);
|
||||
return grpc::CreateChannel(google_uri, creds);
|
||||
}
|
||||
else {
|
||||
auto creds = grpc::GoogleDefaultCredentials();
|
||||
return grpc::CreateChannel(google_uri, creds);
|
||||
}
|
||||
}
|
||||
|
||||
switch_core_session_t* m_session;
|
||||
grpc::ClientContext m_context;
|
||||
std::shared_ptr<grpc::Channel> m_channel;
|
||||
std::unique_ptr<Stub> m_stub;
|
||||
std::unique_ptr< grpc::ClientReaderWriterInterface<Request, Response> > m_streamer;
|
||||
Request m_request;
|
||||
bool m_writesDone;
|
||||
bool m_connected;
|
||||
std::promise<void> m_promise;
|
||||
SimpleBuffer m_audioBuffer;
|
||||
};
|
||||
@@ -10,6 +10,13 @@
|
||||
|
||||
static const uint32_t DEFAULT_SAMPLE_RATE = 8000;
|
||||
|
||||
/* Callback Type Definitions */
|
||||
typedef switch_status_t (*speech_cleanup_callback_t) (switch_core_session_t *, int, switch_media_bug_t *);
|
||||
typedef switch_bool_t (*speech_frame_callback_t) (switch_media_bug_t *, void *);
|
||||
typedef switch_status_t (*speech_init_callback_t) (switch_core_session_t *, responseHandler_t,
|
||||
uint32_t, uint32_t, uint32_t, char *, int, char *, int, int, int, int, int, int, const char *,
|
||||
int, const char *, char *, void **);
|
||||
|
||||
/* Prototypes */
|
||||
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_transcribe_shutdown);
|
||||
SWITCH_MODULE_RUNTIME_FUNCTION(mod_transcribe_runtime);
|
||||
@@ -17,7 +24,42 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_transcribe_load);
|
||||
|
||||
SWITCH_MODULE_DEFINITION(mod_google_transcribe, mod_transcribe_load, mod_transcribe_shutdown, NULL);
|
||||
|
||||
static switch_status_t do_stop(switch_core_session_t *session, char* bugname);
|
||||
static switch_bool_t capture_callback_v1(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type);
|
||||
static switch_bool_t capture_callback_v2(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type);
|
||||
static switch_status_t do_stop(switch_core_session_t *session, char* bugname, speech_cleanup_callback_t cleanup_callback);
|
||||
|
||||
static switch_media_bug_callback_t get_bug_callback_from_version(GoogleCloudServiceVersion version) {
|
||||
switch (version) {
|
||||
case GoogleCloudServiceVersion_v1:
|
||||
return capture_callback_v1;
|
||||
case GoogleCloudServiceVersion_v2:
|
||||
return capture_callback_v2;
|
||||
default:
|
||||
return capture_callback_v1;
|
||||
}
|
||||
}
|
||||
|
||||
static speech_cleanup_callback_t get_cleanup_callback_from_version(GoogleCloudServiceVersion version) {
|
||||
switch (version) {
|
||||
case GoogleCloudServiceVersion_v1:
|
||||
return google_speech_session_cleanup_v1;
|
||||
case GoogleCloudServiceVersion_v2:
|
||||
return google_speech_session_cleanup_v2;
|
||||
default:
|
||||
return google_speech_session_cleanup_v1;
|
||||
}
|
||||
}
|
||||
|
||||
static speech_init_callback_t get_init_callback_from_version(GoogleCloudServiceVersion version) {
|
||||
switch (version) {
|
||||
case GoogleCloudServiceVersion_v1:
|
||||
return google_speech_session_init_v1;
|
||||
case GoogleCloudServiceVersion_v2:
|
||||
return google_speech_session_init_v2;
|
||||
default:
|
||||
return google_speech_session_init_v1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void responseHandler(switch_core_session_t* session, const char * json, const char* bugname) {
|
||||
@@ -34,6 +76,16 @@ static void responseHandler(switch_core_session_t* session, const char * json, c
|
||||
switch_channel_event_set_data(channel, event);
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "transcription-vendor", "google");
|
||||
}
|
||||
else if (0 == strcmp("start_of_speech", json)) {
|
||||
switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, TRANSCRIBE_EVENT_START_OF_SPEECH);
|
||||
switch_channel_event_set_data(channel, event);
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "transcription-vendor", "google");
|
||||
}
|
||||
else if (0 == strcmp("end_of_speech", json)) {
|
||||
switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, TRANSCRIBE_EVENT_END_OF_SPEECH);
|
||||
switch_channel_event_set_data(channel, event);
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "transcription-vendor", "google");
|
||||
}
|
||||
else if (0 == strcmp("end_of_transcript", json)) {
|
||||
switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, TRANSCRIBE_EVENT_END_OF_TRANSCRIPT);
|
||||
switch_channel_event_set_data(channel, event);
|
||||
@@ -69,19 +121,19 @@ static void responseHandler(switch_core_session_t* session, const char * json, c
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "transcription-vendor", "google");
|
||||
}
|
||||
else {
|
||||
int error = 0;
|
||||
cJSON* jMessage = cJSON_Parse(json);
|
||||
if (jMessage) {
|
||||
const char* type = cJSON_GetStringValue(cJSON_GetObjectItem(jMessage, "type"));
|
||||
if (type && 0 == strcmp(type, "error")) {
|
||||
error = 1;
|
||||
switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, TRANSCRIBE_EVENT_ERROR);
|
||||
}
|
||||
cJSON_Delete(jMessage);
|
||||
}
|
||||
if (!error) {
|
||||
switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, TRANSCRIBE_EVENT_RESULTS);
|
||||
}
|
||||
int error = 0;
|
||||
cJSON* jMessage = cJSON_Parse(json);
|
||||
if (jMessage) {
|
||||
const char* type = cJSON_GetStringValue(cJSON_GetObjectItem(jMessage, "type"));
|
||||
if (type && 0 == strcmp(type, "error")) {
|
||||
error = 1;
|
||||
switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, TRANSCRIBE_EVENT_ERROR);
|
||||
}
|
||||
cJSON_Delete(jMessage);
|
||||
}
|
||||
if (!error) {
|
||||
switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, TRANSCRIBE_EVENT_RESULTS);
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "%s json payload: %s.\n", bugname ? bugname : "google_transcribe", json);
|
||||
switch_channel_event_set_data(channel, event);
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "transcription-vendor", "google");
|
||||
@@ -91,29 +143,29 @@ static void responseHandler(switch_core_session_t* session, const char * json, c
|
||||
switch_event_fire(&event);
|
||||
}
|
||||
|
||||
static switch_bool_t capture_callback(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type)
|
||||
static switch_bool_t capture_callback(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type,
|
||||
speech_frame_callback_t frame_callback, speech_cleanup_callback_t cleanup_callback)
|
||||
{
|
||||
switch_core_session_t *session = switch_core_media_bug_get_session(bug);
|
||||
struct cap_cb* cb = (struct cap_cb*) switch_core_media_bug_get_user_data(bug);
|
||||
|
||||
switch (type) {
|
||||
case SWITCH_ABC_TYPE_INIT:
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Got SWITCH_ABC_TYPE_INIT.\n");
|
||||
responseHandler(session, "start_of_transcript", cb->bugname);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Got SWITCH_ABC_TYPE_INIT.\n");
|
||||
responseHandler(session, "start_of_transcript", cb->bugname);
|
||||
break;
|
||||
|
||||
case SWITCH_ABC_TYPE_CLOSE:
|
||||
{
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Got SWITCH_ABC_TYPE_CLOSE, calling google_speech_session_cleanup.\n");
|
||||
responseHandler(session, "end_of_transcript", cb->bugname);
|
||||
google_speech_session_cleanup(session, 1, bug);
|
||||
cleanup_callback(session, 1, bug);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Finished SWITCH_ABC_TYPE_CLOSE.\n");
|
||||
}
|
||||
break;
|
||||
|
||||
case SWITCH_ABC_TYPE_READ:
|
||||
|
||||
return google_speech_frame(bug, user_data);
|
||||
return frame_callback(bug, user_data);
|
||||
break;
|
||||
|
||||
case SWITCH_ABC_TYPE_WRITE:
|
||||
@@ -124,19 +176,27 @@ static switch_bool_t capture_callback(switch_media_bug_t *bug, void *user_data,
|
||||
return SWITCH_TRUE;
|
||||
}
|
||||
|
||||
static switch_bool_t capture_callback_v1(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type) {
|
||||
return capture_callback(bug, user_data, type, google_speech_frame_v1, google_speech_session_cleanup_v1);
|
||||
}
|
||||
|
||||
static switch_bool_t capture_callback_v2(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type) {
|
||||
return capture_callback(bug, user_data, type, google_speech_frame_v2, google_speech_session_cleanup_v2);
|
||||
}
|
||||
|
||||
static switch_status_t transcribe_input_callback(switch_core_session_t *session, void *input, switch_input_type_t input_type, void *data, unsigned int len){
|
||||
if (input_type == SWITCH_INPUT_TYPE_EVENT) {
|
||||
switch_event_t *event;
|
||||
event = (switch_event_t *)input;
|
||||
if (event->event_id == SWITCH_EVENT_DETECTED_SPEECH) {
|
||||
return SWITCH_STATUS_BREAK;
|
||||
}
|
||||
switch_event_t *event;
|
||||
event = (switch_event_t *)input;
|
||||
if (event->event_id == SWITCH_EVENT_DETECTED_SPEECH) {
|
||||
return SWITCH_STATUS_BREAK;
|
||||
}
|
||||
}
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static switch_status_t do_stop(switch_core_session_t *session, char *bugname)
|
||||
static switch_status_t do_stop(switch_core_session_t *session, char *bugname, speech_cleanup_callback_t cleanup_callback)
|
||||
{
|
||||
switch_status_t status = SWITCH_STATUS_SUCCESS;
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
@@ -144,7 +204,7 @@ static switch_status_t do_stop(switch_core_session_t *session, char *bugname)
|
||||
|
||||
if (bug) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Received user command command, calling google_speech_session_cleanup (possibly to stop prev transcribe)\n");
|
||||
status = google_speech_session_cleanup(session, 0, bug);
|
||||
status = cleanup_callback(session, 0, bug);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "stopped transcription.\n");
|
||||
}
|
||||
|
||||
@@ -153,7 +213,8 @@ static switch_status_t do_stop(switch_core_session_t *session, char *bugname)
|
||||
|
||||
static switch_status_t start_capture2(switch_core_session_t *session, switch_media_bug_flag_t flags,
|
||||
uint32_t sample_rate, char* lang, int interim, int single_utterance, int separate_recognition, int max_alternatives,
|
||||
int profinity_filter, int word_time_offset, int punctuation, const char* model, int enhanced, const char* hints, char* play_file)
|
||||
int profanity_filter, int word_time_offset, int punctuation, const char* model, int enhanced, const char* hints,
|
||||
char* play_file, GoogleCloudServiceVersion version)
|
||||
{
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_media_bug_t *bug;
|
||||
@@ -165,7 +226,7 @@ static switch_status_t start_capture2(switch_core_session_t *session, switch_med
|
||||
|
||||
if (switch_channel_get_private(channel, MY_BUG_NAME)) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing bug from previous transcribe\n");
|
||||
do_stop(session, MY_BUG_NAME);
|
||||
do_stop(session, MY_BUG_NAME, get_cleanup_callback_from_version(version));
|
||||
}
|
||||
|
||||
switch_core_session_get_read_impl(session, &read_impl);
|
||||
@@ -175,13 +236,15 @@ static switch_status_t start_capture2(switch_core_session_t *session, switch_med
|
||||
}
|
||||
|
||||
samples_per_second = !strcasecmp(read_impl.iananame, "g722") ? read_impl.actual_samples_per_second : read_impl.samples_per_second;
|
||||
status = get_init_callback_from_version(version)(session, responseHandler, sample_rate, samples_per_second, flags & SMBF_STEREO ? 2 : 1, lang, interim, MY_BUG_NAME, single_utterance,
|
||||
separate_recognition, max_alternatives, profanity_filter, word_time_offset, punctuation, model, enhanced, hints, play_file, &pUserData);
|
||||
|
||||
if (SWITCH_STATUS_FALSE == google_speech_session_init(session, responseHandler, sample_rate, samples_per_second, flags & SMBF_STEREO ? 2 : 1, lang, interim, MY_BUG_NAME, single_utterance,
|
||||
separate_recognition, max_alternatives, profinity_filter, word_time_offset, punctuation, model, enhanced, hints, play_file, &pUserData)) {
|
||||
if (SWITCH_STATUS_FALSE == status) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing google speech session.\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if ((status = switch_core_media_bug_add(session, "google_transcribe", NULL, capture_callback, pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
|
||||
|
||||
if ((status = switch_core_media_bug_add(session, "google_transcribe", NULL, get_bug_callback_from_version(version), pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
|
||||
return status;
|
||||
}
|
||||
|
||||
@@ -197,22 +260,23 @@ static switch_status_t start_capture2(switch_core_session_t *session, switch_med
|
||||
}
|
||||
|
||||
static switch_status_t start_capture(switch_core_session_t *session, switch_media_bug_flag_t flags,
|
||||
char* lang, int interim, char* bugname)
|
||||
char* lang, int interim, char* bugname, GoogleCloudServiceVersion version)
|
||||
{
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_media_bug_t *bug;
|
||||
switch_status_t status;
|
||||
switch_codec_implementation_t read_impl = { 0 };
|
||||
void *pUserData;
|
||||
uint32_t sample_rate = DEFAULT_SAMPLE_RATE;
|
||||
uint32_t samples_per_second;
|
||||
int single_utterance = 0, separate_recognition = 0, max_alternatives = 0, profanity_filter = 0, word_time_offset = 0, punctuation = 0, enhanced = 0;
|
||||
const char* hints = NULL;
|
||||
const char* model = NULL;
|
||||
const char* model = NULL;
|
||||
const char* var;
|
||||
|
||||
|
||||
if (switch_channel_get_private(channel, bugname)) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing bug from previous transcribe\n");
|
||||
do_stop(session, bugname);
|
||||
do_stop(session, bugname, get_cleanup_callback_from_version(version));
|
||||
}
|
||||
|
||||
if (switch_true(switch_channel_get_variable(channel, "GOOGLE_SPEECH_SINGLE_UTTERANCE"))) {
|
||||
@@ -229,6 +293,11 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
|
||||
max_alternatives = atoi(var);
|
||||
}
|
||||
|
||||
// sample rate
|
||||
if ((var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_SAMPLE_RATE"))) {
|
||||
sample_rate = atoi(var);
|
||||
}
|
||||
|
||||
// profanity filter
|
||||
if (switch_true(switch_channel_get_variable(channel, "GOOGLE_SPEECH_PROFANITY_FILTER"))) {
|
||||
profanity_filter = 1;
|
||||
@@ -266,14 +335,16 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
|
||||
}
|
||||
|
||||
samples_per_second = !strcasecmp(read_impl.iananame, "g722") ? read_impl.actual_samples_per_second : read_impl.samples_per_second;
|
||||
status = SWITCH_STATUS_FALSE;
|
||||
status = get_init_callback_from_version(version)(session, responseHandler, sample_rate, samples_per_second, flags & SMBF_STEREO ? 2 : 1, lang, interim, bugname, single_utterance,
|
||||
separate_recognition, max_alternatives, profanity_filter, word_time_offset, punctuation, model, enhanced, hints, NULL, &pUserData);
|
||||
|
||||
if (SWITCH_STATUS_FALSE == google_speech_session_init(session, responseHandler, DEFAULT_SAMPLE_RATE, samples_per_second, flags & SMBF_STEREO ? 2 : 1, lang, interim, bugname, single_utterance,
|
||||
separate_recognition, max_alternatives, profanity_filter, word_time_offset, punctuation, model, enhanced, hints, NULL, &pUserData)) {
|
||||
if (SWITCH_STATUS_FALSE == status) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing google speech session.\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if ((status = switch_core_media_bug_add(session, bugname, NULL, capture_callback, pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
|
||||
if ((status = switch_core_media_bug_add(session, bugname, NULL, get_bug_callback_from_version(version), pUserData, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
|
||||
return status;
|
||||
}
|
||||
|
||||
@@ -283,7 +354,7 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
|
||||
}
|
||||
|
||||
// #define TRANSCRIBE_API_SYNTAX "<uuid> [start|stop] [lang-code] [interim] [single-utterance](bool) [seperate-recognition](bool) [max-alternatives](int) [profinity-filter](bool) [word-time](bool) [punctuation](bool) [model](string) [enhanced](true) [hints](string without space) [play-file]"
|
||||
#define TRANSCRIBE2_API_SYNTAX "<uuid> [start|stop] [lang-code] [interim] [single-utterance] [seperate-recognition] [max-alternatives] [profinity-filter] [word-time] [punctuation] [sample-rate] [model] [enhanced] [hints] [play-file]"
|
||||
#define TRANSCRIBE2_API_SYNTAX "<uuid> [start|stop] [lang-code] [interim] [single-utterance] [separate-recognition] [max-alternatives] [profanity-filter] [word-time] [punctuation] [sample-rate] [model] [enhanced] [hints] [play-file]"
|
||||
SWITCH_STANDARD_API(transcribe2_function)
|
||||
{
|
||||
char *mycmd = NULL, *argv[20] = { 0 };
|
||||
@@ -295,6 +366,9 @@ SWITCH_STANDARD_API(transcribe2_function)
|
||||
|
||||
switch_status_t status = SWITCH_STATUS_FALSE;
|
||||
switch_media_bug_flag_t flags = SMBF_READ_STREAM /* | SMBF_WRITE_STREAM | SMBF_READ_PING */;
|
||||
switch_channel_t *channel;
|
||||
const char* var;
|
||||
GoogleCloudServiceVersion version = GoogleCloudServiceVersion_v1;
|
||||
|
||||
if (!zstr(cmd) && (mycmd = strdup(cmd))) {
|
||||
argc = switch_separate_string(mycmd, ' ', argv, (sizeof(argv) / sizeof(argv[0])));
|
||||
@@ -311,16 +385,24 @@ SWITCH_STANDARD_API(transcribe2_function)
|
||||
switch_core_session_t *lsession = NULL;
|
||||
|
||||
if ((lsession = switch_core_session_locate(argv[0]))) {
|
||||
channel = switch_core_session_get_channel(lsession);
|
||||
if ((var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_CLOUD_SERVICES_VERSION"))) {
|
||||
if (!strcasecmp(var, "v1"))
|
||||
version = GoogleCloudServiceVersion_v1;
|
||||
else if (!strcasecmp(var, "v2"))
|
||||
version = GoogleCloudServiceVersion_v2;
|
||||
}
|
||||
|
||||
if (!strcasecmp(argv[1], "stop")) {
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "stop transcribing\n");
|
||||
status = do_stop(lsession, MY_BUG_NAME);
|
||||
status = do_stop(lsession, MY_BUG_NAME, get_cleanup_callback_from_version(version));
|
||||
} else if (!strcasecmp(argv[1], "start")) {
|
||||
char* lang = argv[2];
|
||||
int interim = argc > 3 && !strcmp(argv[3], "true");
|
||||
char* lang = argv[2];
|
||||
int interim = argc > 3 && !strcmp(argv[3], "true");
|
||||
int single_utterance = !strcmp(argv[4], "true"); // single-utterance
|
||||
int separate_recognition = !strcmp(argv[5], "true"); // sepreate-recognition
|
||||
int separate_recognition = !strcmp(argv[5], "true"); // separate-recognition
|
||||
int max_alternatives = atoi(argv[6]); // max-alternatives
|
||||
int profinity_filter = !strcmp(argv[7], "true"); // profinity-filter
|
||||
int profanity_filter = !strcmp(argv[7], "true"); // profanity-filter
|
||||
int word_time_offset = !strcmp(argv[8], "true"); // word-time
|
||||
int punctuation = !strcmp(argv[9], "true"); //punctuation
|
||||
if (argc > 10) {
|
||||
@@ -336,9 +418,9 @@ SWITCH_STANDARD_API(transcribe2_function)
|
||||
if (argc > 14){
|
||||
play_file = argv[14];
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "start transcribing %s %s\n", lang, interim ? "interim": "complete");
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "start transcribing %s %s\n", lang, interim ? "interim": "complete");
|
||||
status = start_capture2(lsession, flags, sample_rate, lang, interim, single_utterance, separate_recognition,max_alternatives,
|
||||
profinity_filter, word_time_offset, punctuation, model, enhanced, hints, play_file);
|
||||
profanity_filter, word_time_offset, punctuation, model, enhanced, hints, play_file, version);
|
||||
}
|
||||
switch_core_session_rwunlock(lsession);
|
||||
}
|
||||
@@ -363,6 +445,9 @@ SWITCH_STANDARD_API(transcribe_function)
|
||||
int argc = 0;
|
||||
switch_status_t status = SWITCH_STATUS_FALSE;
|
||||
switch_media_bug_flag_t flags = SMBF_READ_STREAM /* | SMBF_WRITE_STREAM | SMBF_READ_PING */;
|
||||
switch_channel_t *channel;
|
||||
const char* var;
|
||||
GoogleCloudServiceVersion version = GoogleCloudServiceVersion_v1;
|
||||
|
||||
if (!zstr(cmd) && (mycmd = strdup(cmd))) {
|
||||
argc = switch_separate_string(mycmd, ' ', argv, (sizeof(argv) / sizeof(argv[0])));
|
||||
@@ -379,20 +464,28 @@ SWITCH_STANDARD_API(transcribe_function)
|
||||
switch_core_session_t *lsession = NULL;
|
||||
|
||||
if ((lsession = switch_core_session_locate(argv[0]))) {
|
||||
channel = switch_core_session_get_channel(lsession);
|
||||
if ((var = switch_channel_get_variable(channel, "GOOGLE_SPEECH_CLOUD_SERVICES_VERSION"))) {
|
||||
if (!strcasecmp(var, "v1"))
|
||||
version = GoogleCloudServiceVersion_v1;
|
||||
else if (!strcasecmp(var, "v2"))
|
||||
version = GoogleCloudServiceVersion_v2;
|
||||
}
|
||||
|
||||
if (!strcasecmp(argv[1], "stop")) {
|
||||
char *bugname = argc > 2 ? argv[2] : MY_BUG_NAME;
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "stop transcribing\n");
|
||||
status = do_stop(lsession, bugname);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "stop transcribing\n");
|
||||
status = do_stop(lsession, bugname, get_cleanup_callback_from_version(version));
|
||||
} else if (!strcasecmp(argv[1], "start")) {
|
||||
char* lang = argv[2];
|
||||
int interim = argc > 3 && !strcmp(argv[3], "interim");
|
||||
char* lang = argv[2];
|
||||
int interim = argc > 3 && !strcmp(argv[3], "interim");
|
||||
char *bugname = argc > 5 ? argv[5] : MY_BUG_NAME;
|
||||
if (argc > 4 && !strcmp(argv[4], "stereo")) {
|
||||
flags |= SMBF_WRITE_STREAM ;
|
||||
flags |= SMBF_STEREO;
|
||||
flags |= SMBF_WRITE_STREAM ;
|
||||
flags |= SMBF_STEREO;
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s start transcribing %s %s\n", bugname, lang, interim ? "interim": "complete");
|
||||
status = start_capture(lsession, flags, lang, interim, bugname);
|
||||
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_INFO, "%s start transcribing %s %s\n", bugname, lang, interim ? "interim": "complete");
|
||||
status = start_capture(lsession, flags, lang, interim, bugname, version);
|
||||
}
|
||||
switch_core_session_rwunlock(lsession);
|
||||
}
|
||||
@@ -423,6 +516,14 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_transcribe_load)
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't register subclass %s!\n", TRANSCRIBE_EVENT_END_OF_UTTERANCE);
|
||||
return SWITCH_STATUS_TERM;
|
||||
}
|
||||
if (switch_event_reserve_subclass(TRANSCRIBE_EVENT_START_OF_SPEECH) != SWITCH_STATUS_SUCCESS) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't register subclass %s!\n", TRANSCRIBE_EVENT_START_OF_SPEECH);
|
||||
return SWITCH_STATUS_TERM;
|
||||
}
|
||||
if (switch_event_reserve_subclass(TRANSCRIBE_EVENT_END_OF_SPEECH) != SWITCH_STATUS_SUCCESS) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't register subclass %s!\n", TRANSCRIBE_EVENT_END_OF_SPEECH);
|
||||
return SWITCH_STATUS_TERM;
|
||||
}
|
||||
if (switch_event_reserve_subclass(TRANSCRIBE_EVENT_START_OF_TRANSCRIPT) != SWITCH_STATUS_SUCCESS) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't register subclass %s!\n", TRANSCRIBE_EVENT_START_OF_TRANSCRIPT);
|
||||
return SWITCH_STATUS_TERM;
|
||||
@@ -473,6 +574,8 @@ SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_transcribe_shutdown)
|
||||
google_speech_cleanup();
|
||||
switch_event_free_subclass(TRANSCRIBE_EVENT_RESULTS);
|
||||
switch_event_free_subclass(TRANSCRIBE_EVENT_END_OF_UTTERANCE);
|
||||
switch_event_free_subclass(TRANSCRIBE_EVENT_START_OF_SPEECH);
|
||||
switch_event_free_subclass(TRANSCRIBE_EVENT_END_OF_SPEECH);
|
||||
switch_event_free_subclass(TRANSCRIBE_EVENT_START_OF_TRANSCRIPT);
|
||||
switch_event_free_subclass(TRANSCRIBE_EVENT_END_OF_TRANSCRIPT);
|
||||
switch_event_free_subclass(TRANSCRIBE_EVENT_NO_AUDIO_DETECTED);
|
||||
@@ -481,4 +584,3 @@ SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_transcribe_shutdown)
|
||||
switch_event_free_subclass(TRANSCRIBE_EVENT_PLAY_INTERRUPT);
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
#define MY_BUG_NAME "google_transcribe"
|
||||
#define TRANSCRIBE_EVENT_RESULTS "google_transcribe::transcription"
|
||||
#define TRANSCRIBE_EVENT_END_OF_UTTERANCE "google_transcribe::end_of_utterance"
|
||||
#define TRANSCRIBE_EVENT_START_OF_SPEECH "google_transcribe::start_of_speech"
|
||||
#define TRANSCRIBE_EVENT_END_OF_SPEECH "google_transcribe::end_of_speech"
|
||||
#define TRANSCRIBE_EVENT_START_OF_TRANSCRIPT "google_transcribe::start_of_transcript"
|
||||
#define TRANSCRIBE_EVENT_END_OF_TRANSCRIPT "google_transcribe::end_of_transcript"
|
||||
#define TRANSCRIBE_EVENT_NO_AUDIO_DETECTED "google_transcribe::no_audio_detected"
|
||||
@@ -37,6 +39,10 @@ struct cap_cb {
|
||||
#else
|
||||
/* per-channel data */
|
||||
typedef void (*responseHandler_t)(switch_core_session_t* session, const char* json, const char* bugname);
|
||||
typedef enum GoogleCloudServiceVersion {
|
||||
GoogleCloudServiceVersion_v1,
|
||||
GoogleCloudServiceVersion_v2
|
||||
} GoogleCloudServiceVersion;
|
||||
|
||||
struct cap_cb {
|
||||
switch_mutex_t *mutex;
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
Copyright 2023, Drachtio Communications Services, LLC
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
include $(top_srcdir)/build/modmake.rulesam
|
||||
MODNAME=mod_playht_tts
|
||||
|
||||
mod_LTLIBRARIES = mod_playht_tts.la
|
||||
mod_playht_tts_la_SOURCES = mod_playht_tts.c playht_glue.cpp
|
||||
mod_playht_tts_la_CFLAGS = $(AM_CFLAGS)
|
||||
mod_playht_tts_la_LIBADD = $(switch_builddir)/libfreeswitch.la
|
||||
mod_playht_tts_la_LDFLAGS = -avoid-version -module -no-undefined -shared -lstdc++ -lboost_system -lboost_thread -lmpg123
|
||||
@@ -0,0 +1,3 @@
|
||||
# mod_playht_tts
|
||||
|
||||
A Freeswitch module that allows speak text to speech audio from playht stream.
|
||||
@@ -0,0 +1,198 @@
|
||||
#include "mod_playht_tts.h"
|
||||
#include "playht_glue.h"
|
||||
|
||||
SWITCH_MODULE_LOAD_FUNCTION(mod_playht_tts_load);
|
||||
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_playht_tts_shutdown);
|
||||
SWITCH_MODULE_DEFINITION(mod_playht_tts, mod_playht_tts_load, mod_playht_tts_shutdown, NULL);
|
||||
|
||||
static void clearPlayht(playht_t* p, int freeAll) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "clearPlayht\n");
|
||||
if (p->api_key) free(p->api_key);
|
||||
if (p->user_id) free(p->user_id);
|
||||
if (p->quality) free(p->quality);
|
||||
if (p->speed) free(p->speed);
|
||||
if (p->seed) free(p->seed);
|
||||
if (p->temperature) free(p->temperature);
|
||||
if (p->voice_engine) free(p->voice_engine);
|
||||
if (p->emotion) free(p->emotion);
|
||||
if (p->voice_guidance) free(p->voice_guidance);
|
||||
if (p->style_guidance) free(p->style_guidance);
|
||||
if (p->text_guidance) free(p->text_guidance);
|
||||
|
||||
|
||||
if (p->request_id) free(p->request_id);
|
||||
if (p->ct) free(p->ct);
|
||||
if (p->err_msg) free(p->err_msg);
|
||||
if (p->name_lookup_time_ms) free(p->name_lookup_time_ms);
|
||||
if (p->connect_time_ms) free(p->connect_time_ms);
|
||||
if (p->final_response_time_ms) free(p->final_response_time_ms);
|
||||
if (p->cache_filename) free(p->cache_filename);
|
||||
|
||||
|
||||
p->api_key = NULL;
|
||||
p->user_id = NULL;
|
||||
p->quality = NULL;
|
||||
p->speed = NULL;
|
||||
p->seed = NULL;
|
||||
p->temperature = NULL;
|
||||
p->voice_engine = NULL;
|
||||
p->emotion = NULL;
|
||||
p->voice_guidance = NULL;
|
||||
p->style_guidance = NULL;
|
||||
p->text_guidance = NULL;
|
||||
|
||||
p->request_id = NULL;
|
||||
p->ct = NULL;
|
||||
p->err_msg = NULL;
|
||||
p->name_lookup_time_ms = NULL;
|
||||
p->connect_time_ms = NULL;
|
||||
p->final_response_time_ms = NULL;
|
||||
p->cache_filename = NULL;
|
||||
|
||||
if (freeAll) {
|
||||
if (p->voice_name) free(p->voice_name);
|
||||
if (p->session_id) free(p->session_id);
|
||||
p->voice_name = NULL;
|
||||
p->session_id = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static playht_t * createOrRetrievePrivateData(switch_speech_handle_t *sh) {
|
||||
playht_t *p = (playht_t *) sh->private_info;
|
||||
if (!p) {
|
||||
p = switch_core_alloc(sh->memory_pool, sizeof(*p));
|
||||
sh->private_info = p;
|
||||
memset(p, 0, sizeof(*p));
|
||||
switch_mutex_init(&p->mutex, SWITCH_MUTEX_NESTED, sh->memory_pool);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "allocated playht_t\n");
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
switch_status_t p_speech_open(switch_speech_handle_t *sh, const char *voice_name, int rate, int channels, switch_speech_flag_t *flags)
|
||||
{
|
||||
playht_t *p = createOrRetrievePrivateData(sh);
|
||||
p->voice_name = strdup(voice_name);
|
||||
p->rate = rate;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "p_speech_open voice: %s, rate %d, channels %d\n", voice_name, rate, channels);
|
||||
return playht_speech_open(p);
|
||||
}
|
||||
|
||||
static switch_status_t p_speech_close(switch_speech_handle_t *sh, switch_speech_flag_t *flags)
|
||||
{
|
||||
switch_status_t rc;
|
||||
playht_t *p = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "p_speech_close\n");
|
||||
|
||||
switch_mutex_destroy(p->mutex);
|
||||
|
||||
rc = playht_speech_close(p);
|
||||
clearPlayht(p, 1);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeswitch will call this function to feed us text to speak
|
||||
*/
|
||||
static switch_status_t p_speech_feed_tts(switch_speech_handle_t *sh, char *text, switch_speech_flag_t *flags)
|
||||
{
|
||||
playht_t *p = createOrRetrievePrivateData(sh);
|
||||
p->draining = 0;
|
||||
p->reads = 0;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "p_speech_feed_tts\n");
|
||||
|
||||
return playht_speech_feed_tts(p, text, flags);
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeswitch calls periodically to get some rendered audio in L16 format. We can provide up to 8k of audio at a time.
|
||||
*/
|
||||
static switch_status_t p_speech_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, switch_speech_flag_t *flags)
|
||||
{
|
||||
playht_t *p = createOrRetrievePrivateData(sh);
|
||||
return playht_speech_read_tts(p, data, datalen, flags);
|
||||
}
|
||||
|
||||
/**
|
||||
* This is called at the end, not sure exactly what we need to do here..
|
||||
*/
|
||||
static void p_speech_flush_tts(switch_speech_handle_t *sh)
|
||||
{
|
||||
playht_t *p = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "p_speech_flush_tts\n");
|
||||
playht_speech_flush_tts(p);
|
||||
|
||||
clearPlayht(p, 0);
|
||||
}
|
||||
|
||||
static void p_text_param_tts(switch_speech_handle_t *sh, char *param, const char *val)
|
||||
{
|
||||
playht_t *p = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "p_text_param_tts: %s=%s\n", param, val);
|
||||
if (0 == strcmp(param, "api_key")) {
|
||||
if (p->api_key) free(p->api_key);
|
||||
p->api_key = strdup(val);
|
||||
} else if (0 == strcmp(param, "user_id")) {
|
||||
if (p->user_id) free(p->user_id);
|
||||
p->user_id = strdup(val);
|
||||
} else if (0 == strcmp(param, "quality")) {
|
||||
if (p->quality) free(p->quality);
|
||||
p->quality = strdup(val);
|
||||
} else if (0 == strcmp(param, "speed")) {
|
||||
if (p->speed) free(p->speed);
|
||||
p->speed = strdup(val);
|
||||
} else if (0 == strcmp(param, "seed")) {
|
||||
if (p->seed) free(p->seed);
|
||||
p->seed = strdup(val);
|
||||
} else if (0 == strcmp(param, "temperature")) {
|
||||
if (p->temperature) free(p->temperature);
|
||||
p->temperature = strdup(val);
|
||||
} else if (0 == strcmp(param, "voice_engine")) {
|
||||
if (p->voice_engine) free(p->voice_engine);
|
||||
p->voice_engine = strdup(val);
|
||||
} else if (0 == strcmp(param, "emotion")) {
|
||||
if (p->emotion) free(p->emotion);
|
||||
p->emotion = strdup(val);
|
||||
} else if (0 == strcmp(param, "voice_guidance")) {
|
||||
if (p->voice_guidance) free(p->voice_guidance);
|
||||
p->voice_guidance = strdup(val);
|
||||
} else if (0 == strcmp(param, "style_guidance")) {
|
||||
if (p->style_guidance) free(p->style_guidance);
|
||||
p->style_guidance = strdup(val);
|
||||
} else if (0 == strcmp(param, "session-uuid")) {
|
||||
if (p->session_id) free(p->session_id);
|
||||
p->session_id = strdup(val);
|
||||
} else if (0 == strcmp(param, "write_cache_file") && switch_true(val)) {
|
||||
p->cache_audio = 1;
|
||||
}
|
||||
}
|
||||
static void p_numeric_param_tts(switch_speech_handle_t *sh, char *param, int val)
|
||||
{
|
||||
}
|
||||
static void p_float_param_tts(switch_speech_handle_t *sh, char *param, double val)
|
||||
{
|
||||
}
|
||||
|
||||
SWITCH_MODULE_LOAD_FUNCTION(mod_playht_tts_load)
|
||||
{
|
||||
switch_speech_interface_t *speech_interface;
|
||||
|
||||
*module_interface = switch_loadable_module_create_module_interface(pool, modname);
|
||||
speech_interface = switch_loadable_module_create_interface(*module_interface, SWITCH_SPEECH_INTERFACE);
|
||||
speech_interface->interface_name = "playht";
|
||||
speech_interface->speech_open = p_speech_open;
|
||||
speech_interface->speech_close = p_speech_close;
|
||||
speech_interface->speech_feed_tts = p_speech_feed_tts;
|
||||
speech_interface->speech_read_tts = p_speech_read_tts;
|
||||
speech_interface->speech_flush_tts = p_speech_flush_tts;
|
||||
speech_interface->speech_text_param_tts = p_text_param_tts;
|
||||
speech_interface->speech_numeric_param_tts = p_numeric_param_tts;
|
||||
speech_interface->speech_float_param_tts = p_float_param_tts;
|
||||
return playht_speech_load();
|
||||
}
|
||||
|
||||
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_playht_tts_shutdown)
|
||||
{
|
||||
return playht_speech_unload();
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
#ifndef __MOD_PLAYHT_TTS_H__
|
||||
#define __MOD_PLAYHT_TTS_H__
|
||||
|
||||
#include <switch.h>
|
||||
typedef struct playht_data {
|
||||
char *voice_name;
|
||||
char *api_key;
|
||||
char *user_id;
|
||||
char *quality;
|
||||
char *speed;
|
||||
char *seed;
|
||||
char *temperature;
|
||||
char *voice_engine;
|
||||
char *emotion;
|
||||
char *voice_guidance;
|
||||
char *style_guidance;
|
||||
char *text_guidance;
|
||||
|
||||
/* result data */
|
||||
long response_code;
|
||||
char *ct;
|
||||
char *request_id;
|
||||
char *name_lookup_time_ms;
|
||||
char *connect_time_ms;
|
||||
char *final_response_time_ms;
|
||||
char *err_msg;
|
||||
char *cache_filename;
|
||||
char *session_id;
|
||||
|
||||
int rate;
|
||||
int draining;
|
||||
int reads;
|
||||
int cache_audio;
|
||||
|
||||
void *conn;
|
||||
void *circularBuffer;
|
||||
switch_mutex_t *mutex;
|
||||
FILE *file;
|
||||
} playht_t;
|
||||
#endif
|
||||
@@ -0,0 +1,996 @@
|
||||
#include "mod_playht_tts.h"
|
||||
#include <switch.h>
|
||||
#include <switch_json.h>
|
||||
#include <curl/curl.h>
|
||||
#include <cstdlib>
|
||||
|
||||
#include <boost/circular_buffer.hpp>
|
||||
#include <boost/thread.hpp>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/asio/ssl.hpp>
|
||||
#include <boost/pool/object_pool.hpp>
|
||||
#include <boost/bind/bind.hpp>
|
||||
#include <boost/tokenizer.hpp>
|
||||
#include <boost/foreach.hpp>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/assign/list_of.hpp>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
|
||||
#include "mpg123.h"
|
||||
|
||||
#define BUFFER_GROW_SIZE (8192)
|
||||
#define MP3_DCACHE 8192 * 2
|
||||
|
||||
typedef boost::circular_buffer<uint16_t> CircularBuffer_t;
|
||||
/* Global information, common to all connections */
|
||||
typedef struct
|
||||
{
|
||||
CURLM *multi;
|
||||
int still_running;
|
||||
} GlobalInfo_t;
|
||||
static GlobalInfo_t global;
|
||||
|
||||
/* Information associated with a specific easy handle */
|
||||
typedef struct
|
||||
{
|
||||
CURL *easy;
|
||||
playht_t* playht;
|
||||
char* body;
|
||||
struct curl_slist *hdr_list;
|
||||
GlobalInfo_t *global;
|
||||
mpg123_handle *mh;
|
||||
char error[CURL_ERROR_SIZE];
|
||||
FILE* file;
|
||||
std::chrono::time_point<std::chrono::high_resolution_clock> startTime;
|
||||
bool flushed;
|
||||
} ConnInfo_t;
|
||||
|
||||
|
||||
static boost::object_pool<ConnInfo_t> pool ;
|
||||
static std::map<curl_socket_t, boost::asio::ip::tcp::socket *> socket_map;
|
||||
static boost::asio::io_service io_service;
|
||||
static boost::asio::deadline_timer timer(io_service);
|
||||
static std::string fullDirPath;
|
||||
static std::thread worker_thread;
|
||||
|
||||
std::string secondsToMillisecondsString(double seconds) {
|
||||
// Convert to milliseconds
|
||||
double milliseconds = seconds * 1000.0;
|
||||
|
||||
// Truncate to remove fractional part
|
||||
long milliseconds_long = static_cast<long>(milliseconds);
|
||||
|
||||
// Convert to string
|
||||
return std::to_string(milliseconds_long);
|
||||
}
|
||||
|
||||
static CURL* createEasyHandle(void) {
|
||||
CURL* easy = curl_easy_init();
|
||||
if(!easy) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "curl_easy_init() failed!\n");
|
||||
return nullptr ;
|
||||
}
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(easy, CURLOPT_USERAGENT, "jambonz/0.8.5");
|
||||
|
||||
// set connect timeout to 3 seconds and total timeout to 109 seconds
|
||||
curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
|
||||
curl_easy_setopt(easy, CURLOPT_TIMEOUT, 10L);
|
||||
|
||||
return easy ;
|
||||
}
|
||||
|
||||
static void cleanupConn(ConnInfo_t *conn) {
|
||||
auto p = conn->playht;
|
||||
|
||||
if (conn->mh) {
|
||||
mpg123_close(conn->mh);
|
||||
mpg123_delete(conn->mh);
|
||||
}
|
||||
|
||||
if( conn->hdr_list ) {
|
||||
curl_slist_free_all(conn->hdr_list);
|
||||
conn->hdr_list = nullptr ;
|
||||
}
|
||||
curl_easy_cleanup(conn->easy);
|
||||
|
||||
if (conn->file) {
|
||||
if (fclose(conn->file) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error closing audio cache file\n");
|
||||
}
|
||||
conn->file = nullptr ;
|
||||
}
|
||||
|
||||
p->conn = nullptr ;
|
||||
p->draining = 1;
|
||||
|
||||
memset(conn, 0, sizeof(ConnInfo_t));
|
||||
pool.destroy(conn) ;
|
||||
}
|
||||
|
||||
/* Check for completed transfers, and remove their easy handles */
|
||||
void check_multi_info(GlobalInfo_t *g) {
|
||||
CURLMsg *msg;
|
||||
int msgs_left;
|
||||
ConnInfo_t *conn;
|
||||
CURL *easy;
|
||||
CURLcode res;
|
||||
|
||||
while((msg = curl_multi_info_read(g->multi, &msgs_left))) {
|
||||
if(msg->msg == CURLMSG_DONE) {
|
||||
long response_code;
|
||||
double namelookup=0, connect=0, total=0 ;
|
||||
char *ct = NULL ;
|
||||
|
||||
easy = msg->easy_handle;
|
||||
res = msg->data.result;
|
||||
curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
|
||||
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &response_code);
|
||||
curl_easy_getinfo(easy, CURLINFO_CONTENT_TYPE, &ct);
|
||||
|
||||
curl_easy_getinfo(easy, CURLINFO_NAMELOOKUP_TIME, &namelookup);
|
||||
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connect);
|
||||
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &total);
|
||||
|
||||
auto p = conn->playht;
|
||||
p->response_code = response_code;
|
||||
if (ct) p->ct = strdup(ct);
|
||||
|
||||
std::string name_lookup_ms = secondsToMillisecondsString(namelookup);
|
||||
std::string connect_ms = secondsToMillisecondsString(connect);
|
||||
std::string final_response_time_ms = secondsToMillisecondsString(total);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,
|
||||
"mod_playht_tts: response: %ld, content-type %s,"
|
||||
"dns(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld, "
|
||||
"connect(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld, "
|
||||
"total(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld\n",
|
||||
response_code, ct,
|
||||
(long)(namelookup), (long)(fmod(namelookup, 1.0) * 1000000),
|
||||
(long)(connect), (long)(fmod(connect, 1.0) * 1000000),
|
||||
(long)(total), (long)(fmod(total, 1.0) * 1000000));
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "name lookup time: %s\n", name_lookup_ms.c_str());
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "connect time: %s\n", connect_ms.c_str());
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "final response time: %s\n", final_response_time_ms.c_str());
|
||||
|
||||
p->name_lookup_time_ms = strdup(name_lookup_ms.c_str());
|
||||
p->connect_time_ms = strdup(connect_ms.c_str());
|
||||
p->final_response_time_ms = strdup(final_response_time_ms.c_str());
|
||||
|
||||
curl_multi_remove_handle(g->multi, easy);
|
||||
cleanupConn(conn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int mcode_test(const char *where, CURLMcode code) {
|
||||
if(CURLM_OK != code) {
|
||||
const char *s;
|
||||
switch(code) {
|
||||
case CURLM_CALL_MULTI_PERFORM:
|
||||
s = "CURLM_CALL_MULTI_PERFORM";
|
||||
break;
|
||||
case CURLM_BAD_HANDLE:
|
||||
s = "CURLM_BAD_HANDLE";
|
||||
break;
|
||||
case CURLM_BAD_EASY_HANDLE:
|
||||
s = "CURLM_BAD_EASY_HANDLE";
|
||||
break;
|
||||
case CURLM_OUT_OF_MEMORY:
|
||||
s = "CURLM_OUT_OF_MEMORY";
|
||||
break;
|
||||
case CURLM_INTERNAL_ERROR:
|
||||
s = "CURLM_INTERNAL_ERROR";
|
||||
break;
|
||||
case CURLM_UNKNOWN_OPTION:
|
||||
s = "CURLM_UNKNOWN_OPTION";
|
||||
break;
|
||||
case CURLM_LAST:
|
||||
s = "CURLM_LAST";
|
||||
break;
|
||||
default:
|
||||
s = "CURLM_unknown";
|
||||
break;
|
||||
case CURLM_BAD_SOCKET:
|
||||
s = "CURLM_BAD_SOCKET";
|
||||
break;
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mcode_test ERROR: %s returns %s:%d\n", where, s, code);
|
||||
|
||||
return -1;
|
||||
}
|
||||
return 0 ;
|
||||
}
|
||||
|
||||
static void remsock(int *f, GlobalInfo_t *g) {
|
||||
if(f) {
|
||||
free(f);
|
||||
f = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/* Called by asio when there is an action on a socket */
|
||||
static void event_cb(GlobalInfo_t *g, curl_socket_t s, int action, const boost::system::error_code & error, int *fdp) {
|
||||
int f = *fdp;
|
||||
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X has action %d\n", s, action) ;
|
||||
|
||||
// Socket already POOL REMOVED.
|
||||
if (f == CURL_POLL_REMOVE) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X removed\n", s);
|
||||
remsock(fdp, g);
|
||||
return;
|
||||
}
|
||||
|
||||
if(socket_map.find(s) == socket_map.end()) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb: socket %#X already closed\n, s");
|
||||
return;
|
||||
}
|
||||
|
||||
/* make sure the event matches what are wanted */
|
||||
if(f == action || f == CURL_POLL_INOUT) {
|
||||
if(error) {
|
||||
action = CURL_CSELECT_ERR;
|
||||
}
|
||||
CURLMcode rc = curl_multi_socket_action(g->multi, s, action, &g->still_running);
|
||||
|
||||
mcode_test("event_cb: curl_multi_socket_action", rc);
|
||||
check_multi_info(g);
|
||||
|
||||
if(g->still_running <= 0) {
|
||||
timer.cancel();
|
||||
}
|
||||
|
||||
/* keep on watching.
|
||||
* the socket may have been closed and/or fdp may have been changed
|
||||
* in curl_multi_socket_action(), so check them both */
|
||||
if(!error && socket_map.find(s) != socket_map.end() &&
|
||||
(f == action || f == CURL_POLL_INOUT)) {
|
||||
boost::asio::ip::tcp::socket *tcp_socket = socket_map.find(s)->second;
|
||||
|
||||
if(action == CURL_POLL_IN) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
action, boost::placeholders::_1, fdp));
|
||||
}
|
||||
if(action == CURL_POLL_OUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
action, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* socket functions */
|
||||
static void setsock(int *fdp, curl_socket_t s, CURL *e, int act, int oldact, GlobalInfo_t *g) {
|
||||
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(s);
|
||||
|
||||
if(it == socket_map.end()) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setsock: socket %#X not found\n, s");
|
||||
return;
|
||||
}
|
||||
|
||||
boost::asio::ip::tcp::socket * tcp_socket = it->second;
|
||||
|
||||
*fdp = act;
|
||||
|
||||
if(act == CURL_POLL_IN) {
|
||||
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_IN, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
else if(act == CURL_POLL_OUT) {
|
||||
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_OUT, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
else if(act == CURL_POLL_INOUT) {
|
||||
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_IN, boost::placeholders::_1, fdp));
|
||||
}
|
||||
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_OUT, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo_t *g) {
|
||||
/* fdp is used to store current action */
|
||||
int *fdp = (int *) calloc(sizeof(int), 1);
|
||||
|
||||
setsock(fdp, s, easy, action, 0, g);
|
||||
curl_multi_assign(g->multi, s, fdp);
|
||||
}
|
||||
|
||||
static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) {
|
||||
GlobalInfo_t *g = &global;
|
||||
|
||||
int *actionp = (int *) sockp;
|
||||
static const char *whatstr[] = { "none", "IN", "OUT", "INOUT", "REMOVE"};
|
||||
|
||||
if(what == CURL_POLL_REMOVE) {
|
||||
*actionp = what;
|
||||
}
|
||||
else {
|
||||
if(!actionp) {
|
||||
addsock(s, e, what, g);
|
||||
}
|
||||
else {
|
||||
setsock(actionp, s, e, what, *actionp, g);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void threadFunc() {
|
||||
/* to make sure the event loop doesn't terminate when there is no work to do */
|
||||
io_service.reset() ;
|
||||
boost::asio::io_service::work work(io_service);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_playht_tts threadFunc - starting\n");
|
||||
|
||||
for(;;) {
|
||||
|
||||
try {
|
||||
io_service.run() ;
|
||||
break ;
|
||||
}
|
||||
catch( std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mod_playht_tts threadFunc - Error: %s\n", e.what());
|
||||
}
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_playht_tts threadFunc - ending\n");
|
||||
}
|
||||
|
||||
|
||||
/* Called by asio when our timeout expires */
|
||||
static void timer_cb(const boost::system::error_code & error, GlobalInfo_t *g)
|
||||
{
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "timer_cb\n");
|
||||
|
||||
if(!error) {
|
||||
CURLMcode rc = curl_multi_socket_action(g->multi, CURL_SOCKET_TIMEOUT, 0, &g->still_running);
|
||||
mcode_test("timer_cb: curl_multi_socket_action", rc);
|
||||
check_multi_info(g);
|
||||
}
|
||||
}
|
||||
|
||||
int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo_t *g) {
|
||||
|
||||
/* cancel running timer */
|
||||
timer.cancel();
|
||||
|
||||
if(timeout_ms >= 0) {
|
||||
// from libcurl 7.88.1-10+deb12u4 does not allow call curl_multi_socket_action or curl_multi_perform in curl_multi callback directly
|
||||
timer.expires_from_now(boost::posix_time::millisec(timeout_ms ? timeout_ms : 1));
|
||||
timer.async_wait(boost::bind(&timer_cb, boost::placeholders::_1, g));
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static std::vector<uint16_t> convert_mp3_to_linear(ConnInfo_t *conn, uint8_t *data, size_t len) {
|
||||
std::vector<uint16_t> linear_data;
|
||||
int eof = 0;
|
||||
int mp3err = 0;
|
||||
unsigned char decode_buf[MP3_DCACHE];
|
||||
|
||||
if(mpg123_feed(conn->mh, data, len) == MPG123_OK) {
|
||||
while(!eof) {
|
||||
size_t usedlen = 0;
|
||||
off_t frame_offset;
|
||||
unsigned char* audio;
|
||||
|
||||
int decode_status = mpg123_decode_frame(conn->mh, &frame_offset, &audio, &usedlen);
|
||||
|
||||
switch(decode_status) {
|
||||
case MPG123_NEW_FORMAT:
|
||||
continue;
|
||||
|
||||
case MPG123_OK:
|
||||
for(size_t i = 0; i < usedlen; i += 2) {
|
||||
uint16_t value = reinterpret_cast<uint16_t*>(audio)[i / 2];
|
||||
linear_data.push_back(value);
|
||||
}
|
||||
break;
|
||||
|
||||
case MPG123_DONE:
|
||||
case MPG123_NEED_MORE:
|
||||
eof = 1;
|
||||
break;
|
||||
|
||||
case MPG123_ERR:
|
||||
default:
|
||||
if(++mp3err >= 5) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Decoder Error!\n");
|
||||
eof = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (eof)
|
||||
break;
|
||||
|
||||
mp3err = 0;
|
||||
}
|
||||
}
|
||||
|
||||
return linear_data;
|
||||
}
|
||||
/* CURLOPT_WRITEFUNCTION */
|
||||
static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
|
||||
bool fireEvent = false;
|
||||
uint8_t *data = (uint8_t *) ptr;
|
||||
size_t bytes_received = size * nmemb;
|
||||
auto p = conn->playht;
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) p->circularBuffer;
|
||||
std::vector<uint16_t> pcm_data;
|
||||
|
||||
if (conn->flushed || cBuffer == nullptr) {
|
||||
/* this will abort the transfer */
|
||||
return 0;
|
||||
}
|
||||
{
|
||||
switch_mutex_lock(p->mutex);
|
||||
|
||||
if (p->response_code > 0 && p->response_code != 200) {
|
||||
std::string body((char *) ptr, bytes_received);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: received body %s\n", body.c_str());
|
||||
p->err_msg = strdup(body.c_str());
|
||||
switch_mutex_unlock(p->mutex);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* cache file will stay in the mp3 format for size (smaller) and simplicity */
|
||||
if (conn->file) fwrite(data, sizeof(uint8_t), bytes_received, conn->file);
|
||||
|
||||
pcm_data = convert_mp3_to_linear(conn, data, bytes_received);
|
||||
size_t bytesResampled = pcm_data.size() * sizeof(uint16_t);
|
||||
|
||||
// Resize the buffer if necessary
|
||||
if (cBuffer->capacity() - cBuffer->size() < (bytesResampled / sizeof(uint16_t))) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "write_cb growing buffer\n");
|
||||
|
||||
//TODO: if buffer exceeds some max size, return CURL_WRITEFUNC_ERROR to abort the transfer
|
||||
cBuffer->set_capacity(cBuffer->size() + std::max((bytesResampled / sizeof(uint16_t)), (size_t)BUFFER_GROW_SIZE));
|
||||
}
|
||||
|
||||
/* Push the data into the buffer */
|
||||
cBuffer->insert(cBuffer->end(), pcm_data.data(), pcm_data.data() + pcm_data.size());
|
||||
|
||||
if (0 == p->reads++) {
|
||||
fireEvent = true;
|
||||
}
|
||||
switch_mutex_unlock(p->mutex);
|
||||
}
|
||||
if (fireEvent && p->session_id) {
|
||||
auto endTime = std::chrono::high_resolution_clock::now();
|
||||
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(endTime - conn->startTime);
|
||||
auto time_to_first_byte_ms = std::to_string(duration.count());
|
||||
switch_core_session_t* session = switch_core_session_locate(p->session_id);
|
||||
if (session) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_core_session_rwunlock(session);
|
||||
if (channel) {
|
||||
switch_event_t *event;
|
||||
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_START) == SWITCH_STATUS_SUCCESS) {
|
||||
switch_channel_event_set_data(channel, event);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: firing playback-started\n");
|
||||
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
|
||||
if (p->request_id) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_playht_request_id", p->request_id);
|
||||
}
|
||||
if (p->name_lookup_time_ms) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_playht_name_lookup_time_ms", p->name_lookup_time_ms);
|
||||
}
|
||||
if (p->connect_time_ms) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_playht_connect_time_ms", p->connect_time_ms);
|
||||
}
|
||||
if (p->final_response_time_ms) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_playht_final_response_time_ms", p->final_response_time_ms);
|
||||
}
|
||||
if (p->voice_name) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_playht_voice_name", p->voice_name);
|
||||
}
|
||||
if (p->cache_filename) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", p->cache_filename);
|
||||
}
|
||||
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
|
||||
switch_event_fire(&event);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: session %s not found\n", p->session_id);
|
||||
}
|
||||
}
|
||||
return bytes_received;
|
||||
}
|
||||
|
||||
static bool parseHeader(const std::string& str, std::string& header, std::string& value) {
|
||||
std::vector<std::string> parts;
|
||||
boost::split(parts, str, boost::is_any_of(":"), boost::token_compress_on);
|
||||
|
||||
if (parts.size() != 2)
|
||||
return false;
|
||||
|
||||
header = boost::trim_copy(parts[0]);
|
||||
value = boost::trim_copy(parts[1]);
|
||||
return true;
|
||||
}
|
||||
|
||||
static int extract_response_code(const std::string& input) {
|
||||
std::size_t space_pos = input.find(' ');
|
||||
if (space_pos == std::string::npos) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid HTTP response format %s\n", input.c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::size_t code_start_pos = space_pos + 1;
|
||||
std::size_t code_end_pos = input.find(' ', code_start_pos);
|
||||
if (code_end_pos == std::string::npos) {
|
||||
code_end_pos = input.length();
|
||||
}
|
||||
|
||||
std::string code_str = input.substr(code_start_pos, code_end_pos - code_start_pos);
|
||||
int response_code = std::stoi(code_str);
|
||||
return response_code;
|
||||
}
|
||||
|
||||
static size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo_t *conn) {
|
||||
size_t bytes_received = size * nitems;
|
||||
const std::string prefix = "HTTP/";
|
||||
playht_t* p = conn->playht;
|
||||
std::string header, value;
|
||||
std::string input(buffer, bytes_received);
|
||||
if (parseHeader(input, header, value)) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s with value %s\n", header.c_str(), value.c_str());
|
||||
if (0 == header.compare("x-job-ids")) p->request_id = strdup(value.c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: %s\n", input.c_str());
|
||||
if (input.rfind(prefix, 0) == 0) {
|
||||
try {
|
||||
p->response_code = extract_response_code(input);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: parsed response code: %ld\n", p->response_code);
|
||||
} catch (const std::invalid_argument& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "header_callback: invalid response code %s\n", input.substr(prefix.length()).c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
return bytes_received;
|
||||
}
|
||||
|
||||
/* CURLOPT_OPENSOCKETFUNCTION */
|
||||
static curl_socket_t opensocket(void *clientp, curlsocktype purpose, struct curl_sockaddr *address) {
|
||||
curl_socket_t sockfd = CURL_SOCKET_BAD;
|
||||
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "opensocket: %d\n", purpose);
|
||||
/* restrict to IPv4 */
|
||||
if(purpose == CURLSOCKTYPE_IPCXN && address->family == AF_INET) {
|
||||
/* create a tcp socket object */
|
||||
boost::asio::ip::tcp::socket *tcp_socket = new boost::asio::ip::tcp::socket(io_service);
|
||||
|
||||
/* open it and get the native handle*/
|
||||
boost::system::error_code ec;
|
||||
tcp_socket->open(boost::asio::ip::tcp::v4(), ec);
|
||||
|
||||
if(ec) {
|
||||
/* An error occurred */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't open socket [%ld][%s]\n", ec, ec.message().c_str());
|
||||
}
|
||||
else {
|
||||
sockfd = tcp_socket->native_handle();
|
||||
|
||||
/* save it for monitoring */
|
||||
socket_map.insert(std::pair<curl_socket_t, boost::asio::ip::tcp::socket *>(sockfd, tcp_socket));
|
||||
}
|
||||
}
|
||||
return sockfd;
|
||||
}
|
||||
|
||||
/* CURLOPT_CLOSESOCKETFUNCTION */
|
||||
static int close_socket(void *clientp, curl_socket_t item) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "close_socket : %#X\n", item);
|
||||
|
||||
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(item);
|
||||
if(it != socket_map.end()) {
|
||||
delete it->second;
|
||||
socket_map.erase(it);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
extern "C" {
|
||||
switch_status_t playht_speech_load() {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "playht_speech_loading..\n");
|
||||
memset(&global, 0, sizeof(GlobalInfo_t));
|
||||
global.multi = curl_multi_init();
|
||||
|
||||
if (!global.multi) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "playht_speech_load curl_multi_init() failed, exiting!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
curl_multi_setopt(global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_SOCKETDATA, &global);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_TIMERDATA, &global);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
||||
|
||||
/* create temp folder for cache files */
|
||||
const char* baseDir = std::getenv("JAMBONZ_TMP_CACHE_FOLDER");
|
||||
if (!baseDir) {
|
||||
baseDir = "/tmp/";
|
||||
}
|
||||
if (strcmp(baseDir, "/") == 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", baseDir);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
fullDirPath = std::string(baseDir) + "tts-cache-files";
|
||||
|
||||
// Create the directory with read, write, and execute permissions for everyone
|
||||
mode_t oldMask = umask(0);
|
||||
int result = mkdir(fullDirPath.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
|
||||
umask(oldMask);
|
||||
if (result != 0) {
|
||||
if (errno != EEXIST) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", fullDirPath.c_str());
|
||||
fullDirPath = "";
|
||||
}
|
||||
else switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "folder %s already exists\n", fullDirPath.c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "created folder %s\n", fullDirPath.c_str());
|
||||
}
|
||||
// init mgp123
|
||||
if (mpg123_init() != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to initiate MPG123");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
/* start worker thread that handles transfers*/
|
||||
std::thread t(threadFunc) ;
|
||||
worker_thread.swap( t ) ;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "playht_speech_loaded..\n");
|
||||
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t playht_speech_unload() {
|
||||
/* stop the ASIO IO service */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "playht_speech_unload: stopping io service\n");
|
||||
io_service.stop();
|
||||
|
||||
/* Join the worker thread */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "playht_speech_unload: wait for worker thread to complete\n");
|
||||
if (worker_thread.joinable()) {
|
||||
worker_thread.join();
|
||||
}
|
||||
|
||||
/* cleanup curl multi handle*/
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "playht_speech_unload: release curl multi\n");
|
||||
curl_multi_cleanup(global.multi);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "playht_speech_unload: completed\n");
|
||||
|
||||
mpg123_exit();
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t playht_speech_open(playht_t* playht) {
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t playht_speech_feed_tts(playht_t* p, char* text, switch_speech_flag_t *flags) {
|
||||
CURLMcode rc;
|
||||
|
||||
const int MAX_CHARS = 20;
|
||||
char tempText[MAX_CHARS + 4]; // +4 for the ellipsis and null terminator
|
||||
|
||||
if (strlen(text) > MAX_CHARS) {
|
||||
strncpy(tempText, text, MAX_CHARS);
|
||||
strcpy(tempText + MAX_CHARS, "...");
|
||||
} else {
|
||||
strcpy(tempText, text);
|
||||
}
|
||||
|
||||
/* open cache file */
|
||||
if (p->cache_audio && fullDirPath.length() > 0) {
|
||||
switch_uuid_t uuid;
|
||||
char uuid_str[SWITCH_UUID_FORMATTED_LENGTH + 1];
|
||||
char outfile[512] = "";
|
||||
int fd;
|
||||
|
||||
switch_uuid_get(&uuid);
|
||||
switch_uuid_format(uuid_str, &uuid);
|
||||
|
||||
switch_snprintf(outfile, sizeof(outfile), "%s%s%s.mp3", fullDirPath.c_str(), SWITCH_PATH_SEPARATOR, uuid_str);
|
||||
p->cache_filename = strdup(outfile);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "writing audio cache file to %s\n", p->cache_filename);
|
||||
|
||||
mode_t oldMask = umask(0);
|
||||
fd = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
|
||||
umask(oldMask);
|
||||
if (fd == -1 ) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
|
||||
}
|
||||
else {
|
||||
p->file = fdopen(fd, "wb");
|
||||
if (!p->file) {
|
||||
close(fd);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!p->api_key) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "playht_speech_feed_tts: no api_key provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (!p->user_id) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "playht_speech_feed_tts: no user_id provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
/* format url*/
|
||||
std::string url = "https://api.play.ht/api/v2/tts/stream";
|
||||
|
||||
/* create the JSON body */
|
||||
cJSON * jResult = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(jResult, "text", text);
|
||||
cJSON_AddStringToObject(jResult, "voice", p->voice_name);
|
||||
cJSON_AddStringToObject(jResult, "output_format", "mp3");
|
||||
cJSON_AddNumberToObject(jResult, "sample_rate", 8000);
|
||||
if (p->voice_engine) {
|
||||
cJSON_AddStringToObject(jResult, "voice_engine", p->voice_engine);
|
||||
}
|
||||
if (p->quality) {
|
||||
cJSON_AddStringToObject(jResult, "quality", p->quality);
|
||||
}
|
||||
if (p->speed) {
|
||||
cJSON_AddNumberToObject(jResult, "speed", atoi(p->speed));
|
||||
}
|
||||
if (p->seed) {
|
||||
cJSON_AddNumberToObject(jResult, "seed", atoi(p->seed));
|
||||
}
|
||||
if (p->temperature) {
|
||||
cJSON_AddNumberToObject(jResult, "temperature", std::strtof(p->temperature, nullptr));
|
||||
}
|
||||
if (p->emotion) {
|
||||
cJSON_AddStringToObject(jResult, "emotion", p->emotion);
|
||||
}
|
||||
if (p->voice_guidance) {
|
||||
cJSON_AddNumberToObject(jResult, "voice_guidance", atoi(p->voice_guidance));
|
||||
}
|
||||
if (p->style_guidance) {
|
||||
cJSON_AddNumberToObject(jResult, "style_guidance", atoi(p->style_guidance));
|
||||
}
|
||||
if (p->text_guidance) {
|
||||
cJSON_AddNumberToObject(jResult, "text_guidance", atoi(p->text_guidance));
|
||||
}
|
||||
char *json = cJSON_PrintUnformatted(jResult);
|
||||
|
||||
cJSON_Delete(jResult);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "playht_speech_feed_tts: [%s] [%s]\n", url.c_str(), tempText);
|
||||
|
||||
ConnInfo_t *conn = pool.malloc() ;
|
||||
|
||||
// COnfigure MPG123
|
||||
int mhError = 0;
|
||||
mpg123_handle *mh = mpg123_new("auto", &mhError);
|
||||
if (!mh) {
|
||||
const char *mhErr = mpg123_plain_strerror(mhError);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error allocating mpg123 handle! %s\n", switch_str_nil(mhErr));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (mpg123_open_feed(mh) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_open_feed!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (mpg123_format_all(mh) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_format_all!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
if (mpg123_param(mh, MPG123_FLAGS, MPG123_MONO_MIX, 0) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error forcing single channel!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
CURL* easy = createEasyHandle();
|
||||
p->conn = (void *) conn ;
|
||||
conn->playht = p;
|
||||
conn->easy = easy;
|
||||
conn->mh = mh;
|
||||
conn->global = &global;
|
||||
conn->hdr_list = NULL ;
|
||||
conn->file = p->file;
|
||||
conn->body = json;
|
||||
conn->flushed = false;
|
||||
|
||||
|
||||
p->circularBuffer = (void *) new CircularBuffer_t(8192);
|
||||
|
||||
if (mpg123_param(mh, MPG123_FORCE_RATE, p->rate /*Hz*/, 0) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_param!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
std::ostringstream api_key_stream;
|
||||
api_key_stream << "AUTHORIZATION: " << p->api_key;
|
||||
std::ostringstream user_id_stream;
|
||||
user_id_stream << "X-USER-ID: " << p->user_id;
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, write_cb);
|
||||
curl_easy_setopt(easy, CURLOPT_WRITEDATA, conn);
|
||||
curl_easy_setopt(easy, CURLOPT_ERRORBUFFER, conn->error);
|
||||
curl_easy_setopt(easy, CURLOPT_PRIVATE, conn);
|
||||
curl_easy_setopt(easy, CURLOPT_VERBOSE, 0L);
|
||||
curl_easy_setopt(easy, CURLOPT_NOPROGRESS, 1L);
|
||||
curl_easy_setopt(easy, CURLOPT_HEADERFUNCTION, header_callback);
|
||||
curl_easy_setopt(easy, CURLOPT_HEADERDATA, conn);
|
||||
|
||||
/* call this function to get a socket */
|
||||
curl_easy_setopt(easy, CURLOPT_OPENSOCKETFUNCTION, opensocket);
|
||||
|
||||
/* call this function to close a socket */
|
||||
curl_easy_setopt(easy, CURLOPT_CLOSESOCKETFUNCTION, close_socket);
|
||||
|
||||
conn->hdr_list = curl_slist_append(conn->hdr_list, api_key_stream.str().c_str());
|
||||
conn->hdr_list = curl_slist_append(conn->hdr_list, user_id_stream.str().c_str());
|
||||
conn->hdr_list = curl_slist_append(conn->hdr_list, "Accept: audio/mpeg");
|
||||
conn->hdr_list = curl_slist_append(conn->hdr_list, "Content-Type: application/json");
|
||||
curl_easy_setopt(easy, CURLOPT_HTTPHEADER, conn->hdr_list);
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
|
||||
//curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
|
||||
|
||||
// libcurl adding random byte to the response body that creates white noise to audio file
|
||||
// https://github.com/curl/curl/issues/10525
|
||||
const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
|
||||
curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
|
||||
|
||||
rc = curl_multi_add_handle(global.multi, conn->easy);
|
||||
mcode_test("new_conn: curl_multi_add_handle", rc);
|
||||
|
||||
/* start a timer to measure the duration until we receive first byte of audio */
|
||||
conn->startTime = std::chrono::high_resolution_clock::now();
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "playht_speech_feed_tts: called curl_multi_add_handle\n");
|
||||
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t playht_speech_read_tts(playht_t* p, void *data, size_t *datalen, switch_speech_flag_t *flags) {
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) p->circularBuffer;
|
||||
std::vector<uint16_t> pcm_data;
|
||||
|
||||
{
|
||||
switch_mutex_lock(p->mutex);
|
||||
ConnInfo_t *conn = (ConnInfo_t *) p->conn;
|
||||
if (p->response_code > 0 && p->response_code != 200) {
|
||||
switch_mutex_unlock(p->mutex);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "playht_speech_read_tts, returning failure\n") ;
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (conn && conn->flushed) {
|
||||
switch_mutex_unlock(p->mutex);
|
||||
return SWITCH_STATUS_BREAK;
|
||||
}
|
||||
if (cBuffer->empty()) {
|
||||
if (p->draining) {
|
||||
switch_mutex_unlock(p->mutex);
|
||||
return SWITCH_STATUS_BREAK;
|
||||
}
|
||||
/* no audio available yet so send silence */
|
||||
memset(data, 255, *datalen);
|
||||
switch_mutex_unlock(p->mutex);
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
size_t size = std::min((*datalen/2), cBuffer->size());
|
||||
pcm_data.insert(pcm_data.end(), cBuffer->begin(), cBuffer->begin() + size);
|
||||
cBuffer->erase(cBuffer->begin(), cBuffer->begin() + size);
|
||||
switch_mutex_unlock(p->mutex);
|
||||
}
|
||||
|
||||
memcpy(data, pcm_data.data(), pcm_data.size() * sizeof(uint16_t));
|
||||
*datalen = pcm_data.size() * sizeof(uint16_t);
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t playht_speech_flush_tts(playht_t* p) {
|
||||
bool download_complete = p->response_code == 200;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "playht_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;
|
||||
ConnInfo_t *conn = (ConnInfo_t *) p->conn;
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) p->circularBuffer;
|
||||
delete cBuffer;
|
||||
p->circularBuffer = nullptr ;
|
||||
|
||||
if (conn) {
|
||||
conn->flushed = true;
|
||||
if (!download_complete) {
|
||||
if (conn->file) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "closing audio cache file %s because download was interrupted\n", p->cache_filename);
|
||||
if (fclose(conn->file) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "error closing audio cache file\n");
|
||||
}
|
||||
conn->file = nullptr ;
|
||||
}
|
||||
|
||||
if (p->cache_filename) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", p->cache_filename);
|
||||
if (unlink(p->cache_filename) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n",
|
||||
p->cache_filename, errno, strerror(errno));
|
||||
}
|
||||
free(p->cache_filename);
|
||||
p->cache_filename = nullptr ;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (p->session_id) {
|
||||
switch_core_session_t* session = switch_core_session_locate(p->session_id);
|
||||
if (session) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_core_session_rwunlock(session);
|
||||
if (channel) {
|
||||
switch_event_t *event;
|
||||
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_STOP) == SWITCH_STATUS_SUCCESS) {
|
||||
switch_channel_event_set_data(channel, event);
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_playht_response_code", std::to_string(p->response_code).c_str());
|
||||
if (p->cache_filename && p->response_code == 200) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", p->cache_filename);
|
||||
}
|
||||
if (p->response_code != 200 && p->err_msg) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_error", p->err_msg);
|
||||
}
|
||||
switch_event_fire(&event);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t playht_speech_close(playht_t* p) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "playht_speech_close\n") ;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#ifndef __PLAYHT_GLUE_H__
|
||||
#define __PLAYHT_GLUE_H__
|
||||
|
||||
switch_status_t playht_speech_load();
|
||||
switch_status_t playht_speech_open(playht_t* playht);
|
||||
switch_status_t playht_speech_feed_tts(playht_t* playht, char* text, switch_speech_flag_t *flags);
|
||||
switch_status_t playht_speech_read_tts(playht_t* playht, void *data, size_t *datalen, switch_speech_flag_t *flags);
|
||||
switch_status_t playht_speech_flush_tts(playht_t* playht);
|
||||
switch_status_t playht_speech_close(playht_t* playht);
|
||||
switch_status_t playht_speech_unload();
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,8 @@
|
||||
Copyright 2023, Drachtio Communications Services, LLC
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
include $(top_srcdir)/build/modmake.rulesam
|
||||
MODNAME=mod_rimelabs_tts
|
||||
|
||||
mod_LTLIBRARIES = mod_rimelabs_tts.la
|
||||
mod_rimelabs_tts_la_SOURCES = mod_rimelabs_tts.c rimelabs_glue.cpp
|
||||
mod_rimelabs_tts_la_CFLAGS = $(AM_CFLAGS)
|
||||
mod_rimelabs_tts_la_LIBADD = $(switch_builddir)/libfreeswitch.la
|
||||
mod_rimelabs_tts_la_LDFLAGS = -avoid-version -module -no-undefined -shared -lstdc++ -lboost_system -lboost_thread
|
||||
@@ -0,0 +1,3 @@
|
||||
# mod_rimelabs_tts
|
||||
|
||||
A Freeswitch module that allows speak text to speech audio from rimelabs stream.
|
||||
@@ -0,0 +1,167 @@
|
||||
#include "mod_rimelabs_tts.h"
|
||||
#include "rimelabs_glue.h"
|
||||
|
||||
SWITCH_MODULE_LOAD_FUNCTION(mod_rimelabs_tts_load);
|
||||
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_rimelabs_tts_shutdown);
|
||||
SWITCH_MODULE_DEFINITION(mod_rimelabs_tts, mod_rimelabs_tts_load, mod_rimelabs_tts_shutdown, NULL);
|
||||
|
||||
static void clearrimelabs(rimelabs_t* d, int freeAll) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "clearrimelabs\n");
|
||||
if (d->api_key) free(d->api_key);
|
||||
if (d->model_id) free(d->model_id);
|
||||
if (d->speed_alpha) free(d->speed_alpha);
|
||||
if (d->reduce_latency) free(d->reduce_latency);
|
||||
|
||||
if (d->ct) free(d->ct);
|
||||
if (d->err_msg) free(d->err_msg);
|
||||
if (d->name_lookup_time_ms) free(d->name_lookup_time_ms);
|
||||
if (d->connect_time_ms) free(d->connect_time_ms);
|
||||
if (d->final_response_time_ms) free(d->final_response_time_ms);
|
||||
if (d->cache_filename) free(d->cache_filename);
|
||||
|
||||
|
||||
d->api_key = NULL;
|
||||
d->model_id = NULL;
|
||||
d->speed_alpha = NULL;
|
||||
d->reduce_latency = NULL;
|
||||
|
||||
d->ct = NULL;
|
||||
d->err_msg = NULL;
|
||||
d->name_lookup_time_ms = NULL;
|
||||
d->connect_time_ms = NULL;
|
||||
d->final_response_time_ms = NULL;
|
||||
d->cache_filename = NULL;
|
||||
|
||||
if (freeAll) {
|
||||
if (d->voice_name) free(d->voice_name);
|
||||
if (d->session_id) free(d->session_id);
|
||||
d->voice_name = NULL;
|
||||
d->session_id = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
static rimelabs_t * createOrRetrievePrivateData(switch_speech_handle_t *sh) {
|
||||
rimelabs_t *d = (rimelabs_t *) sh->private_info;
|
||||
if (!d) {
|
||||
d = switch_core_alloc(sh->memory_pool, sizeof(*d));
|
||||
sh->private_info = d;
|
||||
memset(d, 0, sizeof(*d));
|
||||
switch_mutex_init(&d->mutex, SWITCH_MUTEX_NESTED, sh->memory_pool);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "allocated rimelabs_t\n");
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
switch_status_t d_speech_open(switch_speech_handle_t *sh, const char *voice_name, int rate, int channels, switch_speech_flag_t *flags)
|
||||
{
|
||||
rimelabs_t *d = createOrRetrievePrivateData(sh);
|
||||
d->voice_name = strdup(voice_name);
|
||||
d->rate = rate;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_open voice: %s, rate %d, channels %d\n", voice_name, rate, channels);
|
||||
return rimelabs_speech_open(d);
|
||||
}
|
||||
|
||||
static switch_status_t d_speech_close(switch_speech_handle_t *sh, switch_speech_flag_t *flags)
|
||||
{
|
||||
switch_status_t rc;
|
||||
rimelabs_t *d = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_close\n");
|
||||
|
||||
switch_mutex_destroy(d->mutex);
|
||||
|
||||
rc = rimelabs_speech_close(d);
|
||||
clearrimelabs(d, 1);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeswitch will call this function to feed us text to speak
|
||||
*/
|
||||
static switch_status_t d_speech_feed_tts(switch_speech_handle_t *sh, char *text, switch_speech_flag_t *flags)
|
||||
{
|
||||
rimelabs_t *d = createOrRetrievePrivateData(sh);
|
||||
d->draining = 0;
|
||||
d->reads = 0;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_feed_tts\n");
|
||||
|
||||
return rimelabs_speech_feed_tts(d, text, flags);
|
||||
}
|
||||
|
||||
/**
|
||||
* Freeswitch calls periodically to get some rendered audio in L16 format. We can provide up to 8k of audio at a time.
|
||||
*/
|
||||
static switch_status_t d_speech_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, switch_speech_flag_t *flags)
|
||||
{
|
||||
rimelabs_t *d = createOrRetrievePrivateData(sh);
|
||||
return rimelabs_speech_read_tts(d, data, datalen, flags);
|
||||
}
|
||||
|
||||
/**
|
||||
* This is called at the end, not sure exactly what we need to do here..
|
||||
*/
|
||||
static void d_speech_flush_tts(switch_speech_handle_t *sh)
|
||||
{
|
||||
rimelabs_t *d = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_speech_flush_tts\n");
|
||||
rimelabs_speech_flush_tts(d);
|
||||
|
||||
clearrimelabs(d, 0);
|
||||
}
|
||||
|
||||
static void d_text_param_tts(switch_speech_handle_t *sh, char *param, const char *val)
|
||||
{
|
||||
rimelabs_t *d = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "d_text_param_tts: %s=%s\n", param, val);
|
||||
if (0 == strcmp(param, "api_key")) {
|
||||
if (d->api_key) free(d->api_key);
|
||||
d->api_key = strdup(val);
|
||||
} else if (0 == strcmp(param, "model_id")) {
|
||||
if (d->model_id) free(d->model_id);
|
||||
d->model_id = strdup(val);
|
||||
} else if (0 == strcmp(param, "speed_alpha")) {
|
||||
if (d->speed_alpha) free(d->speed_alpha);
|
||||
d->speed_alpha = strdup(val);
|
||||
} else if (0 == strcmp(param, "reduce_latency")) {
|
||||
if (d->reduce_latency) free(d->reduce_latency);
|
||||
d->reduce_latency = strdup(val);
|
||||
} else if (0 == strcmp(param, "voice")) {
|
||||
if (d->voice_name) free(d->voice_name);
|
||||
d->voice_name = strdup(val);
|
||||
} else if (0 == strcmp(param, "session-uuid")) {
|
||||
if (d->session_id) free(d->session_id);
|
||||
d->session_id = strdup(val);
|
||||
} else if (0 == strcmp(param, "write_cache_file") && switch_true(val)) {
|
||||
d->cache_audio = 1;
|
||||
}
|
||||
}
|
||||
|
||||
static void d_numeric_param_tts(switch_speech_handle_t *sh, char *param, int val)
|
||||
{
|
||||
}
|
||||
static void d_float_param_tts(switch_speech_handle_t *sh, char *param, double val)
|
||||
{
|
||||
}
|
||||
|
||||
SWITCH_MODULE_LOAD_FUNCTION(mod_rimelabs_tts_load)
|
||||
{
|
||||
switch_speech_interface_t *speech_interface;
|
||||
|
||||
*module_interface = switch_loadable_module_create_module_interface(pool, modname);
|
||||
speech_interface = switch_loadable_module_create_interface(*module_interface, SWITCH_SPEECH_INTERFACE);
|
||||
speech_interface->interface_name = "rimelabs";
|
||||
speech_interface->speech_open = d_speech_open;
|
||||
speech_interface->speech_close = d_speech_close;
|
||||
speech_interface->speech_feed_tts = d_speech_feed_tts;
|
||||
speech_interface->speech_read_tts = d_speech_read_tts;
|
||||
speech_interface->speech_flush_tts = d_speech_flush_tts;
|
||||
speech_interface->speech_text_param_tts = d_text_param_tts;
|
||||
speech_interface->speech_numeric_param_tts = d_numeric_param_tts;
|
||||
speech_interface->speech_float_param_tts = d_float_param_tts;
|
||||
return rimelabs_speech_load();
|
||||
}
|
||||
|
||||
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_rimelabs_tts_shutdown)
|
||||
{
|
||||
return rimelabs_speech_unload();
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
#ifndef __MOD_RIMELABS_TTS_H__
|
||||
#define __MOD_RIMELABS_TTS_H__
|
||||
|
||||
#include <switch.h>
|
||||
#include <speex/speex_resampler.h>
|
||||
|
||||
typedef struct rimelabs_data {
|
||||
char *voice_name;
|
||||
char *api_key;
|
||||
char *model_id;
|
||||
char *speed_alpha;
|
||||
char *reduce_latency;
|
||||
|
||||
/* result data */
|
||||
long response_code;
|
||||
char *ct;
|
||||
char *name_lookup_time_ms;
|
||||
char *connect_time_ms;
|
||||
char *final_response_time_ms;
|
||||
char *err_msg;
|
||||
char *cache_filename;
|
||||
char *session_id;
|
||||
|
||||
int rate;
|
||||
int draining;
|
||||
int reads;
|
||||
int cache_audio;
|
||||
|
||||
void *conn;
|
||||
void *circularBuffer;
|
||||
switch_mutex_t *mutex;
|
||||
FILE *file;
|
||||
SpeexResamplerState *resampler;
|
||||
} rimelabs_t;
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,953 @@
|
||||
#include "mod_rimelabs_tts.h"
|
||||
#include <switch.h>
|
||||
#include <switch_json.h>
|
||||
#include <curl/curl.h>
|
||||
#include <cstdlib>
|
||||
#include <sstream>
|
||||
|
||||
#include <boost/circular_buffer.hpp>
|
||||
#include <boost/thread.hpp>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/asio/ssl.hpp>
|
||||
#include <boost/pool/object_pool.hpp>
|
||||
#include <boost/bind/bind.hpp>
|
||||
#include <boost/tokenizer.hpp>
|
||||
#include <boost/foreach.hpp>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/assign/list_of.hpp>
|
||||
#include <boost/algorithm/string.hpp>
|
||||
|
||||
#include <speex/speex_resampler.h>
|
||||
|
||||
#define BUFFER_GROW_SIZE (80000)
|
||||
|
||||
typedef boost::circular_buffer<uint16_t> CircularBuffer_t;
|
||||
/* Global information, common to all connections */
|
||||
typedef struct
|
||||
{
|
||||
CURLM *multi;
|
||||
int still_running;
|
||||
} GlobalInfo_t;
|
||||
static GlobalInfo_t global;
|
||||
|
||||
/* Information associated with a specific easy handle */
|
||||
typedef struct
|
||||
{
|
||||
CURL *easy;
|
||||
rimelabs_t* rimelabs;
|
||||
char* body;
|
||||
struct curl_slist *hdr_list;
|
||||
GlobalInfo_t *global;
|
||||
char error[CURL_ERROR_SIZE];
|
||||
FILE* file;
|
||||
std::chrono::time_point<std::chrono::high_resolution_clock> startTime;
|
||||
bool flushed;
|
||||
|
||||
bool has_last_byte;
|
||||
uint8_t last_byte;
|
||||
} ConnInfo_t;
|
||||
|
||||
|
||||
static boost::object_pool<ConnInfo_t> pool ;
|
||||
static std::map<curl_socket_t, boost::asio::ip::tcp::socket *> socket_map;
|
||||
static boost::asio::io_service io_service;
|
||||
static boost::asio::deadline_timer timer(io_service);
|
||||
static std::string fullDirPath;
|
||||
static std::thread worker_thread;
|
||||
|
||||
std::string secondsToMillisecondsString(double seconds) {
|
||||
// Convert to milliseconds
|
||||
double milliseconds = seconds * 1000.0;
|
||||
|
||||
// Truncate to remove fractional part
|
||||
long milliseconds_long = static_cast<long>(milliseconds);
|
||||
|
||||
// Convert to string
|
||||
return std::to_string(milliseconds_long);
|
||||
}
|
||||
|
||||
static CURL* createEasyHandle(void) {
|
||||
CURL* easy = curl_easy_init();
|
||||
if(!easy) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "curl_easy_init() failed!\n");
|
||||
return nullptr ;
|
||||
}
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(easy, CURLOPT_USERAGENT, "jambonz/0.8.5");
|
||||
|
||||
// set connect timeout to 3 seconds and total timeout to 109 seconds
|
||||
curl_easy_setopt(easy, CURLOPT_CONNECTTIMEOUT_MS, 3000L);
|
||||
curl_easy_setopt(easy, CURLOPT_TIMEOUT, 10L);
|
||||
|
||||
return easy ;
|
||||
}
|
||||
|
||||
static void cleanupConn(ConnInfo_t *conn) {
|
||||
auto d = conn->rimelabs;
|
||||
|
||||
if( conn->hdr_list ) {
|
||||
curl_slist_free_all(conn->hdr_list);
|
||||
conn->hdr_list = nullptr ;
|
||||
}
|
||||
curl_easy_cleanup(conn->easy);
|
||||
|
||||
if (conn->file) {
|
||||
if (fclose(conn->file) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error closing audio cache file\n");
|
||||
}
|
||||
conn->file = nullptr ;
|
||||
}
|
||||
|
||||
d->conn = nullptr ;
|
||||
d->draining = 1;
|
||||
|
||||
memset(conn, 0, sizeof(ConnInfo_t));
|
||||
pool.destroy(conn) ;
|
||||
}
|
||||
|
||||
/* Check for completed transfers, and remove their easy handles */
|
||||
void check_multi_info(GlobalInfo_t *g) {
|
||||
CURLMsg *msg;
|
||||
int msgs_left;
|
||||
ConnInfo_t *conn;
|
||||
CURL *easy;
|
||||
CURLcode res;
|
||||
|
||||
while((msg = curl_multi_info_read(g->multi, &msgs_left))) {
|
||||
if(msg->msg == CURLMSG_DONE) {
|
||||
long response_code;
|
||||
double namelookup=0, connect=0, total=0 ;
|
||||
char *ct = NULL ;
|
||||
|
||||
easy = msg->easy_handle;
|
||||
res = msg->data.result;
|
||||
curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
|
||||
curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &response_code);
|
||||
curl_easy_getinfo(easy, CURLINFO_CONTENT_TYPE, &ct);
|
||||
|
||||
curl_easy_getinfo(easy, CURLINFO_NAMELOOKUP_TIME, &namelookup);
|
||||
curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &connect);
|
||||
curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &total);
|
||||
|
||||
auto d = conn->rimelabs;
|
||||
d->response_code = response_code;
|
||||
if (ct) d->ct = strdup(ct);
|
||||
|
||||
std::string name_lookup_ms = secondsToMillisecondsString(namelookup);
|
||||
std::string connect_ms = secondsToMillisecondsString(connect);
|
||||
std::string final_response_time_ms = secondsToMillisecondsString(total);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG,
|
||||
"mod_rimelabs_tts: response: %ld, content-type %s,"
|
||||
"dns(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld, "
|
||||
"connect(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld, "
|
||||
"total(ms): %" CURL_FORMAT_CURL_OFF_T ".%06ld\n",
|
||||
response_code, ct,
|
||||
(long)(namelookup), (long)(fmod(namelookup, 1.0) * 1000000),
|
||||
(long)(connect), (long)(fmod(connect, 1.0) * 1000000),
|
||||
(long)(total), (long)(fmod(total, 1.0) * 1000000));
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "name lookup time: %s\n", name_lookup_ms.c_str());
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "connect time: %s\n", connect_ms.c_str());
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "final response time: %s\n", final_response_time_ms.c_str());
|
||||
|
||||
d->name_lookup_time_ms = strdup(name_lookup_ms.c_str());
|
||||
d->connect_time_ms = strdup(connect_ms.c_str());
|
||||
d->final_response_time_ms = strdup(final_response_time_ms.c_str());
|
||||
|
||||
curl_multi_remove_handle(g->multi, easy);
|
||||
cleanupConn(conn);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int mcode_test(const char *where, CURLMcode code) {
|
||||
if(CURLM_OK != code) {
|
||||
const char *s;
|
||||
switch(code) {
|
||||
case CURLM_CALL_MULTI_PERFORM:
|
||||
s = "CURLM_CALL_MULTI_PERFORM";
|
||||
break;
|
||||
case CURLM_BAD_HANDLE:
|
||||
s = "CURLM_BAD_HANDLE";
|
||||
break;
|
||||
case CURLM_BAD_EASY_HANDLE:
|
||||
s = "CURLM_BAD_EASY_HANDLE";
|
||||
break;
|
||||
case CURLM_OUT_OF_MEMORY:
|
||||
s = "CURLM_OUT_OF_MEMORY";
|
||||
break;
|
||||
case CURLM_INTERNAL_ERROR:
|
||||
s = "CURLM_INTERNAL_ERROR";
|
||||
break;
|
||||
case CURLM_UNKNOWN_OPTION:
|
||||
s = "CURLM_UNKNOWN_OPTION";
|
||||
break;
|
||||
case CURLM_LAST:
|
||||
s = "CURLM_LAST";
|
||||
break;
|
||||
default:
|
||||
s = "CURLM_unknown";
|
||||
break;
|
||||
case CURLM_BAD_SOCKET:
|
||||
s = "CURLM_BAD_SOCKET";
|
||||
break;
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mcode_test ERROR: %s returns %s:%d\n", where, s, code);
|
||||
|
||||
return -1;
|
||||
}
|
||||
return 0 ;
|
||||
}
|
||||
|
||||
static void remsock(int *f, GlobalInfo_t *g) {
|
||||
if(f) {
|
||||
free(f);
|
||||
f = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/* Called by asio when there is an action on a socket */
|
||||
static void event_cb(GlobalInfo_t *g, curl_socket_t s, int action, const boost::system::error_code & error, int *fdp) {
|
||||
int f = *fdp;
|
||||
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X has action %d\n", s, action) ;
|
||||
|
||||
// Socket already POOL REMOVED.
|
||||
if (f == CURL_POLL_REMOVE) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb socket %#X removed\n", s);
|
||||
remsock(fdp, g);
|
||||
return;
|
||||
}
|
||||
|
||||
if(socket_map.find(s) == socket_map.end()) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "event_cb: socket %#X already closed\n, s");
|
||||
return;
|
||||
}
|
||||
|
||||
/* make sure the event matches what are wanted */
|
||||
if(f == action || f == CURL_POLL_INOUT) {
|
||||
if(error) {
|
||||
action = CURL_CSELECT_ERR;
|
||||
}
|
||||
CURLMcode rc = curl_multi_socket_action(g->multi, s, action, &g->still_running);
|
||||
|
||||
mcode_test("event_cb: curl_multi_socket_action", rc);
|
||||
check_multi_info(g);
|
||||
|
||||
if(g->still_running <= 0) {
|
||||
timer.cancel();
|
||||
}
|
||||
|
||||
/* keep on watching.
|
||||
* the socket may have been closed and/or fdp may have been changed
|
||||
* in curl_multi_socket_action(), so check them both */
|
||||
if(!error && socket_map.find(s) != socket_map.end() &&
|
||||
(f == action || f == CURL_POLL_INOUT)) {
|
||||
boost::asio::ip::tcp::socket *tcp_socket = socket_map.find(s)->second;
|
||||
|
||||
if(action == CURL_POLL_IN) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
action, boost::placeholders::_1, fdp));
|
||||
}
|
||||
if(action == CURL_POLL_OUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
action, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* socket functions */
|
||||
static void setsock(int *fdp, curl_socket_t s, CURL *e, int act, int oldact, GlobalInfo_t *g) {
|
||||
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(s);
|
||||
|
||||
if(it == socket_map.end()) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "setsock: socket %#X not found\n, s");
|
||||
return;
|
||||
}
|
||||
|
||||
boost::asio::ip::tcp::socket * tcp_socket = it->second;
|
||||
|
||||
*fdp = act;
|
||||
|
||||
if(act == CURL_POLL_IN) {
|
||||
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_IN, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
else if(act == CURL_POLL_OUT) {
|
||||
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_OUT, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
else if(act == CURL_POLL_INOUT) {
|
||||
if(oldact != CURL_POLL_IN && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_read_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_IN, boost::placeholders::_1, fdp));
|
||||
}
|
||||
if(oldact != CURL_POLL_OUT && oldact != CURL_POLL_INOUT) {
|
||||
tcp_socket->async_write_some(boost::asio::null_buffers(),
|
||||
boost::bind(&event_cb, g, s,
|
||||
CURL_POLL_OUT, boost::placeholders::_1, fdp));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo_t *g) {
|
||||
/* fdp is used to store current action */
|
||||
int *fdp = (int *) calloc(sizeof(int), 1);
|
||||
|
||||
setsock(fdp, s, easy, action, 0, g);
|
||||
curl_multi_assign(g->multi, s, fdp);
|
||||
}
|
||||
|
||||
static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp) {
|
||||
GlobalInfo_t *g = &global;
|
||||
|
||||
int *actionp = (int *) sockp;
|
||||
static const char *whatstr[] = { "none", "IN", "OUT", "INOUT", "REMOVE"};
|
||||
|
||||
if(what == CURL_POLL_REMOVE) {
|
||||
*actionp = what;
|
||||
}
|
||||
else {
|
||||
if(!actionp) {
|
||||
addsock(s, e, what, g);
|
||||
}
|
||||
else {
|
||||
setsock(actionp, s, e, what, *actionp, g);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void threadFunc() {
|
||||
/* to make sure the event loop doesn't terminate when there is no work to do */
|
||||
io_service.reset() ;
|
||||
boost::asio::io_service::work work(io_service);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_rimelabs_tts threadFunc - starting\n");
|
||||
|
||||
for(;;) {
|
||||
|
||||
try {
|
||||
io_service.run() ;
|
||||
break ;
|
||||
}
|
||||
catch( std::exception& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "mod_rimelabs_tts threadFunc - Error: %s\n", e.what());
|
||||
}
|
||||
}
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "mod_rimelabs_tts threadFunc - ending\n");
|
||||
}
|
||||
|
||||
|
||||
/* Called by asio when our timeout expires */
|
||||
static void timer_cb(const boost::system::error_code & error, GlobalInfo_t *g)
|
||||
{
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "timer_cb\n");
|
||||
|
||||
if(!error) {
|
||||
CURLMcode rc = curl_multi_socket_action(g->multi, CURL_SOCKET_TIMEOUT, 0, &g->still_running);
|
||||
mcode_test("timer_cb: curl_multi_socket_action", rc);
|
||||
check_multi_info(g);
|
||||
}
|
||||
}
|
||||
|
||||
int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo_t *g) {
|
||||
|
||||
/* cancel running timer */
|
||||
timer.cancel();
|
||||
|
||||
if(timeout_ms >= 0) {
|
||||
// from libcurl 7.88.1-10+deb12u4 does not allow call curl_multi_socket_action or curl_multi_perform in curl_multi callback directly
|
||||
timer.expires_from_now(boost::posix_time::millisec(timeout_ms ? timeout_ms : 1));
|
||||
timer.async_wait(boost::bind(&timer_cb, boost::placeholders::_1, g));
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* CURLOPT_WRITEFUNCTION */
|
||||
static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
|
||||
bool fireEvent = false;
|
||||
uint8_t *data = (uint8_t *) ptr;
|
||||
size_t bytes_received = size * nmemb;
|
||||
size_t total_bytes_to_process;
|
||||
auto d = conn->rimelabs;
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) d->circularBuffer;
|
||||
|
||||
if (conn->flushed || cBuffer == nullptr) {
|
||||
/* this will abort the transfer */
|
||||
return 0;
|
||||
}
|
||||
// Buffer to hold combined data if there is unprocessed byte from the last call.
|
||||
std::unique_ptr<uint8_t[]> combinedData;
|
||||
|
||||
if (conn->has_last_byte) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "rimelabs write_cb: processing last byte from previous read\n");
|
||||
|
||||
conn->has_last_byte = false; // We'll handle the last_byte now, so toggle the flag off
|
||||
|
||||
// Allocate memory for the new data array
|
||||
combinedData.reset(new uint8_t[bytes_received + 1]);
|
||||
|
||||
// Prepend the last byte from previous call
|
||||
combinedData[0] = conn->last_byte;
|
||||
|
||||
// Copy the new data following the prepended byte
|
||||
memcpy(combinedData.get() + 1, data, bytes_received);
|
||||
|
||||
// Point our data pointer to the new array
|
||||
data = combinedData.get();
|
||||
|
||||
total_bytes_to_process = bytes_received + 1;
|
||||
} else {
|
||||
total_bytes_to_process = bytes_received;
|
||||
}
|
||||
|
||||
// If we now have an odd total, save the last byte for next time
|
||||
if ((total_bytes_to_process % sizeof(int16_t)) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "rimelabs write_cb: got odd number of bytes %d\n", total_bytes_to_process);
|
||||
|
||||
conn->last_byte = data[total_bytes_to_process - 1];
|
||||
conn->has_last_byte = true;
|
||||
total_bytes_to_process--;
|
||||
}
|
||||
|
||||
int16_t* inputData = reinterpret_cast<int16_t*>(data);
|
||||
if (0 == d->reads++) {
|
||||
fireEvent = true;
|
||||
// Deepgram return PCM linear16 WAV file which contains 44 bytes headers, remove that.
|
||||
inputData += 22;
|
||||
total_bytes_to_process -= 44;
|
||||
}
|
||||
size_t numSamples = total_bytes_to_process / sizeof(int16_t);
|
||||
{
|
||||
switch_mutex_lock(d->mutex);
|
||||
|
||||
if (d->response_code > 0 && d->response_code != 200) {
|
||||
std::string body((char *) ptr, bytes_received);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: received body %s\n", body.c_str());
|
||||
d->err_msg = strdup(body.c_str());
|
||||
switch_mutex_unlock(d->mutex);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* cache file will stay in the mp3 format for size (smaller) and simplicity */
|
||||
if (conn->file) fwrite(inputData, sizeof(int16_t), numSamples, conn->file);
|
||||
|
||||
// Resize the buffer if necessary
|
||||
if (cBuffer->capacity() - cBuffer->size() < numSamples) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "write_cb growing buffer\n");
|
||||
|
||||
//TODO: if buffer exceeds some max size, return CURL_WRITEFUNC_ERROR to abort the transfer
|
||||
cBuffer->set_capacity(cBuffer->size() + std::max(numSamples, (size_t)BUFFER_GROW_SIZE));
|
||||
}
|
||||
|
||||
/* Push the data into the buffer */
|
||||
cBuffer->insert(cBuffer->end(), inputData, inputData + numSamples);
|
||||
|
||||
switch_mutex_unlock(d->mutex);
|
||||
}
|
||||
if (fireEvent && d->session_id) {
|
||||
auto endTime = std::chrono::high_resolution_clock::now();
|
||||
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(endTime - conn->startTime);
|
||||
auto time_to_first_byte_ms = std::to_string(duration.count());
|
||||
switch_core_session_t* session = switch_core_session_locate(d->session_id);
|
||||
if (session) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_core_session_rwunlock(session);
|
||||
if (channel) {
|
||||
switch_event_t *event;
|
||||
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_START) == SWITCH_STATUS_SUCCESS) {
|
||||
switch_channel_event_set_data(channel, event);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: firing playback-started\n");
|
||||
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
|
||||
if (d->name_lookup_time_ms) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_rimelabs_name_lookup_time_ms", d->name_lookup_time_ms);
|
||||
}
|
||||
if (d->connect_time_ms) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_rimelabs_connect_time_ms", d->connect_time_ms);
|
||||
}
|
||||
if (d->final_response_time_ms) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_rimelabs_final_response_time_ms", d->final_response_time_ms);
|
||||
}
|
||||
if (d->voice_name) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_rimelabs_voice_name", d->voice_name);
|
||||
}
|
||||
if (d->cache_filename) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", d->cache_filename);
|
||||
}
|
||||
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_time_to_first_byte_ms", time_to_first_byte_ms.c_str());
|
||||
switch_event_fire(&event);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: session %s not found\n", d->session_id);
|
||||
}
|
||||
}
|
||||
return size*nmemb;
|
||||
}
|
||||
|
||||
static bool parseHeader(const std::string& str, std::string& header, std::string& value) {
|
||||
std::vector<std::string> parts;
|
||||
boost::split(parts, str, boost::is_any_of(":"), boost::token_compress_on);
|
||||
|
||||
if (parts.size() != 2)
|
||||
return false;
|
||||
|
||||
header = boost::trim_copy(parts[0]);
|
||||
value = boost::trim_copy(parts[1]);
|
||||
return true;
|
||||
}
|
||||
|
||||
static int extract_response_code(const std::string& input) {
|
||||
std::size_t space_pos = input.find(' ');
|
||||
if (space_pos == std::string::npos) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid HTTP response format %s\n", input.c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::size_t code_start_pos = space_pos + 1;
|
||||
std::size_t code_end_pos = input.find(' ', code_start_pos);
|
||||
if (code_end_pos == std::string::npos) {
|
||||
code_end_pos = input.length();
|
||||
}
|
||||
|
||||
std::string code_str = input.substr(code_start_pos, code_end_pos - code_start_pos);
|
||||
int response_code = std::stoi(code_str);
|
||||
return response_code;
|
||||
}
|
||||
|
||||
static size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo_t *conn) {
|
||||
size_t bytes_received = size * nitems;
|
||||
const std::string prefix = "HTTP/";
|
||||
rimelabs_t* d = conn->rimelabs;
|
||||
std::string header, value;
|
||||
std::string input(buffer, bytes_received);
|
||||
if (parseHeader(input, header, value)) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s with value %s\n", header.c_str(), value.c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: %s\n", input.c_str());
|
||||
if (input.rfind(prefix, 0) == 0) {
|
||||
try {
|
||||
d->response_code = extract_response_code(input);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: parsed response code: %ld\n", d->response_code);
|
||||
} catch (const std::invalid_argument& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "header_callback: invalid response code %s\n", input.substr(prefix.length()).c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
return bytes_received;
|
||||
}
|
||||
|
||||
/* CURLOPT_OPENSOCKETFUNCTION */
|
||||
static curl_socket_t opensocket(void *clientp, curlsocktype purpose, struct curl_sockaddr *address) {
|
||||
curl_socket_t sockfd = CURL_SOCKET_BAD;
|
||||
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "opensocket: %d\n", purpose);
|
||||
/* restrict to IPv4 */
|
||||
if(purpose == CURLSOCKTYPE_IPCXN && address->family == AF_INET) {
|
||||
/* create a tcp socket object */
|
||||
boost::asio::ip::tcp::socket *tcp_socket = new boost::asio::ip::tcp::socket(io_service);
|
||||
|
||||
/* open it and get the native handle*/
|
||||
boost::system::error_code ec;
|
||||
tcp_socket->open(boost::asio::ip::tcp::v4(), ec);
|
||||
|
||||
if(ec) {
|
||||
/* An error occurred */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Couldn't open socket [%ld][%s]\n", ec, ec.message().c_str());
|
||||
}
|
||||
else {
|
||||
sockfd = tcp_socket->native_handle();
|
||||
|
||||
/* save it for monitoring */
|
||||
socket_map.insert(std::pair<curl_socket_t, boost::asio::ip::tcp::socket *>(sockfd, tcp_socket));
|
||||
}
|
||||
}
|
||||
return sockfd;
|
||||
}
|
||||
|
||||
/* CURLOPT_CLOSESOCKETFUNCTION */
|
||||
static int close_socket(void *clientp, curl_socket_t item) {
|
||||
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "close_socket : %#X\n", item);
|
||||
|
||||
std::map<curl_socket_t, boost::asio::ip::tcp::socket *>::iterator it = socket_map.find(item);
|
||||
if(it != socket_map.end()) {
|
||||
delete it->second;
|
||||
socket_map.erase(it);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
extern "C" {
|
||||
switch_status_t rimelabs_speech_load() {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "rimelabs_speech_loading..\n");
|
||||
memset(&global, 0, sizeof(GlobalInfo_t));
|
||||
global.multi = curl_multi_init();
|
||||
|
||||
if (!global.multi) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "rimelabs_speech_load curl_multi_init() failed, exiting!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
curl_multi_setopt(global.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_SOCKETDATA, &global);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_TIMERDATA, &global);
|
||||
curl_multi_setopt(global.multi, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
|
||||
|
||||
/* create temp folder for cache files */
|
||||
const char* baseDir = std::getenv("JAMBONZ_TMP_CACHE_FOLDER");
|
||||
if (!baseDir) {
|
||||
baseDir = "/tmp/";
|
||||
}
|
||||
if (strcmp(baseDir, "/") == 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", baseDir);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
fullDirPath = std::string(baseDir) + "tts-cache-files";
|
||||
|
||||
// Create the directory with read, write, and execute permissions for everyone
|
||||
mode_t oldMask = umask(0);
|
||||
int result = mkdir(fullDirPath.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
|
||||
umask(oldMask);
|
||||
if (result != 0) {
|
||||
if (errno != EEXIST) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", fullDirPath.c_str());
|
||||
fullDirPath = "";
|
||||
}
|
||||
else switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "folder %s already exists\n", fullDirPath.c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "created folder %s\n", fullDirPath.c_str());
|
||||
}
|
||||
|
||||
/* start worker thread that handles transfers*/
|
||||
std::thread t(threadFunc) ;
|
||||
worker_thread.swap( t ) ;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "rimelabs_speech_loaded..\n");
|
||||
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t rimelabs_speech_unload() {
|
||||
/* stop the ASIO IO service */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "rimelabs_speech_unload: stopping io service\n");
|
||||
io_service.stop();
|
||||
|
||||
/* Join the worker thread */
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "rimelabs_speech_unload: wait for worker thread to complete\n");
|
||||
if (worker_thread.joinable()) {
|
||||
worker_thread.join();
|
||||
}
|
||||
|
||||
/* cleanup curl multi handle*/
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "rimelabs_speech_unload: release curl multi\n");
|
||||
curl_multi_cleanup(global.multi);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "rimelabs_speech_unload: completed\n");
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t rimelabs_speech_open(rimelabs_t* rimelabs) {
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t rimelabs_speech_feed_tts(rimelabs_t* d, char* text, switch_speech_flag_t *flags) {
|
||||
CURLMcode rc;
|
||||
|
||||
const int MAX_CHARS = 20;
|
||||
char tempText[MAX_CHARS + 4]; // +4 for the ellipsis and null terminator
|
||||
|
||||
if (strlen(text) > MAX_CHARS) {
|
||||
strncpy(tempText, text, MAX_CHARS);
|
||||
strcpy(tempText + MAX_CHARS, "...");
|
||||
} else {
|
||||
strcpy(tempText, text);
|
||||
}
|
||||
|
||||
/* open cache file */
|
||||
if (d->cache_audio && fullDirPath.length() > 0) {
|
||||
switch_uuid_t uuid;
|
||||
char uuid_str[SWITCH_UUID_FORMATTED_LENGTH + 1];
|
||||
char outfile[512] = "";
|
||||
int fd;
|
||||
|
||||
switch_uuid_get(&uuid);
|
||||
switch_uuid_format(uuid_str, &uuid);
|
||||
|
||||
switch_snprintf(outfile, sizeof(outfile), "%s%s%s.r8", fullDirPath.c_str(), SWITCH_PATH_SEPARATOR, uuid_str);
|
||||
d->cache_filename = strdup(outfile);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "writing audio cache file to %s\n", d->cache_filename);
|
||||
|
||||
mode_t oldMask = umask(0);
|
||||
fd = open(outfile, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
|
||||
umask(oldMask);
|
||||
if (fd == -1 ) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
|
||||
}
|
||||
else {
|
||||
d->file = fdopen(fd, "wb");
|
||||
if (!d->file) {
|
||||
close(fd);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening cache file %s: %s\n", outfile, strerror(errno));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!d->api_key) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "rimelabs_speech_feed_tts: no api_key provided\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
/* format url*/
|
||||
std::string url = "https://users.rime.ai/v1/rime-tts";
|
||||
|
||||
/* create the JSON body */
|
||||
cJSON * jResult = cJSON_CreateObject();
|
||||
cJSON_AddStringToObject(jResult, "text", text);
|
||||
cJSON_AddNumberToObject(jResult, "samplingRate", 8000);
|
||||
if (d->voice_name) {
|
||||
cJSON_AddStringToObject(jResult, "speaker", d->voice_name);
|
||||
}
|
||||
if (d->model_id) {
|
||||
cJSON_AddStringToObject(jResult, "modelId", d->model_id);
|
||||
}
|
||||
if (d->speed_alpha) {
|
||||
cJSON_AddNumberToObject(jResult, "speedAlpha", std::strtof(d->speed_alpha, nullptr));
|
||||
}
|
||||
if (d->reduce_latency) {
|
||||
cJSON_AddBoolToObject(jResult, "reduceLatency", !strcmp(d->reduce_latency, "true") ? 1 : 0);
|
||||
}
|
||||
char *json = cJSON_PrintUnformatted(jResult);
|
||||
|
||||
cJSON_Delete(jResult);
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "rimelabs_speech_feed_tts: [%s] [%s]\n", url.c_str(), tempText);
|
||||
|
||||
ConnInfo_t *conn = pool.malloc() ;
|
||||
|
||||
CURL* easy = createEasyHandle();
|
||||
d->conn = (void *) conn ;
|
||||
conn->rimelabs = d;
|
||||
conn->easy = easy;
|
||||
conn->global = &global;
|
||||
conn->hdr_list = NULL ;
|
||||
conn->file = d->file;
|
||||
conn->body = json;
|
||||
conn->flushed = false;
|
||||
conn->has_last_byte = false;
|
||||
conn->last_byte = 0;
|
||||
|
||||
|
||||
d->circularBuffer = (void *) new CircularBuffer_t(BUFFER_GROW_SIZE);
|
||||
// Always use rimelabs at rate 8000 for helping cache audio from jambonz.
|
||||
if (d->rate != 8000) {
|
||||
int err;
|
||||
d->resampler = speex_resampler_init(1, 8000, d->rate, SWITCH_RESAMPLE_QUALITY, &err);
|
||||
if (0 != err) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing resampler: %s.\n", speex_resampler_strerror(err));
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
std::ostringstream api_key_stream;
|
||||
api_key_stream << "Authorization: Bearer " << d->api_key;
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, write_cb);
|
||||
curl_easy_setopt(easy, CURLOPT_WRITEDATA, conn);
|
||||
curl_easy_setopt(easy, CURLOPT_ERRORBUFFER, conn->error);
|
||||
curl_easy_setopt(easy, CURLOPT_PRIVATE, conn);
|
||||
curl_easy_setopt(easy, CURLOPT_VERBOSE, 0L);
|
||||
curl_easy_setopt(easy, CURLOPT_NOPROGRESS, 1L);
|
||||
curl_easy_setopt(easy, CURLOPT_HEADERFUNCTION, header_callback);
|
||||
curl_easy_setopt(easy, CURLOPT_HEADERDATA, conn);
|
||||
|
||||
/* call this function to get a socket */
|
||||
curl_easy_setopt(easy, CURLOPT_OPENSOCKETFUNCTION, opensocket);
|
||||
|
||||
/* call this function to close a socket */
|
||||
curl_easy_setopt(easy, CURLOPT_CLOSESOCKETFUNCTION, close_socket);
|
||||
|
||||
conn->hdr_list = curl_slist_append(conn->hdr_list, api_key_stream.str().c_str());
|
||||
conn->hdr_list = curl_slist_append(conn->hdr_list, "Accept: audio/pcm");
|
||||
conn->hdr_list = curl_slist_append(conn->hdr_list, "Content-Type: application/json");
|
||||
curl_easy_setopt(easy, CURLOPT_HTTPHEADER, conn->hdr_list);
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
|
||||
//curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
|
||||
|
||||
// libcurl adding random byte to the response body that creates white noise to audio file
|
||||
// https://github.com/curl/curl/issues/10525
|
||||
const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
|
||||
curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
|
||||
|
||||
rc = curl_multi_add_handle(global.multi, conn->easy);
|
||||
mcode_test("new_conn: curl_multi_add_handle", rc);
|
||||
|
||||
/* start a timer to measure the duration until we receive first byte of audio */
|
||||
conn->startTime = std::chrono::high_resolution_clock::now();
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "rimelabs_speech_feed_tts: called curl_multi_add_handle\n");
|
||||
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t rimelabs_speech_read_tts(rimelabs_t* d, void *data, size_t *datalen, switch_speech_flag_t *flags) {
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) d->circularBuffer;
|
||||
std::vector<uint16_t> pcm_data;
|
||||
|
||||
{
|
||||
switch_mutex_lock(d->mutex);
|
||||
ConnInfo_t *conn = (ConnInfo_t *) d->conn;
|
||||
if (d->response_code > 0 && d->response_code != 200) {
|
||||
switch_mutex_unlock(d->mutex);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "rimelabs_speech_read_tts, returning failure\n") ;
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
if (conn && conn->flushed) {
|
||||
switch_mutex_unlock(d->mutex);
|
||||
return SWITCH_STATUS_BREAK;
|
||||
}
|
||||
if (cBuffer->empty()) {
|
||||
if (d->draining) {
|
||||
switch_mutex_unlock(d->mutex);
|
||||
return SWITCH_STATUS_BREAK;
|
||||
}
|
||||
/* no audio available yet so send silence */
|
||||
memset(data, 255, *datalen);
|
||||
switch_mutex_unlock(d->mutex);
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
size_t size = std::min((*datalen/(2 * d->rate / 8000)), cBuffer->size());
|
||||
pcm_data.insert(pcm_data.end(), cBuffer->begin(), cBuffer->begin() + size);
|
||||
cBuffer->erase(cBuffer->begin(), cBuffer->begin() + size);
|
||||
switch_mutex_unlock(d->mutex);
|
||||
}
|
||||
|
||||
size_t data_size = pcm_data.size();
|
||||
|
||||
if (d->resampler) {
|
||||
std::vector<int16_t> in(pcm_data.begin(), pcm_data.end());
|
||||
|
||||
std::vector<int16_t> out((*datalen));
|
||||
spx_uint32_t in_len = data_size;
|
||||
spx_uint32_t out_len = out.size();
|
||||
speex_resampler_process_interleaved_int(d->resampler, in.data(), &in_len, out.data(), &out_len);
|
||||
|
||||
if (out_len > out.size()) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Resampler output exceeded maximum buffer size!\n");
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
memcpy(data, out.data(), out_len * sizeof(int16_t));
|
||||
*datalen = out_len * sizeof(int16_t);
|
||||
} else {
|
||||
memcpy(data, pcm_data.data(), pcm_data.size() * sizeof(uint16_t));
|
||||
*datalen = pcm_data.size() * sizeof(uint16_t);
|
||||
}
|
||||
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t rimelabs_speech_flush_tts(rimelabs_t* d) {
|
||||
bool download_complete = d->response_code == 200;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "rimelabs_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;
|
||||
|
||||
ConnInfo_t *conn = (ConnInfo_t *) d->conn;
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) d->circularBuffer;
|
||||
delete cBuffer;
|
||||
d->circularBuffer = nullptr ;
|
||||
|
||||
// destroy resampler
|
||||
if (d->resampler) {
|
||||
speex_resampler_destroy(d->resampler);
|
||||
d->resampler = NULL;
|
||||
}
|
||||
|
||||
if (conn) {
|
||||
conn->flushed = true;
|
||||
if (!download_complete) {
|
||||
if (conn->file) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "closing audio cache file %s because download was interrupted\n", d->cache_filename);
|
||||
if (fclose(conn->file) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "error closing audio cache file\n");
|
||||
}
|
||||
conn->file = nullptr ;
|
||||
}
|
||||
|
||||
if (d->cache_filename) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "removing audio cache file %s because download was interrupted\n", d->cache_filename);
|
||||
if (unlink(d->cache_filename) != 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "cleanupConn: error removing audio cache file %s: %d:%s\n",
|
||||
d->cache_filename, errno, strerror(errno));
|
||||
}
|
||||
free(d->cache_filename);
|
||||
d->cache_filename = nullptr ;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (d->session_id) {
|
||||
switch_core_session_t* session = switch_core_session_locate(d->session_id);
|
||||
if (session) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_core_session_rwunlock(session);
|
||||
if (channel) {
|
||||
switch_event_t *event;
|
||||
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_STOP) == SWITCH_STATUS_SUCCESS) {
|
||||
switch_channel_event_set_data(channel, event);
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "Playback-File-Type", "tts_stream");
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_rimelabs_response_code", std::to_string(d->response_code).c_str());
|
||||
if (d->cache_filename && d->response_code == 200) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_cache_filename", d->cache_filename);
|
||||
}
|
||||
if (d->response_code != 200 && d->err_msg) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_error", d->err_msg);
|
||||
}
|
||||
switch_event_fire(&event);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: failed to create event\n");
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t rimelabs_speech_close(rimelabs_t* w) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "rimelabs_speech_close\n") ;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#ifndef __RIMELABS_GLUE_H__
|
||||
#define __RIMELABS_GLUE_H__
|
||||
|
||||
switch_status_t rimelabs_speech_load();
|
||||
switch_status_t rimelabs_speech_open(rimelabs_t* rimelabs);
|
||||
switch_status_t rimelabs_speech_feed_tts(rimelabs_t* rimelabs, char* text, switch_speech_flag_t *flags);
|
||||
switch_status_t rimelabs_speech_read_tts(rimelabs_t* rimelabs, void *data, size_t *datalen, switch_speech_flag_t *flags);
|
||||
switch_status_t rimelabs_speech_flush_tts(rimelabs_t* rimelabs);
|
||||
switch_status_t rimelabs_speech_close(rimelabs_t* rimelabs);
|
||||
switch_status_t rimelabs_speech_unload();
|
||||
|
||||
#endif
|
||||
@@ -5,4 +5,4 @@ mod_LTLIBRARIES = mod_whisper_tts.la
|
||||
mod_whisper_tts_la_SOURCES = mod_whisper_tts.c whisper_glue.cpp
|
||||
mod_whisper_tts_la_CFLAGS = $(AM_CFLAGS)
|
||||
mod_whisper_tts_la_LIBADD = $(switch_builddir)/libfreeswitch.la
|
||||
mod_whisper_tts_la_LDFLAGS = -avoid-version -module -no-undefined -shared `pkg-config --libs boost` -lstdc++ -lmpg123
|
||||
mod_whisper_tts_la_LDFLAGS = -avoid-version -module -no-undefined -shared -lstdc++ -lboost_system -lboost_thread -lmpg123
|
||||
|
||||
@@ -6,12 +6,16 @@ SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_whisper_tts_shutdown);
|
||||
SWITCH_MODULE_DEFINITION(mod_whisper_tts, mod_whisper_tts_load, mod_whisper_tts_shutdown, NULL);
|
||||
|
||||
static void clearWhisper(whisper_t* w, int freeAll) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "clearWhisper\n");
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "clearWhisper\n");
|
||||
if (w->api_key) free(w->api_key);
|
||||
if (w->model_id) free(w->model_id);
|
||||
if (w->speed) free(w->speed);
|
||||
if (w->request_id) free(w->request_id);
|
||||
if (w->reported_latency) free(w->reported_latency);
|
||||
if (w->reported_organization) free(w->reported_organization);
|
||||
if (w->reported_ratelimit_requests) free(w->reported_ratelimit_requests);
|
||||
if (w->reported_ratelimit_remaining_requests) free(w->reported_ratelimit_remaining_requests);
|
||||
if (w->reported_ratelimit_reset_requests) free(w->reported_ratelimit_reset_requests);
|
||||
if (w->ct) free(w->ct);
|
||||
if (w->err_msg) free(w->err_msg);
|
||||
if (w->name_lookup_time_ms) free(w->name_lookup_time_ms);
|
||||
@@ -25,6 +29,10 @@ static void clearWhisper(whisper_t* w, int freeAll) {
|
||||
w->speed = NULL;
|
||||
w->request_id = NULL;
|
||||
w->reported_latency = NULL;
|
||||
w->reported_organization = NULL;
|
||||
w->reported_ratelimit_requests = NULL;
|
||||
w->reported_ratelimit_remaining_requests = NULL;
|
||||
w->reported_ratelimit_reset_requests = NULL;
|
||||
w->ct = NULL;
|
||||
w->err_msg = NULL;
|
||||
w->name_lookup_time_ms = NULL;
|
||||
@@ -65,7 +73,7 @@ static switch_status_t w_speech_close(switch_speech_handle_t *sh, switch_speech_
|
||||
{
|
||||
switch_status_t rc;
|
||||
whisper_t *w = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_speech_close\n");
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_speech_close\n");
|
||||
|
||||
switch_mutex_destroy(w->mutex);
|
||||
|
||||
@@ -83,7 +91,7 @@ static switch_status_t w_speech_feed_tts(switch_speech_handle_t *sh, char *text,
|
||||
w->draining = 0;
|
||||
w->reads = 0;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_speech_feed_tts\n");
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_speech_feed_tts\n");
|
||||
|
||||
return whisper_speech_feed_tts(w, text, flags);
|
||||
}
|
||||
@@ -94,7 +102,6 @@ static switch_status_t w_speech_feed_tts(switch_speech_handle_t *sh, char *text,
|
||||
static switch_status_t w_speech_read_tts(switch_speech_handle_t *sh, void *data, size_t *datalen, switch_speech_flag_t *flags)
|
||||
{
|
||||
whisper_t *w = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_speech_read_tts\n");
|
||||
return whisper_speech_read_tts(w, data, datalen, flags);
|
||||
}
|
||||
|
||||
@@ -104,7 +111,7 @@ static switch_status_t w_speech_read_tts(switch_speech_handle_t *sh, void *data,
|
||||
static void w_speech_flush_tts(switch_speech_handle_t *sh)
|
||||
{
|
||||
whisper_t *w = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_speech_flush_tts\n");
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_speech_flush_tts\n");
|
||||
whisper_speech_flush_tts(w);
|
||||
|
||||
clearWhisper(w, 0);
|
||||
@@ -113,7 +120,7 @@ static void w_speech_flush_tts(switch_speech_handle_t *sh)
|
||||
static void w_text_param_tts(switch_speech_handle_t *sh, char *param, const char *val)
|
||||
{
|
||||
whisper_t *w = createOrRetrievePrivateData(sh);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "w_text_param_tts: %s=%s\n", param, val);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "w_text_param_tts: %s=%s\n", param, val);
|
||||
if (0 == strcmp(param, "api_key")) {
|
||||
if (w->api_key) free(w->api_key);
|
||||
w->api_key = strdup(val);
|
||||
|
||||
@@ -11,7 +11,18 @@ typedef struct whisper_data {
|
||||
/* result data */
|
||||
long response_code;
|
||||
char *ct;
|
||||
//whisper headers
|
||||
//openai-organization
|
||||
char *reported_organization;
|
||||
//openai-processing-ms
|
||||
char *reported_latency;
|
||||
//x-ratelimit-limit-requests
|
||||
char *reported_ratelimit_requests;
|
||||
//x-ratelimit-remaining-requests
|
||||
char *reported_ratelimit_remaining_requests;
|
||||
//x-ratelimit-reset-requests
|
||||
char *reported_ratelimit_reset_requests;
|
||||
//x-request-id
|
||||
char *request_id;
|
||||
char *name_lookup_time_ms;
|
||||
char *connect_time_ms;
|
||||
|
||||
@@ -436,14 +436,12 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) w->circularBuffer;
|
||||
std::vector<uint16_t> pcm_data;
|
||||
|
||||
if (conn->flushed) {
|
||||
if (conn->flushed || cBuffer == nullptr) {
|
||||
/* this will abort the transfer */
|
||||
return 0;
|
||||
}
|
||||
{
|
||||
switch_mutex_lock(w->mutex);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "write_cb: received data, response %ld\n",
|
||||
w->response_code);
|
||||
|
||||
if (w->response_code > 0 && w->response_code != 200) {
|
||||
std::string body((char *) ptr, bytes_received);
|
||||
@@ -482,6 +480,7 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
|
||||
switch_core_session_t* session = switch_core_session_locate(w->session_id);
|
||||
if (session) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_core_session_rwunlock(session);
|
||||
if (channel) {
|
||||
switch_event_t *event;
|
||||
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_START) == SWITCH_STATUS_SUCCESS) {
|
||||
@@ -496,6 +495,18 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
|
||||
if (w->request_id) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_whisper_request_id", w->request_id);
|
||||
}
|
||||
if (w->reported_organization) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_whisper_reported_organization", w->reported_latency);
|
||||
}
|
||||
if (w->reported_ratelimit_requests) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_whisper_reported_ratelimit_requests", w->reported_ratelimit_requests);
|
||||
}
|
||||
if (w->reported_ratelimit_remaining_requests) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_whisper_reported_ratelimit_remaining_requests", w->reported_ratelimit_remaining_requests);
|
||||
}
|
||||
if (w->reported_ratelimit_reset_requests) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_whisper_reported_ratelimit_reset_requests", w->reported_ratelimit_reset_requests);
|
||||
}
|
||||
if (w->name_lookup_time_ms) {
|
||||
switch_event_add_header_string(event, SWITCH_STACK_BOTTOM, "variable_tts_whisper_name_lookup_time_ms", w->name_lookup_time_ms);
|
||||
}
|
||||
@@ -525,7 +536,6 @@ static size_t write_cb(void *ptr, size_t size, size_t nmemb, ConnInfo_t *conn) {
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
|
||||
}
|
||||
switch_core_session_rwunlock(session);
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: session %s not found\n", w->session_id);
|
||||
@@ -546,9 +556,27 @@ static bool parseHeader(const std::string& str, std::string& header, std::string
|
||||
return true;
|
||||
}
|
||||
|
||||
static int extract_response_code(const std::string& input) {
|
||||
std::size_t space_pos = input.find(' ');
|
||||
if (space_pos == std::string::npos) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid HTTP response format %s\n", input.c_str());
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::size_t code_start_pos = space_pos + 1;
|
||||
std::size_t code_end_pos = input.find(' ', code_start_pos);
|
||||
if (code_end_pos == std::string::npos) {
|
||||
code_end_pos = input.length();
|
||||
}
|
||||
|
||||
std::string code_str = input.substr(code_start_pos, code_end_pos - code_start_pos);
|
||||
int response_code = std::stoi(code_str);
|
||||
return response_code;
|
||||
}
|
||||
|
||||
static size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo_t *conn) {
|
||||
size_t bytes_received = size * nitems;
|
||||
const std::string prefix = "HTTP/2 ";
|
||||
const std::string prefix = "HTTP/";
|
||||
whisper_t* w = conn->whisper;
|
||||
std::string header, value;
|
||||
std::string input(buffer, bytes_received);
|
||||
@@ -556,13 +584,17 @@ static size_t header_callback(char *buffer, size_t size, size_t nitems, ConnInfo
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s with value %s\n", header.c_str(), value.c_str());
|
||||
if (0 == header.compare("openai-processing-ms")) w->reported_latency = strdup(value.c_str());
|
||||
else if (0 == header.compare("x-request-id")) w->request_id = strdup(value.c_str());
|
||||
else if (0 == header.compare("openai-organization")) w->reported_organization = strdup(value.c_str());
|
||||
else if (0 == header.compare("x-ratelimit-limit-requests")) w->reported_ratelimit_requests = strdup(value.c_str());
|
||||
else if (0 == header.compare("x-ratelimit-remaining-requests")) w->reported_ratelimit_remaining_requests = strdup(value.c_str());
|
||||
else if (0 == header.compare("x-ratelimit-reset-requests")) w->reported_ratelimit_reset_requests = strdup(value.c_str());
|
||||
}
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "recv header: %s\n", input.c_str());
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: %s\n", input.c_str());
|
||||
if (input.rfind(prefix, 0) == 0) {
|
||||
try {
|
||||
w->response_code = std::stoi(input.substr(prefix.length()));
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "parsed response code: %ld\n", w->response_code);
|
||||
w->response_code = extract_response_code(input);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "header_callback: parsed response code: %ld\n", w->response_code);
|
||||
} catch (const std::invalid_argument& e) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "header_callback: invalid response code %s\n", input.substr(prefix.length()).c_str());
|
||||
}
|
||||
@@ -632,14 +664,14 @@ extern "C" {
|
||||
/* create temp folder for cache files */
|
||||
const char* baseDir = std::getenv("JAMBONZ_TMP_CACHE_FOLDER");
|
||||
if (!baseDir) {
|
||||
baseDir = "/var/";
|
||||
baseDir = "/tmp/";
|
||||
}
|
||||
if (strcmp(baseDir, "/") == 0) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "failed to create folder %s\n", baseDir);
|
||||
return SWITCH_STATUS_FALSE;
|
||||
}
|
||||
|
||||
fullDirPath = std::string(baseDir) + "jambonz-tts-cache-files";
|
||||
fullDirPath = std::string(baseDir) + "tts-cache-files";
|
||||
|
||||
// Create the directory with read, write, and execute permissions for everyone
|
||||
mode_t oldMask = umask(0);
|
||||
@@ -810,6 +842,7 @@ extern "C" {
|
||||
switch_codec_implementation_t read_impl;
|
||||
switch_core_session_t *psession = switch_core_session_locate(w->session_id);
|
||||
switch_core_session_get_read_impl(psession, &read_impl);
|
||||
switch_core_session_rwunlock(psession);
|
||||
uint32_t samples_per_second = !strcasecmp(read_impl.iananame, "g722") ? read_impl.actual_samples_per_second : read_impl.samples_per_second;
|
||||
if (mpg123_param(mh, MPG123_FORCE_RATE, samples_per_second /*Hz*/, 0) != MPG123_OK) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error mpg123_param!\n");
|
||||
@@ -843,7 +876,10 @@ extern "C" {
|
||||
curl_easy_setopt(easy, CURLOPT_POSTFIELDS, conn->body);
|
||||
//curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, body.length());
|
||||
|
||||
curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
|
||||
// libcurl adding random byte to the response body that creates white noise to audio file
|
||||
// https://github.com/curl/curl/issues/10525
|
||||
const bool disable_http_2 = switch_true(std::getenv("DISABLE_HTTP2_FOR_TTS_STREAMING"));
|
||||
curl_easy_setopt(easy, CURLOPT_HTTP_VERSION, disable_http_2 ? CURL_HTTP_VERSION_1_1 : CURL_HTTP_VERSION_2_0);
|
||||
|
||||
rc = curl_multi_add_handle(global.multi, conn->easy);
|
||||
mcode_test("new_conn: curl_multi_add_handle", rc);
|
||||
@@ -897,8 +933,7 @@ extern "C" {
|
||||
|
||||
switch_status_t whisper_speech_flush_tts(whisper_t* w) {
|
||||
bool download_complete = w->response_code == 200;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "whisper_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;
|
||||
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "whisper_speech_flush_tts, download complete? %s\n", download_complete ? "yes" : "no") ;
|
||||
ConnInfo_t *conn = (ConnInfo_t *) w->conn;
|
||||
CircularBuffer_t *cBuffer = (CircularBuffer_t *) w->circularBuffer;
|
||||
delete cBuffer;
|
||||
@@ -930,6 +965,7 @@ extern "C" {
|
||||
switch_core_session_t* session = switch_core_session_locate(w->session_id);
|
||||
if (session) {
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_core_session_rwunlock(session);
|
||||
if (channel) {
|
||||
switch_event_t *event;
|
||||
if (switch_event_create(&event, SWITCH_EVENT_PLAYBACK_STOP) == SWITCH_STATUS_SUCCESS) {
|
||||
@@ -951,14 +987,13 @@ extern "C" {
|
||||
else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "write_cb: channel not found\n");
|
||||
}
|
||||
switch_core_session_rwunlock(session);
|
||||
}
|
||||
}
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
switch_status_t whisper_speech_close(whisper_t* w) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "whisper_speech_close\n") ;
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "whisper_speech_close\n") ;
|
||||
return SWITCH_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user