ben.carrasco/freeswitch-modules
1
0
Fork 0
mirror of https://github.com/jambonz/freeswitch-modules.git synced 2026-01-25 02:08:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Dave Horton
2024-08-22 09:41:03 -04:00
parent c588758ecd
commit 71a61c068d
5 changed files with 123 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
mod_dialogflow_cx/google_glue.cpp
View File

@@ -19,6 +19,8 @@
#include "mod_dialogflow_cx.h"
#include "parser.h"
#define DEFAULT_INTENT "00000000-0000-0000-0000-000000000000"
using google::cloud::dialogflow::cx::v3::Sessions;
using google::cloud::dialogflow::cx::v3::StreamingDetectIntentRequest;
using google::cloud::dialogflow::cx::v3::StreamingDetectIntentResponse;
@@ -112,12 +114,17 @@ void tokenize(std::string const &str, const char delim, std::vector<std::string>
class GStreamer {
public:
GStreamer(switch_core_session_t *session, const char* lang, char* projectId, char* event, char* text) :
m_lang(lang), m_sessionId(switch_core_session_get_uuid(session)), m_environment("draft"), m_regionId("us"),
m_speakingRate(), m_pitch(), m_volume(), m_voiceName(""), m_voiceGender(""), m_effects(""),
m_sentimentAnalysis(false), m_finished(false), m_packets(0) {
GStreamer(switch_core_session_t *session, const char* lang, char* region, char* projectId, char* agentId,
char* environmentId, char* event, char* text) :
m_lang(lang), m_sessionId(switch_core_session_get_uuid(session)), m_agent(agentId), m_projectId(projectId),
m_environment( nullptr != environmentId ? environmentId : "draft"), m_regionId(nullptr != region ? region : "us"),
m_speakingRate(), m_pitch(), m_volume(), m_voiceName(""), m_voiceGender(""), m_effects(""),
m_sentimentAnalysis(false), m_finished(false), m_packets(0) {
const char* var;
switch_channel_t* channel = switch_core_session_get_channel(session);
// TOODO: handle via channel vars
/*
std::vector<std::string> tokens;
const char delim = ':';
tokenize(projectId, delim, tokens);
@@ -136,16 +143,17 @@ public:
else if (9 == idx && s.length() > 0) m_sentimentAnalysis = (s == "true");
idx++;
}
*/
std::string endpoint = "dialogflow.googleapis.com";
if (0 != m_regionId.compare("us")) {
endpoint = m_regionId;
endpoint.append("-dialogflow.googleapis.com:443");
endpoint.append("-dialogflow.googleapis.com");
}
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO,
"GStreamer dialogflow endpoint is %s, region is %s, project is %s, environment is %s\n",
endpoint.c_str(), m_regionId.c_str(), m_projectId.c_str(), m_environment.c_str());
"GStreamer dialogflow endpoint is %s, region is %s, project is %s, agent is %s, environment is %s\n",
endpoint.c_str(), m_regionId.c_str(), m_projectId.c_str(), m_agent.c_str(), m_environment.c_str());
if (var = switch_channel_get_variable(channel, "GOOGLE_APPLICATION_CREDENTIALS")) {
auto callCreds = grpc::ServiceAccountJWTAccessCredentials(var, INT64_MAX);
@@ -172,8 +180,14 @@ public:
m_context= std::make_shared<grpc::ClientContext>();
m_stub = Sessions::NewStub(m_channel);
snprintf(szSession, 256, "projects/%s/locations/%s/agent/environments/%s/users/-/sessions/%s",
if (0 == m_environment.compare("draft")) {
snprintf(szSession, 256, "projects/%s/locations/%s/agents/%s/sessions/%s",
m_projectId.c_str(), m_regionId.c_str(), m_agent.c_str(), m_sessionId.c_str());
}
else {
snprintf(szSession, 256, "projects/%s/locations/%s/agents/%s/environments/%s/sessions/%s",
m_projectId.c_str(), m_regionId.c_str(), m_environment.c_str(), m_sessionId.c_str());
}
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "GStreamer::startStream session %s, event %s, text %s %p\n", szSession, event, text, this);
@@ -190,21 +204,26 @@ public:
queryInput->set_language_code(m_lang.c_str());
}
else {
auto* audio_input = queryInput->mutable_audio();
auto* audio_config = audio_input->mutable_config();
audio_config->set_sample_rate_hertz(16000);
audio_config->set_enable_word_info(false);
audio_config->set_audio_encoding(AudioEncoding::AUDIO_ENCODING_LINEAR_16);
audio_config->set_single_utterance(false);
/**
* Note: there are other parameters that can be set in the audio config, such as:
* hints, model, model variant, barge in config
*
*/
queryInput->set_language_code(m_lang.c_str());
auto* intentInput = queryInput->mutable_intent();
intentInput->set_intent(DEFAULT_INTENT);
}
/*
auto* audio_input = queryInput->mutable_audio();
auto* audio_config = audio_input->mutable_config();
audio_config->set_sample_rate_hertz(16000);
audio_config->set_enable_word_info(false);
audio_config->set_audio_encoding(AudioEncoding::AUDIO_ENCODING_LINEAR_16);
audio_config->set_single_utterance(false);
*/
/**
* Note: there are other parameters that can be set in the audio config, such as:
* hints, model, model variant, barge in config
*
*/
queryInput->set_language_code(m_lang.c_str());
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "GStreamer::startStream checking OutputAudioConfig custom parameters: speaking rate %f,"
" pitch %f, volume %f, voice name '%s' gender '%s', effects '%s'\n", m_speakingRate,
m_pitch, m_volume, m_voiceName.c_str(), m_voiceGender.c_str(), m_effects.c_str());
@@ -254,6 +273,13 @@ public:
m_request->clear_query_input();
m_request->clear_query_params();
auto* audio_config = m_request->mutable_query_input()->mutable_audio()->mutable_config();
audio_config->set_sample_rate_hertz(16000);
audio_config->set_enable_word_info(false);
audio_config->set_audio_encoding(AudioEncoding::AUDIO_ENCODING_LINEAR_16);
audio_config->set_single_utterance(false);
m_request->mutable_query_input()->mutable_audio()->set_audio(data, datalen);
m_packets++;
@@ -293,6 +319,7 @@ private:
std::shared_ptr<StreamingDetectIntentRequest> m_request;
std::string m_lang;
std::string m_projectId;
std::string m_agent;
std::string m_environment;
std::string m_regionId;
double m_speakingRate;
@@ -454,9 +481,12 @@ extern "C" {
errorHandler_t errorHandler,
uint32_t samples_per_second,
char* lang,
char* region,
char* projectId,
char* event,
char* text,
char* agentId,
char* environmentId,
char* event,
char* text,
struct cap_cb **ppUserData
) {
switch_status_t status = SWITCH_STATUS_SUCCESS;
@@ -485,7 +515,10 @@ extern "C" {
strncpy(cb->lang, lang, MAX_LANG);
strncpy(cb->projectId, lang, MAX_PROJECT_ID);
cb->streamer = new GStreamer(session, lang, projectId, event, text);
strncpy(cb->agentId, agentId, MAX_PROJECT_ID);
if (nullptr != environmentId) strncpy(cb->environmentId, environmentId, MAX_PROJECT_ID);
if (nullptr != region) strncpy(cb->region, region, MAX_REGION);
cb->streamer = new GStreamer(session, lang, region, projectId, agentId, environmentId, event, text);
cb->resampler = speex_resampler_init(1, 8000, 16000, SWITCH_RESAMPLE_QUALITY, &err);
if (0 != err) {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "%s: Error initializing resampler: %s.\n",

19
mod_dialogflow_cx/google_glue.h
View File

@@ -1,10 +1,21 @@
#ifndef __GOOGLE_GLUE_H__
#define __GOOGLE_GLUE_H__
#ifndef __GOOGLE_GLUE_CX_H__
#define __GOOGLE_GLUE_CX_H__
switch_status_t google_dialogflow_cx_init();
switch_status_t google_dialogflow_cx_cleanup();
switch_status_t google_dialogflow_cx_session_init(switch_core_session_t *session, responseHandler_t responseHandler, errorHandler_t errorHandler,
uint32_t samples_per_second, char* lang, char* projectId, char* welcomeEvent, char *text, struct cap_cb **cb);
switch_status_t google_dialogflow_cx_session_init(
switch_core_session_t *session,
responseHandler_t responseHandler,
errorHandler_t errorHandler,
uint32_t samples_per_second,
char* lang,
char* region,
char* projectId,
char* agentId,
char* environmentId,
char* event,
char* text,
struct cap_cb **ppUserData);
switch_status_t google_dialogflow_cx_session_stop(switch_core_session_t *session, int channelIsClosing);
switch_bool_t google_dialogflow_cx_frame(switch_media_bug_t *bug, void* user_data);

60
mod_dialogflow_cx/mod_dialogflow_cx.c
View File

@@ -6,10 +6,6 @@
#include "mod_dialogflow_cx.h"
#include "google_glue.h"
#define DEFAULT_INTENT_TIMEOUT_SECS (30)
#define DIALOGFLOW_INTENT "dialogflow_cx_intent"
#define DIALOGFLOW_INTENT_AUDIO_FILE "dialogflow_cx_intent_audio_file"
/* Prototypes */
SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_dialogflow_cx_shutdown);
SWITCH_MODULE_RUNTIME_FUNCTION(mod_dialogflow_cx_runtime);
@@ -74,7 +70,8 @@ static switch_bool_t capture_callback(switch_media_bug_t *bug, void *user_data,
return SWITCH_TRUE;
}
static switch_status_t start_capture(switch_core_session_t *session, switch_media_bug_flag_t flags, char* lang, char*projectId, char* event, char* text)
static switch_status_t start_capture(switch_core_session_t *session, switch_media_bug_flag_t flags, char* lang, char* region, char* projectId,
char *agentId, char *environmentId, char* event, char* text)
{
switch_channel_t *channel = switch_core_session_get_channel(session);
switch_media_bug_t *bug;
@@ -83,7 +80,7 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
switch_status_t status = SWITCH_STATUS_SUCCESS;
if (switch_channel_get_private(channel, MY_BUG_NAME)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "a dialogflow is already running on this channel, we will stop it.\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "a dialogflow_cx is already running on this channel, we will stop it.\n");
do_stop(session);
}
@@ -93,18 +90,18 @@ static switch_status_t start_capture(switch_core_session_t *session, switch_medi
goto done;
}
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "starting dialogflow with project %s, language %s, event %s, text %s.\n",
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "starting dialogflow_cx with project %s, language %s, event %s, text %s.\n",
projectId, lang, event, text);
switch_core_session_get_read_impl(session, &read_impl);
if (SWITCH_STATUS_FALSE == google_dialogflow_cx_session_init(session, responseHandler, errorHandler,
read_impl.samples_per_second, lang, projectId, event, text, &cb)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing google dialogflow session.\n");
read_impl.samples_per_second, lang, region, projectId, agentId, environmentId, event, text, &cb)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error initializing google dialogflow_cx session.\n");
status = SWITCH_STATUS_FALSE;
goto done;
}
if ((status = switch_core_media_bug_add(session, "dialogflow", NULL, capture_callback, (void *) cb, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
if ((status = switch_core_media_bug_add(session, MY_BUG_NAME, NULL, capture_callback, (void *) cb, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error adding bug.\n");
status = SWITCH_STATUS_FALSE;
goto done;
@@ -127,15 +124,15 @@ static switch_status_t do_stop(switch_core_session_t *session)
switch_media_bug_t *bug = switch_channel_get_private(channel, MY_BUG_NAME);
if (bug) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Received user command command to stop dialogflow.\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Received user command command to stop dialogflow_cx.\n");
status = google_dialogflow_cx_session_stop(session, 0);
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "stopped dialogflow.\n");
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "stopped dialogflow_cx.\n");
}
return status;
}
#define DIALOGFLOW_API_START_SYNTAX "<uuid> project-id lang-code [event]"
#define DIALOGFLOW_API_START_SYNTAX "<uuid> region project-id agent-id environment-id lang-code [event] [text]"
SWITCH_STANDARD_API(dialogflow_cx_api_start_function)
{
char *mycmd = NULL, *argv[10] = { 0 };
@@ -148,7 +145,7 @@ SWITCH_STANDARD_API(dialogflow_cx_api_start_function)
argc = switch_separate_string(mycmd, ' ', argv, (sizeof(argv) / sizeof(argv[0])));
}
if (zstr(cmd) || argc < 3) {
if (zstr(cmd) || argc < 6) {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Error with command %s %s %s.\n", cmd, argv[0], argv[1]);
stream->write_function(stream, "-USAGE: %s\n", DIALOGFLOW_API_START_SYNTAX);
goto done;
@@ -158,18 +155,27 @@ SWITCH_STANDARD_API(dialogflow_cx_api_start_function)
if ((lsession = switch_core_session_locate(argv[0]))) {
char *event = NULL;
char *text = NULL;
char *projectId = argv[1];
char *lang = argv[2];
if (argc > 3) {
event = argv[3];
}
if (argc > 4) {
char *region = argv[1];
char *projectId = argv[2];
char *agentId = argv[3];
char *environmentId = argv[4];
char *lang = argv[5];
if (0 == strcmp("default", environmentId)) {
environmentId = NULL;
}
if (0 == strcmp("default", region)) {
region = NULL;
}
if (argc > 6) {
event = argv[6];
if (0 == strcmp("none", event)) {
event = NULL;
}
text = argv[4];
}
status = start_capture(lsession, flags, lang, projectId, event, text);
if (argc > 7) {
text = argv[6];
}
status = start_capture(lsession, flags, lang, region, projectId, agentId, environmentId, event, text);
switch_core_session_rwunlock(lsession);
}
}
@@ -265,12 +271,12 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_dialogflow_cx_load)
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "Google Dialogflow CX API successfully loaded\n");
SWITCH_ADD_API(api_interface, "dialogflow_cx_start", "Start a google dialogflow cx", dialogflow_cx_api_start_function, DIALOGFLOW_API_START_SYNTAX);
SWITCH_ADD_API(api_interface, "dialogflow_CX_stop", "Terminate a google dialogflow cx", dialogflow_cx_api_stop_function, DIALOGFLOW_API_STOP_SYNTAX);
SWITCH_ADD_API(api_interface, "dialogflow_cx_stop", "Terminate a google dialogflow cx", dialogflow_cx_api_stop_function, DIALOGFLOW_API_STOP_SYNTAX);
switch_console_set_complete("add dialogflow_stop_cx");
switch_console_set_complete("add dialogflow_start_cx project lang");
switch_console_set_complete("add dialogflow_start_cx project lang timeout-secs");
switch_console_set_complete("add dialogflow_start_cx project lang timeout-secs event");
switch_console_set_complete("add dialogflow_cx_stop");
switch_console_set_complete("add dialogflow_cx_start project lang");
switch_console_set_complete("add dialogflow_cx_start project lang timeout-secs");
switch_console_set_complete("add dialogflow_cx_start project lang timeout-secs event");
/* indicate that the module should continue to be loaded */
return SWITCH_STATUS_SUCCESS;

7
mod_dialogflow_cx/mod_dialogflow_cx.h
View File

@@ -14,8 +14,9 @@
#define DIALOGFLOW_CX_EVENT_ERROR "dialogflow_cx::error"
#define MAX_LANG (12)
#define MAX_PROJECT_ID (128)
#define MAX_PROJECT_ID (256)
#define MAX_PATHLEN (256)
#define MAX_REGION (56)
/* per-channel data */
typedef void (*responseHandler_t)(switch_core_session_t* session, const char * type, char* json);
@@ -31,7 +32,9 @@ struct cap_cb {
switch_thread_t* thread;
char lang[MAX_LANG];
char projectId[MAX_PROJECT_ID];
char agentId[MAX_PROJECT_ID];
char environmentId[MAX_PROJECT_ID];
char region[MAX_REGION];
};
#endif

10
mod_dialogflow_cx/parser.cpp
View File

@@ -198,6 +198,16 @@ cJSON* GRPCParser::parse(const AdvancedSettings_LoggingSettings& o) {
return json;
}
cJSON* GRPCParser::parse(const SentimentAnalysisResult& o) {
cJSON * json = cJSON_CreateObject();
cJSON_AddNumberToObject(json, "score", o.score());
cJSON_AddNumberToObject(json, "magnitude", o.magnitude());
return json;
}
cJSON* GRPCParser::parse(const GcsDestination& o) {
cJSON * json = cJSON_CreateObject();