FS-11014 [core] add vad to core

src/include/switch.h

@@ -145,6 +145,7 @@
 #include "switch_core_video.h"
 #include "switch_jitterbuffer.h"
 #include "switch_estimators.h"
+#include "switch_vad.h"
 #include <libteletone.h>
 
 

src/include/switch_types.h

@@ -622,6 +622,14 @@ typedef enum {
 } switch_vad_flag_enum_t;
 typedef uint32_t switch_vad_flag_t;
 
+typedef enum {
+	SWITCH_VAD_STATE_NONE,
+	SWITCH_VAD_STATE_START_TALKING,
+	SWITCH_VAD_STATE_TALKING,
+	SWITCH_VAD_STATE_STOP_TALKING,
+	SWITCH_VAD_STATE_ERROR
+} switch_vad_state_t;
+typedef struct switch_vad_s switch_vad_t;
 
 typedef struct error_period {
 	int64_t start;

src/include/switch_vad.h

@@ -0,0 +1,66 @@
+/*
+ * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ * Copyright (C) 2018, Anthony Minessale II <anthm@freeswitch.org>
+ *
+ * Version: MPL 1.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ *
+ * The Initial Developer of the Original Code is
+ * Anthony Minessale II <anthm@freeswitch.org>
+ * Portions created by the Initial Developer are Copyright (C)
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Seven Du <dujinfang@gmail.com>
+ *
+ *
+ * switch_vad.h VAD code with optional libfvad
+ *
+ */
+/*!
+  \defgroup vad1 VAD code with optional libfvad
+  \ingroup core1
+  \{
+*/
+#ifndef FREESWITCH_VAD_H
+#define FREESWITCH_VAD_H
+
+SWITCH_BEGIN_EXTERN_C
+
+SWITCH_DECLARE(switch_vad_t *) switch_vad_init(int sample_rate, int channels);
+
+/*
+ * Valid modes are -1 ("disable fvad, using native"), 0 ("quality"), 1 ("low bitrate"), 2 ("aggressive"), and 3 * ("very aggressive").
+ * The default mode is -1.
+*/
+
+SWITCH_DECLARE(int) switch_vad_set_mode(switch_vad_t *vad, int mode);
+SWITCH_DECLARE(void) switch_vad_set_param(switch_vad_t *vad, const char *key, int val);
+SWITCH_DECLARE(switch_vad_state_t) switch_vad_process(switch_vad_t *vad, int16_t *data, unsigned int samples);
+SWITCH_DECLARE(void) switch_vad_reset(switch_vad_t *vad);
+SWITCH_DECLARE(void) switch_vad_destroy(switch_vad_t **vad);
+
+SWITCH_END_EXTERN_C
+#endif
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */

src/mod/applications/mod_dptools/mod_dptools.c

@@ -6128,6 +6128,73 @@ SWITCH_STANDARD_APP(deduplicate_dtmf_app_function)
 	}
 }
 
+SWITCH_STANDARD_APP(vad_test_function)
+{
+	switch_channel_t *channel = switch_core_session_get_channel(session);
+	switch_codec_implementation_t imp = { 0 };
+	switch_vad_t *vad;
+	switch_frame_t *frame = { 0 };
+	switch_vad_state_t vad_state;
+	int mode = -1;
+	const char *var = NULL;
+	int tmp;
+
+	if (!zstr(data)) {
+		mode = atoi(data);
+
+		if (mode > 3) mode = 3;
+	}
+
+	switch_core_session_raw_read(session);
+	switch_core_session_get_read_impl(session, &imp);
+
+	vad = switch_vad_init(imp.samples_per_second, imp.number_of_channels);
+	switch_assert(vad);
+	switch_vad_set_mode(vad, mode);
+
+	if ((var = switch_channel_get_variable(channel, "vad_hangover_len"))) {
+		tmp = atoi(var);
+
+		if (tmp > 0) switch_vad_set_param(vad, "hangover_len", tmp);
+	}
+
+	if ((var = switch_channel_get_variable(channel, "vad_thresh"))) {
+		tmp = atoi(var);
+
+		if (tmp > 0) switch_vad_set_param(vad, "thresh", tmp);
+	}
+
+	if ((var = switch_channel_get_variable(channel, "vad_timeout_len"))) {
+		tmp = atoi(var);
+
+		if (tmp > 0) switch_vad_set_param(vad, "timeout_len", tmp);
+	}
+
+	while(switch_channel_ready(channel)) {
+		switch_core_session_read_frame(session, &frame, SWITCH_IO_FLAG_NONE, 0);
+
+		if (switch_test_flag(frame, SFF_CNG)) {
+			continue;
+		}
+
+		vad_state = switch_vad_process(vad, frame->data, frame->datalen / 2);
+
+		if (vad_state == SWITCH_VAD_STATE_START_TALKING) {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "START TALKING\n");
+			switch_core_session_write_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
+		} else if (vad_state == SWITCH_VAD_STATE_STOP_TALKING) {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "STOP TALKING\n");
+		} else if (vad_state == SWITCH_VAD_STATE_TALKING) {
+			switch_core_session_write_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
+		} else {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "vad_state: %d\n", vad_state);
+		}
+	}
+
+	switch_vad_destroy(&vad);
+	switch_core_session_reset(session, SWITCH_TRUE, SWITCH_TRUE);
+}
+
 #define SPEAK_DESC "Speak text to a channel via the tts interface"
 #define DISPLACE_DESC "Displace audio from a file to the channels input"
 #define SESS_REC_DESC "Starts a background recording of the entire session"
@@ -6462,6 +6529,7 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_dptools_load)
 	SWITCH_ADD_APP(app_interface, "pickup", "Pickup", "Pickup a call", pickup_function, PICKUP_SYNTAX, SAF_SUPPORT_NOMEDIA);
 	SWITCH_ADD_APP(app_interface, "deduplicate_dtmf", "Prevent duplicate inband + 2833 dtmf", "", deduplicate_dtmf_app_function, "[only_rtp]", SAF_SUPPORT_NOMEDIA);
 
+	SWITCH_ADD_APP(app_interface, "vad_test", "VAD test", "VAD test, mode = -1(default), 0, 1, 2, 3", vad_test_function, "[mode]", SAF_NONE);
 
 	SWITCH_ADD_DIALPLAN(dp_interface, "inline", inline_dialplan_hunt);
 

src/switch_vad.c

@@ -0,0 +1,219 @@
+/*
+ * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ * Copyright (C) 2018, Anthony Minessale II <anthm@freeswitch.org>
+ *
+ * Version: MPL 1.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ *
+ * The Initial Developer of the Original Code is
+ * Anthony Minessale II <anthm@freeswitch.org>
+ * Portions created by the Initial Developer are Copyright (C)
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Seven Du <dujinfang@gmail.com>
+ *
+ *
+ * switch_vad.c VAD code with optional libfvad
+ *
+ */
+
+#include <switch.h>
+
+#ifdef SWITCH_HAVE_FVAD
+#include <fvad.h>
+#endif
+
+struct switch_vad_s {
+	int talking;
+	int talked;
+	int talk_hits;
+	int hangover;
+	int hangover_len;
+	int divisor;
+	int thresh;
+	int timeout_len;
+	int timeout;
+	int channels;
+	int sample_rate;
+
+	int _hangover_len;
+	int _thresh;
+	int _timeout_len;
+#ifdef SWITCH_HAVE_FVAD
+	Fvad *fvad;
+#endif
+};
+
+SWITCH_DECLARE(switch_vad_t *) switch_vad_init(int sample_rate, int channels)
+{
+	switch_vad_t *vad = malloc(sizeof(switch_vad_t));
+
+	if (!vad) return NULL;
+
+	memset(vad, 0, sizeof(*vad));
+	vad->sample_rate = sample_rate ? sample_rate : 8000;
+	vad->channels = channels;
+	vad->_hangover_len = 25;
+	vad->_thresh = 100;
+	vad->_timeout_len = 25;
+
+	switch_vad_reset(vad);
+
+	return vad;
+}
+
+//Valid modes are 0 ("quality"), 1 ("low bitrate"), 2 ("aggressive"), and 3 * ("very aggressive"). The default mode is 0.
+SWITCH_DECLARE(int) switch_vad_set_mode(switch_vad_t *vad, int mode)
+{
+#ifdef SWITCH_HAVE_FVAD
+	int ret;
+
+	if (mode < 0) return 0;
+
+	vad->fvad = fvad_new();
+	ret = fvad_set_mode(vad->fvad, mode);
+	fvad_set_sample_rate(vad->fvad, vad->sample_rate);
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "libfvad started, mode = %d\n", mode);
+	return ret;
+#else
+	return 0;
+#endif
+}
+
+SWITCH_DECLARE(void) switch_vad_set_param(switch_vad_t *vad, const char *key, int val)
+{
+	if (!key) return;
+
+	if (!strcmp(key, "hangover_len")) {
+		vad->hangover_len = vad->_hangover_len = val;
+	} else if (!strcmp(key, "thresh")) {
+		vad->thresh = vad->_thresh = val;
+	} else if (!strcmp(key, "timeout_len")) {
+		vad->timeout = vad->timeout_len = vad->_timeout_len = val;
+	}
+}
+
+SWITCH_DECLARE(void) switch_vad_reset(switch_vad_t *vad)
+{
+#ifdef SWITCH_HAVE_FVAD
+	if (vad->fvad) {
+		fvad_reset(vad->fvad);
+		return;
+	}
+#endif
+
+	vad->talking = 0;
+	vad->talked = 0;
+	vad->talk_hits = 0;
+	vad->hangover = 0;
+	vad->hangover_len = vad->_hangover_len;
+	vad->divisor = vad->sample_rate / 8000;
+	vad->thresh = vad->_thresh;
+	vad->timeout_len = vad->_timeout_len;
+	vad->timeout = vad->timeout_len;
+}
+
+SWITCH_DECLARE(switch_vad_state_t) switch_vad_process(switch_vad_t *vad, int16_t *data, unsigned int samples)
+{
+	int energy = 0, j = 0, count = 0;
+	int score = 0;
+	switch_vad_state_t vad_state = SWITCH_VAD_STATE_NONE;
+
+#ifdef SWITCH_HAVE_FVAD
+	if (vad->fvad) {
+		int ret = fvad_process(vad->fvad, data, samples);
+
+		// printf("%d ", ret); fflush(stdout);
+
+		score = vad->thresh + ret - 1;
+	} else {
+#endif
+
+	for (energy = 0, j = 0, count = 0; count < samples; count++) {
+		energy += abs(data[j]);
+		j += vad->channels;
+	}
+
+	score = (uint32_t) (energy / (samples / vad->divisor));
+
+#ifdef SWITCH_HAVE_FVAD
+	}
+#endif
+
+	// printf("%d ", score); fflush(stdout);
+	// printf("yay %d %d %d\n", score, vad->hangover, vad->talking);
+
+	if (vad->talking && score < vad->thresh) {
+		if (vad->hangover > 0) {
+			vad->hangover--;
+		} else {// if (hangover <= 0) {
+			vad->talking = 0;
+			vad->talk_hits = 0;
+			vad->hangover = 0;
+		}
+	} else {
+		if (score >= vad->thresh) {
+			vad_state = vad->talking ? SWITCH_VAD_STATE_TALKING : SWITCH_VAD_STATE_START_TALKING;
+			vad->talking = 1;
+			vad->hangover = vad->hangover_len;
+		}
+	}
+
+	// printf("WTF %d %d %d\n", score, vad->talked, vad->talking);
+
+	if (vad->talking) {
+		vad->talk_hits++;
+		// printf("WTF %d %d %d\n", vad->talking, vad->talk_hits, vad->talked);
+		if (vad->talk_hits > 10) {
+			vad->talked = 1;
+			vad_state = SWITCH_VAD_STATE_TALKING;
+		}
+	} else {
+		vad->talk_hits = 0;
+	}
+
+	if (vad->timeout > 0 && !vad->talking) {
+		vad->timeout--;
+	}
+
+	if ((vad->talked && !vad->talking)) {
+		// printf("NOT TALKING ANYMORE\n");
+		vad->talked = 0;
+		vad->timeout = vad->timeout_len;
+		vad_state = SWITCH_VAD_STATE_STOP_TALKING;
+	} else {
+		// if (vad->skip > 0) {
+		// 	vad->skip--;
+		// }
+	}
+
+	if (vad_state) return vad_state;
+
+	return SWITCH_VAD_STATE_NONE;
+}
+
+SWITCH_DECLARE(void) switch_vad_destroy(switch_vad_t **vad)
+{
+	if (*vad) {
+
+#ifdef SWITCH_HAVE_FVAD
+		if ((*vad)->fvad) free ((*vad)->fvad);
+#endif
+
+		free(*vad);
+		*vad = NULL;
+	}
+}