fix merge

This commit is contained in:
Anthony Minessale
2016-09-27 16:44:25 -05:00
1137 changed files with 118714 additions and 203850 deletions
+1
View File
@@ -144,6 +144,7 @@
#include "switch_core_media.h"
#include "switch_core_video.h"
#include "switch_jitterbuffer.h"
#include "switch_estimators.h"
#include <libteletone.h>
+4
View File
@@ -325,6 +325,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess
switch_codec_control_type_t *rtype,
void **ret_data);
SWITCH_DECLARE(switch_bool_t) switch_core_media_codec_get_cap(switch_core_session_t *session,
switch_media_type_t mtype,
switch_codec_flag_t flag);
#define switch_core_media_gen_key_frame(_session) switch_core_media_codec_control(_session, SWITCH_MEDIA_TYPE_VIDEO, SWITCH_IO_WRITE, \
SCC_VIDEO_GEN_KEYFRAME, SCCT_NONE, NULL, SCCT_NONE, NULL, NULL, NULL) \
+100
View File
@@ -0,0 +1,100 @@
/*
* FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
* Copyright (C) 2005-2015, Anthony Minessale II <anthm@freeswitch.org>
*
* Version: MPL 1.1
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
*
* The Initial Developer of the Original Code is
* Anthony Minessale II <anthm@freeswitch.org>
* Portions created by the Initial Developer are Copyright (C)
* the Initial Developer. All Rights Reserved.
*
* Contributor(s):
*
* Dragos Oancea <droancea@yahoo.com>
*
* switch_estimators.h -- Estimators for Packet Loss, Jitter, RTT , etc
*
*/
#ifndef SWITCH_ESTIMATORS_H
#define SWITCH_ESTIMATORS_H
#include <switch.h>
SWITCH_BEGIN_EXTERN_C
struct kalman_estimator_s {
/* initial values for the Kalman filter */
float val_estimate_last ;
float P_last ;
/* the noise in the system:
The amount of noise in your measurements and the state-transitions
(e.g. the standard deviation of the signal noise, and how 'wrong' your simplified model
of the state-transitions are) => These are Q and R matrices */
float Q ; /* the process noise covariance matrix */
float R ; /* the measurement noise covariance matrix */
float K; /* P_temp * H^T * (H* P_temp * H^T + R)^-1 */
float P; /* the Kalman gain (calculated) */
float val_estimate; /* x_temp_est + K * (z_measured - H * x_temp_est) */
float val_measured; /* the 'noisy' value we measured */
};
struct cusum_kalman_detector_s {
/* initial values for the CUSUM Kalman filter */
float val_estimate_last;
float val_desired_last;
float P_last;
float K_last;
float delta;
float measurement_noise_e;
float variance_Re;
float measurement_noise_v;
float variance_Rv;
float g_last;
/*constants per model*/
float epsilon;
float h;
/* for calculating variance */
float last_average;
float last_q;
float N; /*how many samples we have so far (eg: how many RTCP we received, granted that we can calculate RTT for each one of them)*/
};
typedef struct kalman_estimator_s kalman_estimator_t;
typedef struct cusum_kalman_detector_s cusum_kalman_detector_t;
SWITCH_DECLARE(void) switch_kalman_init(kalman_estimator_t *est, float Q, float R);
SWITCH_DECLARE(switch_bool_t) switch_kalman_cusum_init(cusum_kalman_detector_t *detect_change, float epsilon,float h);
SWITCH_DECLARE(switch_bool_t) switch_kalman_estimate(kalman_estimator_t * est, float measurement, int system_model);
SWITCH_DECLARE (switch_bool_t) switch_kalman_cusum_detect_change(cusum_kalman_detector_t * detector, float measurement, float rtt_avg);
SWITCH_DECLARE(switch_bool_t) switch_kalman_is_slow_link(kalman_estimator_t * est_loss, kalman_estimator_t * est_rtt);
SWITCH_END_EXTERN_C
#endif
/* For Emacs:
* Local Variables:
* mode:c
* indent-tabs-mode:t
* tab-width:4
* c-basic-offset:4
* End:
* For VIM:
* vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
*/
+10
View File
@@ -759,6 +759,8 @@ typedef enum {
SWITCH_ZRTP_FLAG_SECURE_MITM_RECV,
SWITCH_RTP_FLAG_DEBUG_RTP_READ,
SWITCH_RTP_FLAG_DEBUG_RTP_WRITE,
SWITCH_RTP_FLAG_ESTIMATORS,
SWITCH_RTP_FLAG_ADJ_BITRATE_CAP,
SWITCH_RTP_FLAG_VIDEO,
SWITCH_RTP_FLAG_ENABLE_RTCP,
SWITCH_RTP_FLAG_RTCP_MUX,
@@ -1648,6 +1650,7 @@ typedef enum {
SWITCH_CODEC_FLAG_AAL2 = (1 << 6),
SWITCH_CODEC_FLAG_PASSTHROUGH = (1 << 7),
SWITCH_CODEC_FLAG_READY = (1 << 8),
SWITCH_CODEC_FLAG_HAS_ADJ_BITRATE = (1 << 14),
SWITCH_CODEC_FLAG_HAS_PLC = (1 << 15),
SWITCH_CODEC_FLAG_VIDEO_PATCHING = (1 << 16)
} switch_codec_flag_enum_t;
@@ -2290,6 +2293,7 @@ typedef enum {
SCC_VIDEO_BANDWIDTH,
SCC_VIDEO_RESET,
SCC_AUDIO_PACKET_LOSS,
SCC_AUDIO_ADJUST_BITRATE,
SCC_DEBUG,
SCC_CODEC_SPECIFIC
} switch_codec_control_command_t;
@@ -2608,6 +2612,12 @@ typedef struct switch_waitlist_s {
struct switch_jb_s;
typedef struct switch_jb_s switch_jb_t;
//struct kalman_estimator_s;
//typedef struct kalman_estimator_s kalman_estimator_t;
//struct cusum_kalman_detector_s;
//typedef struct cusum_kalman_detector_s cusum_kalman_detector_t;
struct switch_img_txt_handle_s;
typedef struct switch_img_txt_handle_s switch_img_txt_handle_t;
@@ -1336,7 +1336,7 @@ void conference_loop_output(conference_member_t *member)
switch_change_sln_volume(write_frame.data, write_frame.samples * member->conference->channels, member->volume_out_level);
}
write_frame.timestamp = timer.samplecount;
//write_frame.timestamp = timer.samplecount;
if (member->fnode) {
conference_member_add_file_data(member, write_frame.data, write_frame.datalen);
+81 -3
View File
@@ -34,6 +34,11 @@
#include "switch.h"
#include "opus.h"
#define SWITCH_OPUS_MIN_BITRATE 6000
#define SWITCH_OPUS_MAX_BITRATE 510000
#define SWITCH_OPUS_MIN_FEC_BITRATE 12400
SWITCH_MODULE_LOAD_FUNCTION(mod_opus_load);
SWITCH_MODULE_DEFINITION(mod_opus, mod_opus_load, NULL, NULL);
@@ -91,6 +96,15 @@ struct dec_stats {
};
typedef struct dec_stats dec_stats_t;
struct codec_control_state {
int keep_fec;
opus_int32 current_bitrate;
opus_int32 wanted_bitrate;
uint32_t increase_step;
uint32_t decrease_step;
};
typedef struct codec_control_state codec_control_state_t;
struct opus_context {
OpusEncoder *encoder_object;
OpusDecoder *decoder_object;
@@ -103,6 +117,7 @@ struct opus_context {
int look_check;
int look_ts;
dec_stats_t decoder_stats;
codec_control_state_t control_state;
};
struct {
@@ -116,6 +131,7 @@ struct {
int asymmetric_samplerates;
int keep_fec;
int fec_decode;
int adjust_bitrate;
int debuginfo;
uint32_t use_jb_lookahead;
switch_mutex_t *mutex;
@@ -253,7 +269,7 @@ static switch_status_t switch_opus_fmtp_parse(const char *fmtp, switch_codec_fmt
if (!strcasecmp(data, "maxaveragebitrate")) {
codec_settings->maxaveragebitrate = atoi(arg);
if (codec_settings->maxaveragebitrate < 6000 || codec_settings->maxaveragebitrate > 510000) {
if (codec_settings->maxaveragebitrate < SWITCH_OPUS_MIN_BITRATE || codec_settings->maxaveragebitrate > SWITCH_OPUS_MAX_BITRATE) {
codec_settings->maxaveragebitrate = 0; /* values outside the range between 6000 and 510000 SHOULD be ignored */
}
}
@@ -600,6 +616,7 @@ static switch_status_t switch_opus_init(switch_codec_t *codec, switch_codec_flag
opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(fec_bitrate));
/* will override the maxaveragebitrate set in opus.conf.xml */
opus_codec_settings.maxaveragebitrate = fec_bitrate;
context->control_state.keep_fec = opus_prefs.keep_fec ;
}
}
}
@@ -607,6 +624,10 @@ static switch_status_t switch_opus_init(switch_codec_t *codec, switch_codec_flag
if (opus_codec_settings.usedtx) {
opus_encoder_ctl(context->encoder_object, OPUS_SET_DTX(opus_codec_settings.usedtx));
}
if (opus_prefs.adjust_bitrate) {
switch_set_flag(codec, SWITCH_CODEC_FLAG_HAS_ADJ_BITRATE);
}
}
if (decoding) {
@@ -972,9 +993,11 @@ static switch_status_t opus_load_config(switch_bool_t reload)
opus_prefs.keep_fec = atoi(val);
} else if (!strcasecmp(key, "advertise-useinbandfec")) { /*decoder, has meaning only for FMTP: useinbandfec=1 by default */
opus_prefs.fec_decode = atoi(val);
} else if (!strcasecmp(key, "adjust-bitrate")) { /* encoder, this setting will make the encoder adjust its bitrate based on a feedback loop (RTCP). This is not "VBR".*/
opus_prefs.adjust_bitrate = atoi(val);
} else if (!strcasecmp(key, "maxaveragebitrate")) {
opus_prefs.maxaveragebitrate = atoi(val);
if (opus_prefs.maxaveragebitrate < 6000 || opus_prefs.maxaveragebitrate > 510000) {
if (opus_prefs.maxaveragebitrate < SWITCH_OPUS_MIN_BITRATE || opus_prefs.maxaveragebitrate > SWITCH_OPUS_MAX_BITRATE) {
opus_prefs.maxaveragebitrate = 0; /* values outside the range between 6000 and 510000 SHOULD be ignored */
}
} else if (!strcasecmp(key, "maxplaybackrate")) {
@@ -1132,13 +1155,68 @@ static switch_status_t switch_opus_control(switch_codec_t *codec,
}
if (globals.debug || context->debug) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus Adjusting packet loss percent from %d%% to %d%%!\n",
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting packet loss percent from %d%% to %d%%!\n",
context->old_plpct, plpct);
}
}
context->old_plpct = plpct;
}
break;
case SCC_AUDIO_ADJUST_BITRATE:
{
const char *cmd = (const char *)cmd_data;
if (!zstr(cmd)) {
opus_int32 current_bitrate=context->control_state.current_bitrate;
if (!strcasecmp(cmd, "increase")) {
/* https://wiki.xiph.org/OpusFAQ
"[...]Opus scales from about 6 to 512 kb/s, in increments of 0.4 kb/s (one byte with 20 ms frames).
Opus can have more than 1200 possible bitrates[...]" */
int br_step = context->control_state.increase_step?context->control_state.increase_step:400;
opus_encoder_ctl(context->encoder_object, OPUS_GET_BITRATE(&current_bitrate));
if (opus_prefs.maxaveragebitrate > current_bitrate) {
opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(current_bitrate+br_step));
if ((context->control_state.keep_fec) && (current_bitrate > SWITCH_OPUS_MIN_FEC_BITRATE)) {
opus_prefs.keep_fec = 1; /* enable back FEC if it was disabled by SCC_AUDIO_ADJUST_BITRATE, we have enough network bandwidth now */
}
if (globals.debug || context->debug) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (increase)\n", current_bitrate+br_step);
}
}
} else if (!strcasecmp(cmd, "decrease")) {
int br_step = context->control_state.decrease_step?context->control_state.decrease_step:400;
opus_encoder_ctl(context->encoder_object, OPUS_GET_BITRATE(&current_bitrate));
if (current_bitrate > SWITCH_OPUS_MIN_BITRATE) {
if ((context->control_state.keep_fec) && (current_bitrate < SWITCH_OPUS_MIN_FEC_BITRATE)) {
opus_prefs.keep_fec = 0; /* no point to try to keep FEC enabled anymore, we're low on network bandwidth (that's why we ended up here) */
}
opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(current_bitrate-br_step));
if (globals.debug || context->debug) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (decrease)\n", current_bitrate-br_step);
}
}
} else if (!strcasecmp(cmd, "default")) {
/*restore default bitrate */
opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(opus_prefs.maxaveragebitrate));
if (context->control_state.keep_fec) {
opus_prefs.keep_fec = 1; /* enable back FEC, we have enough network bandwidth now */
}
if (globals.debug || context->debug) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (configured maxaveragebitrate)\n", opus_prefs.maxaveragebitrate);
}
} else {
/* set Opus minimum bitrate */
opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(SWITCH_OPUS_MIN_BITRATE));
if (context->control_state.keep_fec) {
opus_prefs.keep_fec = 0; /* do not enforce FEC anymore, we're low on network bandwidth */
}
if (globals.debug || context->debug) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (minimum)\n", SWITCH_OPUS_MIN_BITRATE);
}
}
}
}
break;
default:
break;
}
+4 -2
View File
@@ -4840,11 +4840,13 @@ switch_status_t config_sofia(sofia_config_t reload, char *profile_name)
} else {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid ext-rtp-ip\n");
}
} else if (!strcasecmp(var, "rtp-ip") && val) {
} else if (!strcasecmp(var, "rtp-ip")) {
char *ip = mod_sofia_globals.guess_ip;
char buf[64];
if (!strcmp(val, "0.0.0.0")) {
if (zstr(val)) {
ip = mod_sofia_globals.guess_ip;
} else if (!strcmp(val, "0.0.0.0")) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Invalid IP 0.0.0.0 replaced with %s\n", mod_sofia_globals.guess_ip);
} else if (!strncasecmp(val, "interface:", 10)) {
char *ifname = val+10;
+1 -1
View File
@@ -1826,7 +1826,7 @@ static void client_run(jsock_t *jsock)
ws_close(&jsock->ws, WS_NONE);
goto error;
} else {
die("%s WS SETUP FAILED [%s]\n", jsock->name, jsock->ws.buffer);
die("%s WS SETUP FAILED\n", jsock->name);
}
}
+1 -1
View File
@@ -111,7 +111,7 @@ SWITCH_DECLARE(void *) switch_core_hash_delete_locked(switch_hash_t *hash, const
switch_mutex_lock(mutex);
}
switch_core_hash_delete(hash, key);
ret = switch_core_hash_delete(hash, key);
if (mutex) {
switch_mutex_unlock(mutex);
+211 -81
View File
@@ -376,7 +376,7 @@ SWITCH_DECLARE(uint32_t) switch_core_media_get_video_fps(switch_core_session_t *
{
switch_media_handle_t *smh;
time_t now;
uint32_t fps;
uint32_t fps, elapsed = 0;
switch_assert(session);
@@ -390,11 +390,13 @@ SWITCH_DECLARE(uint32_t) switch_core_media_get_video_fps(switch_core_session_t *
now = switch_epoch_time_now(NULL);
if (!(smh->vid_started && smh->vid_frames && smh->vid_started < now)) {
elapsed = now - smh->vid_started;
if (!(smh->vid_started && smh->vid_frames && elapsed > 0)) {
return 0;
}
fps = switch_round_to_step(smh->vid_frames / (now - smh->vid_started), 5);
fps = switch_round_to_step(smh->vid_frames / (elapsed), 5);
if (smh->vid_frames > 1000) {
smh->vid_started = switch_epoch_time_now(NULL);
@@ -2186,7 +2188,7 @@ static void check_jb_sync(switch_core_session_t *session)
switch_rtp_get_video_buffer_size(v_engine->rtp_session, &min_frames, &max_frames, &cur_frames, NULL);
fps = switch_core_media_get_video_fps(session);
fps = video_globals.fps;
if (!fps) return;
@@ -2508,23 +2510,23 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_read_frame(switch_core_session
goto end;
}
if (type == SWITCH_MEDIA_TYPE_VIDEO && engine->read_frame.m) {
if (!smh->vid_started) {
smh->vid_started = switch_epoch_time_now(NULL);
}
smh->vid_frames++;
if ((smh->vid_frames % 15) == 0) {
switch_core_media_get_video_fps(session);
}
if (smh->vid_frames == 1 || ((smh->vid_frames % 300) == 0)) {
check_jb_sync(session);
if (type == SWITCH_MEDIA_TYPE_VIDEO) {
if (engine->read_frame.m) {
if (!smh->vid_started) {
smh->vid_started = switch_epoch_time_now(NULL);
}
smh->vid_frames++;
if ((smh->vid_frames % 5) == 0) {
switch_core_media_get_video_fps(session);
}
if (video_globals.fps && (!video_globals.synced || ((smh->vid_frames % 300) == 0))) {
check_jb_sync(session);
}
}
}
/* re-set codec if necessary */
if (type != SWITCH_MEDIA_TYPE_TEXT && engine->reset_codec > 0) {
const char *val;
@@ -4103,7 +4105,7 @@ SWITCH_DECLARE(uint8_t) switch_core_media_negotiate_sdp(switch_core_session_t *s
int codec_ms = 0;
uint32_t remote_codec_rate = 0, fmtp_remote_codec_rate = 0;
const char *tmp;
int m_idx = 0;
int m_idx = 0, skip_rtcp = 0, skip_video_rtcp = 0;
int nm_idx = 0;
int vmatch_pt = 0;
int rtcp_auto_audio = 0, rtcp_auto_video = 0;
@@ -4606,11 +4608,18 @@ SWITCH_DECLARE(uint8_t) switch_core_media_negotiate_sdp(switch_core_session_t *s
}
}
skip_rtcp = 0;
for (attr = m->m_attributes; attr; attr = attr->a_next) {
if (!strcasecmp(attr->a_name, "rtcp-mux") || !strcasecmp(attr->a_name, "ice-ufrag")) {
skip_rtcp = 1;
}
}
if (!strcasecmp(attr->a_name, "rtcp") && attr->a_value) {
switch_channel_set_variable(session->channel, "rtp_remote_audio_rtcp_port", attr->a_value);
for (attr = m->m_attributes; attr; attr = attr->a_next) {
if (!strcasecmp(attr->a_name, "rtcp") && attr->a_value && !skip_rtcp) {
a_engine->remote_rtcp_port = (switch_port_t)atoi(attr->a_value);
switch_channel_set_variable_printf(session->channel, "rtp_remote_audio_rtcp_port", "%d", a_engine->remote_rtcp_port);
if (!smh->mparams->rtcp_audio_interval_msec) {
smh->mparams->rtcp_audio_interval_msec = SWITCH_RTCP_AUDIO_INTERVAL_MSEC;
}
@@ -5254,6 +5263,67 @@ SWITCH_DECLARE(uint8_t) switch_core_media_negotiate_sdp(switch_core_session_t *s
}
}
skip_video_rtcp = 0;
for (attr = m->m_attributes; attr; attr = attr->a_next) {
if (!strcasecmp(attr->a_name, "rtcp-mux") || !strcasecmp(attr->a_name, "ice-ufrag")) {
skip_video_rtcp = 1;
}
}
for (attr = m->m_attributes; attr; attr = attr->a_next) {
if (!strcasecmp(attr->a_name, "framerate") && attr->a_value) {
//framerate = atoi(attr->a_value);
} else if (!strcasecmp(attr->a_name, "rtcp-fb")) {
if (!zstr(attr->a_value)) {
if (switch_stristr("fir", attr->a_value)) {
v_engine->fir++;
}
if (switch_stristr("pli", attr->a_value)) {
v_engine->pli++;
}
if (switch_stristr("nack", attr->a_value)) {
v_engine->nack++;
}
if (switch_stristr("tmmbr", attr->a_value)) {
v_engine->tmmbr++;
}
rtcp_auto_video = 1;
smh->mparams->rtcp_video_interval_msec = SWITCH_RTCP_VIDEO_INTERVAL_MSEC;
}
} else if (!strcasecmp(attr->a_name, "rtcp") && attr->a_value && !skip_video_rtcp) {
v_engine->remote_rtcp_port = (switch_port_t)atoi(attr->a_value);
switch_channel_set_variable_printf(session->channel, "rtp_remote_video_rtcp_port", "%d", v_engine->remote_rtcp_port);
if (!smh->mparams->rtcp_video_interval_msec) {
smh->mparams->rtcp_video_interval_msec = SWITCH_RTCP_VIDEO_INTERVAL_MSEC;
}
got_video_rtcp = 1;
} else if (!got_video_crypto && !strcasecmp(attr->a_name, "crypto") && !zstr(attr->a_value)) {
int crypto_tag;
if (!(smh->mparams->ndlb & SM_NDLB_ALLOW_CRYPTO_IN_AVP) &&
!switch_true(switch_channel_get_variable(session->channel, "rtp_allow_crypto_in_avp"))) {
if (m->m_proto != sdp_proto_srtp && !got_webrtc) {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "a=crypto in RTP/AVP, refer to rfc3711\n");
match = 0;
goto done;
}
}
crypto = attr->a_value;
crypto_tag = atoi(crypto);
got_video_crypto = switch_core_session_check_incoming_crypto(session,
"rtp_has_video_crypto",
SWITCH_MEDIA_TYPE_VIDEO, crypto, crypto_tag, sdp_type);
}
}
if (switch_true(switch_channel_get_variable_dup(session->channel, "inherit_codec", SWITCH_FALSE, -1))) {
vmatch_pt = 1;
}
@@ -5270,58 +5340,6 @@ SWITCH_DECLARE(uint8_t) switch_core_media_negotiate_sdp(switch_core_session_t *s
}
}
for (attr = m->m_attributes; attr; attr = attr->a_next) {
if (!strcasecmp(attr->a_name, "framerate") && attr->a_value) {
//framerate = atoi(attr->a_value);
} else if (!strcasecmp(attr->a_name, "rtcp-fb")) {
if (!zstr(attr->a_value)) {
if (switch_stristr("fir", attr->a_value)) {
v_engine->fir++;
}
if (switch_stristr("pli", attr->a_value)) {
v_engine->pli++;
}
if (switch_stristr("nack", attr->a_value)) {
v_engine->nack++;
}
if (switch_stristr("tmmbr", attr->a_value)) {
v_engine->tmmbr++;
}
rtcp_auto_video = 1;
}
} else if (!strcasecmp(attr->a_name, "rtcp") && attr->a_value) {
switch_channel_set_variable(session->channel, "rtp_remote_video_rtcp_port", attr->a_value);
v_engine->remote_rtcp_port = (switch_port_t)atoi(attr->a_value);
if (!smh->mparams->rtcp_video_interval_msec) {
smh->mparams->rtcp_video_interval_msec = SWITCH_RTCP_VIDEO_INTERVAL_MSEC;
}
got_video_rtcp = 1;
} else if (!got_video_crypto && !strcasecmp(attr->a_name, "crypto") && !zstr(attr->a_value)) {
int crypto_tag;
if (!(smh->mparams->ndlb & SM_NDLB_ALLOW_CRYPTO_IN_AVP) &&
!switch_true(switch_channel_get_variable(session->channel, "rtp_allow_crypto_in_avp"))) {
if (m->m_proto != sdp_proto_srtp && !got_webrtc) {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "a=crypto in RTP/AVP, refer to rfc3711\n");
match = 0;
goto done;
}
}
crypto = attr->a_value;
crypto_tag = atoi(crypto);
got_video_crypto = switch_core_session_check_incoming_crypto(session,
"rtp_has_video_crypto",
SWITCH_MEDIA_TYPE_VIDEO, crypto, crypto_tag, sdp_type);
}
}
if (!(rm_encoding = map->rm_encoding)) {
rm_encoding = "";
}
@@ -5467,7 +5485,7 @@ SWITCH_DECLARE(uint8_t) switch_core_media_negotiate_sdp(switch_core_session_t *s
}
if (rtcp_auto_audio || rtcp_auto_video) {
if (rtcp_auto_audio && !got_audio_rtcp && audio_port) {
if (rtcp_auto_audio && !skip_rtcp && !got_audio_rtcp && audio_port) {
switch_channel_set_variable_printf(session->channel, "rtp_remote_audio_rtcp_port", "%d", audio_port + 1);
a_engine->remote_rtcp_port = audio_port + 1;
@@ -5475,7 +5493,7 @@ SWITCH_DECLARE(uint8_t) switch_core_media_negotiate_sdp(switch_core_session_t *s
smh->mparams->rtcp_audio_interval_msec = SWITCH_RTCP_AUDIO_INTERVAL_MSEC;
}
}
if (rtcp_auto_video && !got_video_rtcp && video_port) {
if (rtcp_auto_video && !skip_video_rtcp && !got_video_rtcp && video_port) {
switch_channel_set_variable_printf(session->channel, "rtp_remote_video_rtcp_port", "%d", video_port + 1);
v_engine->remote_rtcp_port = video_port + 1;
if (!smh->mparams->rtcp_video_interval_msec) {
@@ -5732,6 +5750,59 @@ SWITCH_DECLARE(switch_file_handle_t *) switch_core_media_get_video_file(switch_c
return fh;
}
static void switch_core_session_write_blank_video(switch_core_session_t *session, uint32_t ms)
{
switch_frame_t fr = { 0 };
int i = 0;
switch_rgb_color_t bgcolor = { 0 };
int buflen = SWITCH_RTP_MAX_BUF_LEN;
unsigned char buf[SWITCH_RTP_MAX_BUF_LEN];
switch_media_handle_t *smh;
switch_image_t *blank_img = NULL;
uint32_t frames = 0, frame_ms = 0;
uint32_t fps, width, height;
switch_assert(session != NULL);
if (!(smh = session->media_handle)) {
return;
}
fps = smh->vid_params.fps;
width = smh->vid_params.width;
height = smh->vid_params.height;
if (!width) width = 640;
if (!height) height = 480;
if (!fps) fps = 15;
if (!(width && height && fps)) {
return;
}
fr.packet = buf;
fr.packetlen = buflen;
fr.data = buf + 12;
fr.buflen = buflen - 12;
blank_img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, width, height, 1);
switch_color_set_rgb(&bgcolor, "#000000");
switch_img_fill(blank_img, 0, 0, blank_img->d_w, blank_img->d_h, &bgcolor);
frame_ms = (uint32_t) 1000 / fps;
frames = (uint32_t) ms / frame_ms;
switch_core_media_gen_key_frame(session);
for (i = 0; i < frames; i++) {
switch_core_session_write_video_frame(session, &fr, SWITCH_IO_FLAG_NONE, 0);
switch_yield(frame_ms * 1000);
}
switch_core_media_gen_key_frame(session);
switch_img_free(&blank_img);
}
static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void *obj)
{
@@ -5744,6 +5815,7 @@ static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void
switch_timer_t timer = { 0 };
int fps;
switch_video_read_flag_t read_flags = SVR_FLUSH|SVR_BLOCK;
switch_core_session_t *b_session = NULL;
if (switch_core_session_read_lock(session) != SWITCH_STATUS_SUCCESS) {
return NULL;
@@ -5753,7 +5825,15 @@ static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void
return NULL;
}
switch_core_session_get_partner(session, &b_session);
switch_channel_set_flag(session->channel, CF_VIDEO_WRITING);
switch_channel_set_flag(session->channel, CF_VIDEO_PAUSE_READ);
if (b_session) {
switch_channel_set_flag(b_session->channel, CF_VIDEO_PAUSE_READ);
switch_channel_set_flag(b_session->channel, CF_VIDEO_BLANK);
}
v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO];
@@ -5786,7 +5866,7 @@ static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void
switch_core_timer_next(&timer);
switch_mutex_lock(v_engine->mh.file_write_mutex);
if (smh->video_write_fh->mm.source_fps && smh->video_write_fh->mm.source_fps != fps) {
if (smh->video_write_fh && smh->video_write_fh->mm.source_fps && smh->video_write_fh->mm.source_fps != fps) {
switch_core_timer_destroy(&timer);
switch_core_timer_init(&timer, "soft", (int)(1000 / fps) , 1, switch_core_session_get_pool(session));
}
@@ -5795,7 +5875,7 @@ static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void
wstatus = switch_core_file_read_video(smh->video_write_fh, &fr, read_flags);
if (wstatus == SWITCH_STATUS_SUCCESS) {
switch_core_session_write_video_frame(session, &fr, SWITCH_IO_FLAG_NONE, SVR_FLUSH);
switch_core_session_write_video_frame(session, &fr, SWITCH_IO_FLAG_FORCE, SVR_FLUSH);
switch_img_free(&fr.img);
} else if (wstatus != SWITCH_STATUS_BREAK && wstatus != SWITCH_STATUS_IGNORE) {
switch_set_flag_locked(smh->video_write_fh, SWITCH_FILE_FLAG_VIDEO_EOF);
@@ -5808,6 +5888,13 @@ static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void
switch_core_session_rwunlock(session);
if (b_session) {
switch_channel_clear_flag(b_session->channel, CF_VIDEO_PAUSE_READ);
switch_channel_clear_flag(b_session->channel, CF_VIDEO_BLANK);
switch_core_session_rwunlock(b_session);
}
switch_channel_clear_flag(session->channel, CF_VIDEO_PAUSE_READ);
switch_channel_clear_flag(session->channel, CF_VIDEO_WRITING);
smh->video_write_thread_running = 0;
@@ -5941,7 +6028,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_file(switch_core_ses
if (fh) {
switch_threadattr_t *thd_attr = NULL;
switch_core_session_write_blank_video(session, 200);
switch_threadattr_create(&thd_attr, switch_core_session_get_pool(session));
switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE);
smh->video_write_thread_running = 1;
@@ -5958,6 +6045,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_file(switch_core_ses
switch_thread_join(&st, smh->video_write_thread);
switch_mutex_lock(v_engine->mh.file_write_mutex);
smh->video_write_thread = NULL;
switch_core_session_write_blank_video(session, 200);
}
smh->video_write_fh = fh;
@@ -6258,7 +6346,7 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi
switch_mutex_unlock(mh->file_read_mutex);
}
if (switch_channel_test_flag(channel, CF_VIDEO_WRITING) || session->video_read_callback) {
if ((switch_channel_test_flag(channel, CF_VIDEO_WRITING) || session->video_read_callback) && !switch_channel_test_flag(channel, CF_VIDEO_BLANK)) {
send_blank = 0;
}
@@ -6817,6 +6905,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_choose_port(switch_core_sessio
lookup_rtpip = smh->mparams->rtpip;
if (!lookup_rtpip) {
return SWITCH_STATUS_FALSE;
}
/* Don't do anything if we're in proxy mode or if a (remote) port already has been found */
if (!force) {
if (switch_channel_test_flag(session->channel, CF_PROXY_MODE) ||
@@ -8125,7 +8217,8 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_activate_rtp(switch_core_sessi
switch_rtp_set_payload_map(v_engine->rtp_session, &v_engine->payload_map);
switch_channel_set_flag(session->channel, CF_VIDEO);
switch_core_session_start_video_thread(session);
switch_rtp_set_video_buffer_size(v_engine->rtp_session, 1, 0);
if ((ssrc = switch_channel_get_variable(session->channel, "rtp_use_video_ssrc"))) {
uint32_t ssrc_ul = (uint32_t) strtoul(ssrc, NULL, 10);
switch_rtp_set_ssrc(v_engine->rtp_session, ssrc_ul);
@@ -9094,7 +9187,7 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess
if (switch_channel_test_flag(smh->session->channel, CF_ICE)) {
gen_ice(session, SWITCH_MEDIA_TYPE_AUDIO, ip, port);
switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=msid-semantic: WMS %s\r\na=end-of-candidates\r\n", smh->msid);
switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=msid-semantic: WMS %s\r\n", smh->msid);
}
if (a_engine->codec_negotiated) {
@@ -9287,6 +9380,7 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess
}
}
switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=end-of-candidates\r\n");
switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=ssrc:%u cname:%s\r\n", a_engine->ssrc, smh->cname);
switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=ssrc:%u msid:%s a0\r\n", a_engine->ssrc, smh->msid);
@@ -9824,6 +9918,8 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess
#ifdef GOOGLE_ICE
switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=ice-options:google-ice\r\n");
#endif
switch_snprintf(buf + strlen(buf), SDPBUFLEN - strlen(buf), "a=end-of-candidates\r\n");
}
@@ -12772,6 +12868,35 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess
return SWITCH_STATUS_FALSE;
}
SWITCH_DECLARE(switch_bool_t) switch_core_media_codec_get_cap(switch_core_session_t *session,
switch_media_type_t mtype,
switch_codec_flag_t flag) {
switch_rtp_engine_t *engine = NULL;
switch_media_handle_t *smh = NULL;
switch_codec_t *codec = NULL;
switch_assert(session);
if (!(smh = session->media_handle)) {
return SWITCH_FALSE;
}
if (!(engine = &smh->engines[mtype])) {
return SWITCH_FALSE;
}
codec = &engine->write_codec;
if (!switch_core_codec_ready(codec)) {
return SWITCH_FALSE;
}
if (switch_test_flag(codec, flag)){
return SWITCH_TRUE;
}
return SWITCH_FALSE;
}
SWITCH_DECLARE(switch_status_t) switch_core_session_write_encoded_video_frame(switch_core_session_t *session,
switch_frame_t *frame, switch_io_flag_t flags, int stream_id)
@@ -12850,6 +12975,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor
return SWITCH_STATUS_FALSE;
}
if (switch_channel_test_flag(session->channel, CF_VIDEO_WRITING) && !(flags & SWITCH_IO_FLAG_FORCE)) {
return SWITCH_STATUS_SUCCESS;
}
if (switch_core_session_media_flow(session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_RECVONLY || switch_core_session_media_flow(session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_INACTIVE) {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG3, "Writing video to RECVONLY/INACTIVE session\n");
return SWITCH_STATUS_SUCCESS;
@@ -13109,6 +13238,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core
if (switch_channel_test_flag(session->channel, CF_VIDEO_PAUSE_READ)) {
*frame = &runtime.dummy_cng_frame;
switch_channel_video_sync(session->channel);
switch_cond_next();
return SWITCH_STATUS_SUCCESS;
}
+263
View File
@@ -0,0 +1,263 @@
/*
* FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
* Copyright (C) 2005-2015, Anthony Minessale II <anthm@freeswitch.org>
*
* Version: MPL 1.1
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
*
* The Initial Developer of the Original Code is
* Anthony Minessale II <anthm@freeswitch.org>
* Portions created by the Initial Developer are Copyright (C)
* the Initial Developer. All Rights Reserved.
*
* Contributor(s):
*
* Dragos Oancea <droancea@yahoo.com>
*
* switch_estimators.c -- Estimators and Detectors (try to read into the future: packet loss, jitter, RTT, etc)
*
*/
#include <switch_estimators.h>
#include <switch.h>
#ifndef _MSC_VER
#include <switch_private.h>
#endif
#undef PACKAGE_NAME
#undef PACKAGE_STRING
#undef PACKAGE_TARNAME
#undef PACKAGE_VERSION
#undef PACKAGE_BUGREPORT
#undef VERSION
#undef PACKAGE
#undef inline
#include <datatypes.h>
#include <switch_types.h>
#define KALMAN_SYSTEM_MODELS 3 /*loss, jitter, rtt*/
#define EST_LOSS 0
#define EST_JITTER 1
#define EST_RTT 2
/* This function initializes the Kalman System Model
*
* xk+1 = A*xk + wk
* zk = H*xk + vk
* xk = state variable (must exist in physical world - measurable )
* zk = measurment
* wk,vk - white noise
* A = state trasition matrix , (n x n ) matrix
* H = state-to-measurment matrix , ( n x n ) matrix
* Noise covariance:
* Q: Covariance matrix of wk, ( n x n ) diagonal matrix
* R: Covariance matrix of vk , ( m x m ) diagonal matrix
* R: if you want to be affected less by the measurement and get the estimate with less variation, increase R
* Q: if you want to be affected more by the measurement and get the estimate with more variation, decrease Q
*
* (Phil Kim book)
*
*/
void switch_kalman_init(kalman_estimator_t *est, float Q, float R)
{
est -> val_estimate_last = 0 ;
est -> P_last = 0;
est -> Q = Q; /*accuracy of system model */ /* SYSTEM MODEL: TO BE DEDUCTED */
est -> R = R; /*accuracy of measurement*/ /* SYSTEM MODEL: TO BE DEDUCTED */
est -> K = 0;
est -> val_estimate = 0 ;
est -> val_measured = 0 ; // [0-100 %] or [0-5000] or [0-2sec]
}
/*
CUSUM Kalman functions to detect sudden change over a predefined thereshold.
y(t) = sampled RTT
x(t)= desired RTT
Model:
x(t+1) = x(t) + delta(t)*v(t)
y(t) = x(t) + e(t)
Noisy characteristic of RTT captured by measurment noise e(t) with variance Re.
The step changes in the desired RTT x(t) is modeled as the process noise v(t)
with variance Rv and the discrete variable delta(t) .
If a change occurs at time t, then delta(t) = 1 otherwise delta(t) = 0.
avg(x(t)) = avg(x(t-1)) + K(t)(y(t) - avg(x(t-1)))
K(t) = P(t-1)/(P(t-1) + Re))
P(t) = (1-K(t))P(t-1) + delta(t-1)* Rv
e(t) = y(t) - avg(x(t))
g(t) = max(g(t-1) + e(t) - epsilon,0)
if g(t) > 0 then
delta(t) = 1 // alarm
g(t) = 0
else
delta(t) = 0
endif
constants:
epsilon = 0.005
h = 0.05
*/
switch_bool_t switch_kalman_cusum_init (cusum_kalman_detector_t *detect_change, float epsilon, float h)
{
cusum_kalman_detector_t *detector_change = detect_change;
if (epsilon < 0 || h < 0) {
return FALSE;
}
detector_change -> val_estimate_last = 0;
detector_change -> val_desired_last = 0;
detector_change -> P_last = 0;
detector_change -> K_last = 0;
detector_change -> delta = 0;
detector_change -> measurement_noise_e = 0;
detector_change -> variance_Re = 0;
detector_change -> measurement_noise_v = 0;
detector_change -> variance_Rv = 0;
detector_change -> g_last = 0;
/*per system model*/
detector_change -> epsilon = epsilon;
detector_change -> h = h;
/*variance*/
detector_change -> last_average = 0;
detector_change -> last_q = 0;
detector_change -> N = 0;
return TRUE;
}
switch_bool_t switch_kalman_cusum_detect_change(cusum_kalman_detector_t * detector, float measurement, float avg)
{
float K=0;
float P=0;
float g=0;
float desired_val;
float current_average;
float current_q;
float sample_variance_Re = 0;
/*variance*/
detector->N++;
current_average = detector->last_average + (measurement - detector->last_average)/detector->N ;
if (avg > current_average) {
current_average = avg;
}
current_q = detector-> last_q + (measurement - detector->last_average) * (measurement - current_average);
if (detector->N != 0)
sample_variance_Re = sqrt(current_q/detector->N);
detector->variance_Re = sample_variance_Re;
detector->variance_Rv = sample_variance_Re;
if (sample_variance_Re != 0) {
K = detector->P_last / (detector->P_last + detector->variance_Re);
desired_val = detector->val_desired_last + K * (measurement - detector->variance_Re);
P = (1 - K) * detector->P_last + detector->delta * detector->variance_Rv;
detector->measurement_noise_e = measurement - desired_val;
g = detector->g_last + detector->measurement_noise_e - detector->epsilon;
if (g > detector->h) {
detector->delta = 1;
g = 0;
} else {
detector->delta = 0;
}
/* update last vals for calculating variance */
detector->last_average = current_average;
/* update lasts (cusum)*/
detector -> g_last = g;
detector -> P_last = P;
detector -> val_desired_last = desired_val;
}
if (detector->delta == 1) {
return TRUE;
}
return FALSE;
}
/* Kalman filter abstract ( measure and estimate 1 single value per system model )
* Given the measurment and the system model together with the current state ,
* the function puts an estimate in the estimator struct */
switch_bool_t switch_kalman_estimate (kalman_estimator_t * est, float measurement, int system_model)
{
/*system model can be about: loss, jitter, rtt*/
float val_estimate;
float val_temp_est = est->val_estimate_last;
float P_temp = est->P_last + est->Q;
if (system_model >= KALMAN_SYSTEM_MODELS) {
return SWITCH_FALSE ;
}
/*sanitize input a little bit, just in case */
if (system_model == EST_LOSS ) {
if ((measurement > 100) && (measurement < 0)) {
return SWITCH_FALSE ;
}
}
if (system_model == EST_JITTER) {
if ((measurement > 10000) && (measurement < 0)) {
return SWITCH_FALSE;
}
}
if (system_model == EST_RTT) {
if ((measurement > 2 ) && (measurement < 0)) {
return SWITCH_FALSE;
}
}
/* calculate the Kalman gain */
est->K = P_temp * (1.0/(P_temp + est->R));
/* real life measurement */
est->val_measured = measurement ;
val_estimate = val_temp_est + est->K * (est->val_measured - val_temp_est);
est->P = (1 - est->K) * P_temp;
/*update lasts*/
est->P_last = est->P;
/* save the estimated value (future) */
est->val_estimate_last = val_estimate;
return SWITCH_TRUE;
}
switch_bool_t switch_kalman_is_slow_link(kalman_estimator_t * est_loss, kalman_estimator_t * est_rtt)
{
float thresh_packet_loss = 5; /* % */
float thresh_rtt = 0.8 ; /*seconds*/
if ((est_loss->val_estimate_last > thresh_packet_loss) &&
(est_rtt->val_estimate_last > thresh_rtt )) {
return SWITCH_TRUE;
}
return SWITCH_FALSE;
}
/* For Emacs:
* Local Variables:
* mode:c
* indent-tabs-mode:t
* tab-width:4
* c-basic-offset:4
* End:
* For VIM:
* vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
*/
+2 -69
View File
@@ -79,13 +79,8 @@ static void video_bridge_thread(switch_core_session_t *session, void *obj)
switch_channel_t *b_channel = switch_core_session_get_channel(vh->session_b);
switch_status_t status;
switch_frame_t *read_frame = 0;
int set_decoded_read = 0, refresh_timer = 0, playing_file = 0;
switch_frame_t fr = { 0 };
unsigned char *buf = NULL;
int send_blank = 0;
int set_decoded_read = 0, refresh_timer = 0;
int refresh_cnt = 300;
int blank_cnt = 30;
switch_image_t *blank_img = NULL;
vh->up = 1;
@@ -101,7 +96,7 @@ static void video_bridge_thread(switch_core_session_t *session, void *obj)
if (switch_channel_media_up(channel)) {
switch_codec_t *a_codec = switch_core_session_get_video_read_codec(vh->session_a);
switch_codec_t *b_codec = switch_core_session_get_video_write_codec(vh->session_b);
switch_file_handle_t *fh;
if (switch_channel_test_flag(channel, CF_VIDEO_REFRESH_REQ)) {
refresh_timer = refresh_cnt;
@@ -139,66 +134,6 @@ static void video_bridge_thread(switch_core_session_t *session, void *obj)
refresh_timer--;
}
if (send_blank) {
send_blank--;
fr.img = blank_img;
switch_core_media_gen_key_frame(session);
switch_core_session_write_video_frame(session, &fr, SWITCH_IO_FLAG_NONE, 0);
switch_yield(30000);
continue;
}
if ((fh = switch_core_media_get_video_file(session, SWITCH_RW_WRITE)) && switch_test_flag(fh, SWITCH_FILE_OPEN)) {
switch_status_t wstatus;
if (!buf) {
int buflen = SWITCH_RTP_MAX_BUF_LEN;
buf = switch_core_session_alloc(session, buflen);
fr.packet = buf;
fr.packetlen = buflen;
fr.data = buf + 12;
fr.buflen = buflen - 12;
}
wstatus = switch_core_file_read_video(fh, &fr, SVR_FLUSH | SVR_BLOCK);
if (!blank_img && fr.img) {
switch_rgb_color_t bgcolor = { 0 };
blank_img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, fr.img->d_w, fr.img->d_h, 1);
switch_color_set_rgb(&bgcolor, "#000000");
switch_img_fill(blank_img, 0, 0, blank_img->d_w, blank_img->d_h, &bgcolor);
}
if (!playing_file) {
playing_file = 1;
switch_core_media_gen_key_frame(session);
send_blank = blank_cnt;
refresh_timer = refresh_cnt;
switch_channel_video_sync(channel);
switch_channel_video_sync(b_channel);
continue;
}
if (wstatus == SWITCH_STATUS_SUCCESS) {
switch_core_session_write_video_frame(session, &fr, SWITCH_IO_FLAG_NONE, 0);
switch_img_free(&fr.img);
} else if (wstatus != SWITCH_STATUS_BREAK) {
switch_core_media_set_video_file(session, NULL, SWITCH_RW_WRITE);
}
continue;
}
if (playing_file) {
playing_file = 0;
send_blank = blank_cnt;
refresh_timer = refresh_cnt;
switch_channel_video_sync(channel);
switch_channel_video_sync(b_channel);
}
status = switch_core_session_read_video_frame(vh->session_a, &read_frame, SWITCH_IO_FLAG_NONE, 0);
if (!SWITCH_READ_ACCEPTABLE(status)) {
@@ -228,8 +163,6 @@ static void video_bridge_thread(switch_core_session_t *session, void *obj)
}
}
switch_img_free(&blank_img);
if (set_decoded_read) {
switch_channel_clear_flag_recursive(channel, CF_VIDEO_DECODED_READ);
}
+161 -16
View File
@@ -35,6 +35,7 @@
//#define DEBUG_MISSED_SEQ
//#define DEBUG_EXTRA
//#define DEBUG_RTCP
#define DEBUG_ESTIMATORS
#include <switch.h>
#ifndef _MSC_VER
@@ -55,6 +56,8 @@
#include <srtp_priv.h>
#include <switch_ssl.h>
#include <switch_jitterbuffer.h>
#include <switch_estimators.h>
#define JITTER_LEAD_FRAMES 10
#define READ_INC(rtp_session) switch_mutex_lock(rtp_session->read_mutex); rtp_session->reading++
@@ -170,6 +173,10 @@ typedef struct {
#pragma pack(pop, r1)
#endif
#define KALMAN_SYSTEM_MODELS 3 /*loss, jitter, rtt*/
#define EST_LOSS 0
#define EST_JITTER 1
#define EST_RTT 2
typedef struct {
switch_rtcp_ext_hdr_t header;
@@ -446,6 +453,8 @@ struct switch_rtp {
switch_core_session_t *session;
payload_map_t **pmaps;
payload_map_t *pmap_tail;
kalman_estimator_t *estimators[KALMAN_SYSTEM_MODELS];
cusum_kalman_detector_t *detectors[KALMAN_SYSTEM_MODELS];
int ice_adj;
uint8_t has_rtp;
uint8_t has_rtcp;
@@ -1860,6 +1869,24 @@ static void rtcp_stats_init(switch_rtp_t *rtp_session)
} else {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "rtcp_stats_init: %s ssrc[%u] base_seq[%u]\n", rtp_type(rtp_session), stats->ssrc, stats->base_seq);
}
if (rtp_session->flags[SWITCH_RTP_FLAG_ENABLE_RTCP] && (switch_core_media_codec_get_cap(rtp_session->session,
SWITCH_MEDIA_TYPE_AUDIO, SWITCH_CODEC_FLAG_HAS_ADJ_BITRATE))) {
kalman_estimator_t *estimators[KALMAN_SYSTEM_MODELS];
cusum_kalman_detector_t *detectors[KALMAN_SYSTEM_MODELS];
rtp_session->flags[SWITCH_RTP_FLAG_ADJ_BITRATE_CAP] = 1;
rtp_session->flags[SWITCH_RTP_FLAG_ESTIMATORS] = 1;
rtp_session->estimators[EST_LOSS] = switch_core_alloc(rtp_session->pool, sizeof(*estimators[0]));
switch_kalman_init(rtp_session->estimators[EST_LOSS],0.1,0.1);
rtp_session->estimators[EST_RTT] = switch_core_alloc(rtp_session->pool, sizeof(*estimators[0]));
switch_kalman_init(rtp_session->estimators[EST_RTT],0.03,1);
rtp_session->detectors[EST_RTT] = switch_core_alloc(rtp_session->pool, sizeof(*detectors[0]));
switch_kalman_cusum_init(rtp_session->detectors[EST_RTT],0.005,0.5);
rtp_session->detectors[EST_LOSS] = switch_core_alloc(rtp_session->pool, sizeof(*detectors[0]));
switch_kalman_cusum_init(rtp_session->detectors[EST_LOSS],0.005,0.5);
}
}
static int rtcp_stats(switch_rtp_t *rtp_session)
@@ -2785,6 +2812,7 @@ SWITCH_DECLARE(void) switch_rtp_reset_jb(switch_rtp_t *rtp_session)
SWITCH_DECLARE(void) switch_rtp_reset_vb(switch_rtp_t *rtp_session)
{
if (rtp_session->vb) {
switch_jb_reset(rtp_session->vb);
}
@@ -3164,7 +3192,7 @@ static int do_dtls(switch_rtp_t *rtp_session, switch_dtls_t *dtls)
ret = SSL_get_error(dtls->ssl, ret);
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_ERROR, "%s DTLS packet read err %d\n", rtp_type(rtp_session), ret);
}
if (dtls_states[dtls->state]) {
r = dtls_states[dtls->state](rtp_session, dtls);
}
@@ -3368,8 +3396,11 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_add_dtls(switch_rtp_t *rtp_session, d
//SSL_CTX_set_verify(dtls->ssl_ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL);
SSL_CTX_set_verify(dtls->ssl_ctx, SSL_VERIFY_NONE, NULL);
SSL_CTX_set_cipher_list(dtls->ssl_ctx, "ALL");
SSL_CTX_set_cipher_list(dtls->ssl_ctx, "ECDH:!RC4:!SSLv3:RSA_WITH_AES_128_CBC_SHA");
//SSL_CTX_set_cipher_list(dtls->ssl_ctx, "ECDHE-RSA-AES256-GCM-SHA384");
//SSL_CTX_set_cipher_list(dtls->ssl_ctx, "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH");
//SSL_CTX_set_cipher_list(dtls->ssl_ctx, "SUITEB128");
SSL_CTX_set_read_ahead(dtls->ssl_ctx, 1);
#ifdef HAVE_OPENSSL_DTLS_SRTP
//SSL_CTX_set_tlsext_use_srtp(dtls->ssl_ctx, "SRTP_AES128_CM_SHA1_80:SRTP_AES128_CM_SHA1_32");
SSL_CTX_set_tlsext_use_srtp(dtls->ssl_ctx, "SRTP_AES128_CM_SHA1_80");
@@ -3428,11 +3459,11 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_add_dtls(switch_rtp_t *rtp_session, d
SSL_set_verify(dtls->ssl, SSL_VERIFY_NONE, NULL);
SSL_set_app_data(dtls->ssl, dtls);
BIO_ctrl(dtls->read_bio, BIO_CTRL_DGRAM_SET_MTU, 1400, NULL);
BIO_ctrl(dtls->write_bio, BIO_CTRL_DGRAM_SET_MTU, 1400, NULL);
SSL_set_mtu(dtls->ssl, 1400);
BIO_ctrl(dtls->write_bio, BIO_C_SET_BUFF_SIZE, 1400, NULL);
BIO_ctrl(dtls->read_bio, BIO_C_SET_BUFF_SIZE, 1400, NULL);
//BIO_ctrl(dtls->read_bio, BIO_CTRL_DGRAM_SET_MTU, 1400, NULL);
//BIO_ctrl(dtls->write_bio, BIO_CTRL_DGRAM_SET_MTU, 1400, NULL);
//SSL_set_mtu(dtls->ssl, 1400);
//BIO_ctrl(dtls->write_bio, BIO_C_SET_BUFF_SIZE, 1400, NULL);
//BIO_ctrl(dtls->read_bio, BIO_C_SET_BUFF_SIZE, 1400, NULL);
@@ -3696,7 +3727,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_change_interval(switch_rtp_t *rtp_ses
switch_status_t status = SWITCH_STATUS_SUCCESS;
int change_timer = 0;
if (rtp_session->ms_per_packet && rtp_session->ms_per_packet != ms_per_packet) {
if (rtp_session->ms_per_packet != ms_per_packet || rtp_session->samples_per_interval != samples_per_interval) {
change_timer = 1;
}
@@ -4117,7 +4148,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_set_video_buffer_size(switch_rtp_t *r
}
if (!max_frames || frames >= max_frames) {
max_frames = frames + 8;
max_frames = frames * 10;
}
rtp_session->last_max_vb_frames = max_frames;
@@ -4174,7 +4205,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_activate_jitter_buffer(switch_rtp_t *
}
if (max_queue_frames < queue_frames) {
max_queue_frames = queue_frames * 6;
max_queue_frames = queue_frames * 3;
}
@@ -5920,6 +5951,8 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t
uint32_t sec, ntp_sec, ntp_usec, lsr_now;
uint32_t lsr;
uint32_t packet_ssrc;
double rtt_now = 0;
int rtt_increase = 0, packet_loss_increase=0;
now = switch_time_now(); /* number of microseconds since 00:00:00 january 1, 1970 UTC */
sec = (uint32_t)(now/1000000); /* converted to second (NTP most significant bits) */
@@ -5974,10 +6007,6 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t
((float)(uint8_t)percent_fraction * .3));
}
if (!rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && rtp_session->rtcp_frame.reports[i].loss_avg != old_avg) {
switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT, (void *)&rtp_session->rtcp_frame.reports[i].loss_avg, SCCT_NONE, NULL, NULL, NULL);
}
rtp_session->rtcp_frame.reports[i].ssrc = ntohl(report->ssrc);
rtp_session->rtcp_frame.reports[i].fraction = (uint8_t)report->fraction;
rtp_session->rtcp_frame.reports[i].lost = ntohl(report->lost);
@@ -5986,7 +6015,6 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t
rtp_session->rtcp_frame.reports[i].lsr = ntohl(report->lsr);
rtp_session->rtcp_frame.reports[i].dlsr = ntohl(report->dlsr);
if (rtp_session->rtcp_frame.reports[i].lsr && !rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU]) {
double rtt_now;
switch_time_exp_gmt(&now_hr,now);
/* Calculating RTT = A - DLSR - LSR */
rtt_now = (double)(lsr_now - rtp_session->rtcp_frame.reports[i].dlsr - rtp_session->rtcp_frame.reports[i].lsr)/65536;
@@ -6004,6 +6032,123 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "RTT average %f\n",
rtp_session->rtcp_frame.reports[i].rtt_avg);
}
if (rtp_session->flags[SWITCH_RTP_FLAG_ADJ_BITRATE_CAP] && rtp_session->flags[SWITCH_RTP_FLAG_ESTIMATORS] && !rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) {
/* SWITCH_RTP_FLAG_ADJ_BITRATE_CAP : Can the codec change its bitrate on the fly per API command ? */
#ifdef DEBUG_ESTIMATORS
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "Current packet loss: [%d %%] Current RTT: [%f ms]\n", percent_fraction, rtt_now);
#endif
switch_kalman_estimate(rtp_session->estimators[EST_RTT], rtt_now, EST_RTT);
if (switch_kalman_cusum_detect_change(rtp_session->detectors[EST_RTT], rtt_now, rtp_session->estimators[EST_RTT]->val_estimate_last)) {
/* sudden change in the mean value of RTT */
#ifdef DEBUG_ESTIMATORS
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"Sudden change in the mean value of RTT !\n");
#endif
rtt_increase = 1;
}
switch_kalman_estimate(rtp_session->estimators[EST_LOSS], percent_fraction, EST_LOSS);
if (switch_kalman_cusum_detect_change(rtp_session->detectors[EST_LOSS], percent_fraction, rtp_session->estimators[EST_LOSS]->val_estimate_last)){
/* sudden change in the mean value of packet loss */
#ifdef DEBUG_ESTIMATORS
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"Sudden change in the mean value of packet loss!\n");
#endif
packet_loss_increase = 1;
}
#ifdef DEBUG_ESTIMATORS
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "ESTIMATORS: Packet loss will be: [%f] RTT will be: [%f ms]\n",
rtp_session->estimators[EST_LOSS]->val_estimate_last, rtp_session->estimators[EST_RTT]->val_estimate_last);
#endif
if (rtp_session->rtcp_frame.reports[i].loss_avg != old_avg) {
/*getting bad*/
if (switch_kalman_is_slow_link(rtp_session->estimators[EST_LOSS],
rtp_session->estimators[EST_RTT])) {
/* going to minimum bitrate */
#ifdef DEBUG_ESTIMATORS
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "Slow link conditions: Loss average: [%d %%], Previous loss: [%d %%]. \
Going to minimum bitrate!",rtp_session->rtcp_frame.reports[i].loss_avg, old_avg);
#endif
switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE, SCCT_STRING, "minimum", SCCT_NONE, NULL, NULL, NULL);
/* if after going to minimum bitrate we still have packet loss then we increase ptime. TODO */
} else if (packet_loss_increase && (rtp_session->estimators[EST_LOSS]->val_estimate_last >= 5)) {
/* sudden change in the mean value of packet loss percentage */
switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE,
SCCT_STRING, "decrease",
SCCT_NONE, NULL, NULL, NULL);
#ifdef DEBUG_ESTIMATORS
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"Sudden change in the mean value of packet loss percentage !\n");
#endif
switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
(void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
SCCT_NONE, NULL, NULL, NULL);
} else if (!rtt_increase && rtp_session->estimators[EST_LOSS]->val_estimate_last >= rtp_session->rtcp_frame.reports[i].loss_avg ) {
/* lossy because of congestion (queues full somewhere -> some packets are dropped , but RTT is good ), packet loss with many small gaps */
#ifdef DEBUG_ESTIMATORS
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "packet loss, but RTT is not bad\n");
#endif
switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
(void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
SCCT_NONE, NULL, NULL, NULL);
} else if ((rtp_session->estimators[EST_LOSS]->val_estimate_last < 1) && packet_loss_increase) {
#ifdef DEBUG_ESTIMATORS
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "small packet loss average\n");
#endif
/*small loss_avg*/
switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE,
SCCT_STRING, "default",
SCCT_NONE, NULL, NULL, NULL);
switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
(void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
SCCT_NONE, NULL, NULL, NULL);
} else if ((rtp_session->estimators[EST_LOSS]->val_estimate_last < 5) &&
(rtp_session->rtcp_frame.reports[i].rtt_avg < rtp_session->estimators[EST_RTT]->val_estimate_last)) {
/* estimate that packet loss will decrease, we can increase the bitrate */
switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE,
SCCT_STRING, "increase",
SCCT_NONE, NULL, NULL, NULL);
switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
(void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
SCCT_NONE, NULL, NULL, NULL);
} else {
/* *do nothing about bitrate, just pass the packet loss to the codec */
#ifdef DEBUG_ESTIMATORS
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"do nothing about bitrate, just pass the packet loss to the codec\n");
#endif
switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
(void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
SCCT_NONE, NULL, NULL, NULL);
}
}
} else {
if (!rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && rtp_session->rtcp_frame.reports[i].loss_avg != old_avg) {
switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
(void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
SCCT_NONE, NULL, NULL, NULL);
}
}
}
rtp_session->rtcp_frame.report_count = (uint16_t)i;