From f4ad4ed6c47b8961a180dfadc02f972a811878d2 Mon Sep 17 00:00:00 2001 From: Anthony Minessale Date: Fri, 10 Mar 2017 21:14:17 -0600 Subject: [PATCH] FS-10126: [freeswitch-core] General Video Improvements #resolve --- src/include/switch_core.h | 3 + src/include/switch_module_interfaces.h | 1 + src/include/switch_types.h | 9 +- .../mod_conference/conference_file.c | 1 - .../mod_conference/conference_video.c | 18 +- .../languages/mod_managed/freeswitch_wrap.cxx | 94 ++++++++ src/mod/languages/mod_managed/managed/swig.cs | 69 +++++- src/switch_core_media.c | 217 +++++++++++++++--- src/switch_ivr_bridge.c | 29 ++- src/switch_jitterbuffer.c | 114 +++++++-- src/switch_rtp.c | 191 ++++++++------- src/switch_time.c | 8 + 12 files changed, 596 insertions(+), 158 deletions(-) diff --git a/src/include/switch_core.h b/src/include/switch_core.h index 128dec3f15..dcbe308067 100644 --- a/src/include/switch_core.h +++ b/src/include/switch_core.h @@ -1307,6 +1307,9 @@ SWITCH_DECLARE(uint32_t) switch_core_session_flush_private_events(switch_core_se SWITCH_DECLARE(switch_status_t) switch_core_session_read_frame(_In_ switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags, int stream_id); +SWITCH_DECLARE(switch_bool_t) switch_core_session_transcoding(switch_core_session_t *session_a, switch_core_session_t *session_b, switch_media_type_t type); +SWITCH_DECLARE(void) switch_core_session_passthru(switch_core_session_t *session, switch_media_type_t type, switch_bool_t on); + /*! \brief Read a video frame from a session \param session the session to read from diff --git a/src/include/switch_module_interfaces.h b/src/include/switch_module_interfaces.h index 90c2a9052d..faebf15094 100644 --- a/src/include/switch_module_interfaces.h +++ b/src/include/switch_module_interfaces.h @@ -215,6 +215,7 @@ struct switch_timer { unsigned int samples; /*! current sample count based on samples parameter */ uint32_t samplecount; + uint32_t last_samplecount; /*! the timer interface provided from a loadable module */ switch_timer_interface_t *timer_interface; /*! the timer's memory pool */ diff --git a/src/include/switch_types.h b/src/include/switch_types.h index 9f86622a63..04b46c555e 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -774,11 +774,10 @@ typedef enum { SWITCH_RTP_FLAG_MUTE, SWITCH_RTP_FLAG_NACK, SWITCH_RTP_FLAG_TMMBR, - SWITCH_RTP_FLAG_GEN_TS_DELTA, - SWITCH_RTP_FLAG_GEN_TS_MANUAL, SWITCH_RTP_FLAG_DETECT_SSRC, SWITCH_RTP_FLAG_TEXT, SWITCH_RTP_FLAG_OLD_FIR, + SWITCH_RTP_FLAG_PASSTHRU, SWITCH_RTP_FLAG_INVALID } switch_rtp_flag_t; @@ -909,7 +908,6 @@ typedef enum { */ - } switch_rtp_bug_flag_t; #ifdef _MSC_VER @@ -1558,6 +1556,8 @@ typedef struct switch_vid_params_s { uint32_t width; uint32_t height; uint32_t fps; + uint32_t d_width; + uint32_t d_height; } switch_vid_params_t; @@ -1608,7 +1608,8 @@ typedef enum { SFF_USE_VIDEO_TIMESTAMP = (1 << 16), SFF_ENCODED = (1 << 17), SFF_TEXT_LINE_BREAK = (1 << 18), - SFF_IS_KEYFRAME = (1 << 19) + SFF_IS_KEYFRAME = (1 << 19), + SFF_EXTERNAL = (1 << 20) } switch_frame_flag_enum_t; typedef uint32_t switch_frame_flag_t; diff --git a/src/mod/applications/mod_conference/conference_file.c b/src/mod/applications/mod_conference/conference_file.c index 0c125cd7a4..4eab9d9c49 100644 --- a/src/mod/applications/mod_conference/conference_file.c +++ b/src/mod/applications/mod_conference/conference_file.c @@ -95,7 +95,6 @@ switch_status_t conference_file_close(conference_obj_t *conference, conference_f if (conference->canvases[node->canvas_id]->timer.timer_interface) { conference->canvases[node->canvas_id]->timer.interval = conference->video_fps.ms; conference->canvases[node->canvas_id]->timer.samples = conference->video_fps.samples; - switch_core_timer_sync(&conference->canvases[node->canvas_id]->timer); conference->canvases[node->canvas_id]->send_keyframe = 1; } conference->playing_video_file = 0; diff --git a/src/mod/applications/mod_conference/conference_video.c b/src/mod/applications/mod_conference/conference_video.c index 6b72d1ba38..ea5fc919fb 100644 --- a/src/mod/applications/mod_conference/conference_video.c +++ b/src/mod/applications/mod_conference/conference_video.c @@ -1723,7 +1723,7 @@ void conference_video_write_canvas_image_to_codec_group(conference_obj_t *confer } if (frame->timestamp) { - switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME); + switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME|SFF_USE_VIDEO_TIMESTAMP); } frame->packetlen = frame->datalen + 12; @@ -1753,7 +1753,9 @@ void conference_video_write_canvas_image_to_codec_group(conference_obj_t *confer switch_core_session_request_video_refresh(imember->session); } - if (switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_RECVONLY || switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_INACTIVE) { + if (switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_RECVONLY || + switch_channel_test_flag(imember->channel, CF_VIDEO_WRITING) || + switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_INACTIVE) { switch_core_session_rwunlock(imember->session); continue; } @@ -3476,6 +3478,7 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr if (!imember->rec && (!imember->session || !switch_channel_test_flag(imember->channel, CF_VIDEO_READY) || !imember->canvas || + switch_channel_test_flag(imember->channel, CF_VIDEO_WRITING) || switch_core_session_read_lock(imember->session) != SWITCH_STATUS_SUCCESS)) { continue; } @@ -3680,7 +3683,9 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr switch_core_session_request_video_refresh(imember->session); } - if (switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_RECVONLY || switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_INACTIVE) { + if (switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_RECVONLY || + switch_channel_test_flag(imember->channel, CF_VIDEO_WRITING) || + switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_INACTIVE) { switch_core_session_rwunlock(imember->session); continue; } @@ -3695,13 +3700,14 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr conference_video_set_canvas_fgimg(canvas, NULL); } - switch_set_flag(&write_frame, SFF_RAW_RTP); + switch_set_flag(&write_frame, SFF_RAW_RTP|SFF_USE_VIDEO_TIMESTAMP|SFF_RAW_RTP_PARSE_FRAME); write_frame.img = write_img; write_frame.packet = packet; write_frame.data = ((uint8_t *)packet) + 12; write_frame.datalen = 0; write_frame.buflen = SWITCH_RTP_MAX_BUF_LEN - 12; write_frame.packetlen = 0; + write_frame.timestamp = timestamp; //switch_core_session_write_video_frame(imember->session, &write_frame, SWITCH_IO_FLAG_NONE, 0); @@ -4034,7 +4040,9 @@ void *SWITCH_THREAD_FUNC conference_video_super_muxing_thread_run(switch_thread_ switch_core_session_request_video_refresh(imember->session); } - if (switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_RECVONLY || switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_INACTIVE) { + if (switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_RECVONLY || + switch_channel_test_flag(imember->channel, CF_VIDEO_WRITING) || + switch_core_session_media_flow(imember->session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_INACTIVE) { switch_core_session_rwunlock(imember->session); continue; } diff --git a/src/mod/languages/mod_managed/freeswitch_wrap.cxx b/src/mod/languages/mod_managed/freeswitch_wrap.cxx index 1d8337342a..8eb15dbe38 100644 --- a/src/mod/languages/mod_managed/freeswitch_wrap.cxx +++ b/src/mod/languages/mod_managed/freeswitch_wrap.cxx @@ -5754,6 +5754,50 @@ SWIGEXPORT unsigned long SWIGSTDCALL CSharp_switch_vid_params_t_fps_get(void * j } +SWIGEXPORT void SWIGSTDCALL CSharp_switch_vid_params_t_d_width_set(void * jarg1, unsigned long jarg2) { + switch_vid_params_s *arg1 = (switch_vid_params_s *) 0 ; + uint32_t arg2 ; + + arg1 = (switch_vid_params_s *)jarg1; + arg2 = (uint32_t)jarg2; + if (arg1) (arg1)->d_width = arg2; +} + + +SWIGEXPORT unsigned long SWIGSTDCALL CSharp_switch_vid_params_t_d_width_get(void * jarg1) { + unsigned long jresult ; + switch_vid_params_s *arg1 = (switch_vid_params_s *) 0 ; + uint32_t result; + + arg1 = (switch_vid_params_s *)jarg1; + result = (uint32_t) ((arg1)->d_width); + jresult = (unsigned long)result; + return jresult; +} + + +SWIGEXPORT void SWIGSTDCALL CSharp_switch_vid_params_t_d_height_set(void * jarg1, unsigned long jarg2) { + switch_vid_params_s *arg1 = (switch_vid_params_s *) 0 ; + uint32_t arg2 ; + + arg1 = (switch_vid_params_s *)jarg1; + arg2 = (uint32_t)jarg2; + if (arg1) (arg1)->d_height = arg2; +} + + +SWIGEXPORT unsigned long SWIGSTDCALL CSharp_switch_vid_params_t_d_height_get(void * jarg1) { + unsigned long jresult ; + switch_vid_params_s *arg1 = (switch_vid_params_s *) 0 ; + uint32_t result; + + arg1 = (switch_vid_params_s *)jarg1; + result = (uint32_t) ((arg1)->d_height); + jresult = (unsigned long)result; + return jresult; +} + + SWIGEXPORT void * SWIGSTDCALL CSharp_new_switch_vid_params_t() { void * jresult ; switch_vid_params_s *result = 0 ; @@ -12952,6 +12996,34 @@ SWIGEXPORT int SWIGSTDCALL CSharp_switch_core_session_read_frame(void * jarg1, v } +SWIGEXPORT int SWIGSTDCALL CSharp_switch_core_session_transcoding(void * jarg1, void * jarg2, int jarg3) { + int jresult ; + switch_core_session_t *arg1 = (switch_core_session_t *) 0 ; + switch_core_session_t *arg2 = (switch_core_session_t *) 0 ; + switch_media_type_t arg3 ; + switch_bool_t result; + + arg1 = (switch_core_session_t *)jarg1; + arg2 = (switch_core_session_t *)jarg2; + arg3 = (switch_media_type_t)jarg3; + result = (switch_bool_t)switch_core_session_transcoding(arg1,arg2,arg3); + jresult = result; + return jresult; +} + + +SWIGEXPORT void SWIGSTDCALL CSharp_switch_core_session_passthru(void * jarg1, int jarg2, int jarg3) { + switch_core_session_t *arg1 = (switch_core_session_t *) 0 ; + switch_media_type_t arg2 ; + switch_bool_t arg3 ; + + arg1 = (switch_core_session_t *)jarg1; + arg2 = (switch_media_type_t)jarg2; + arg3 = (switch_bool_t)jarg3; + switch_core_session_passthru(arg1,arg2,arg3); +} + + SWIGEXPORT int SWIGSTDCALL CSharp_switch_core_session_read_video_frame(void * jarg1, void * jarg2, unsigned long jarg3, int jarg4) { int jresult ; switch_core_session_t *arg1 = (switch_core_session_t *) 0 ; @@ -24674,6 +24746,28 @@ SWIGEXPORT unsigned long SWIGSTDCALL CSharp_switch_timer_samplecount_get(void * } +SWIGEXPORT void SWIGSTDCALL CSharp_switch_timer_last_samplecount_set(void * jarg1, unsigned long jarg2) { + switch_timer *arg1 = (switch_timer *) 0 ; + uint32_t arg2 ; + + arg1 = (switch_timer *)jarg1; + arg2 = (uint32_t)jarg2; + if (arg1) (arg1)->last_samplecount = arg2; +} + + +SWIGEXPORT unsigned long SWIGSTDCALL CSharp_switch_timer_last_samplecount_get(void * jarg1) { + unsigned long jresult ; + switch_timer *arg1 = (switch_timer *) 0 ; + uint32_t result; + + arg1 = (switch_timer *)jarg1; + result = (uint32_t) ((arg1)->last_samplecount); + jresult = (unsigned long)result; + return jresult; +} + + SWIGEXPORT void SWIGSTDCALL CSharp_switch_timer_timer_interface_set(void * jarg1, void * jarg2) { switch_timer *arg1 = (switch_timer *) 0 ; switch_timer_interface_t *arg2 = (switch_timer_interface_t *) 0 ; diff --git a/src/mod/languages/mod_managed/managed/swig.cs b/src/mod/languages/mod_managed/managed/swig.cs index 0625ddfff2..db045db6f5 100644 --- a/src/mod/languages/mod_managed/managed/swig.cs +++ b/src/mod/languages/mod_managed/managed/swig.cs @@ -2122,6 +2122,15 @@ else return ret; } + public static switch_bool_t switch_core_session_transcoding(SWIGTYPE_p_switch_core_session session_a, SWIGTYPE_p_switch_core_session session_b, switch_media_type_t type) { + switch_bool_t ret = (switch_bool_t)freeswitchPINVOKE.switch_core_session_transcoding(SWIGTYPE_p_switch_core_session.getCPtr(session_a), SWIGTYPE_p_switch_core_session.getCPtr(session_b), (int)type); + return ret; + } + + public static void switch_core_session_passthru(SWIGTYPE_p_switch_core_session session, switch_media_type_t type, switch_bool_t on) { + freeswitchPINVOKE.switch_core_session_passthru(SWIGTYPE_p_switch_core_session.getCPtr(session), (int)type, (int)on); + } + public static switch_status_t switch_core_session_read_video_frame(SWIGTYPE_p_switch_core_session session, SWIGTYPE_p_p_switch_frame frame, uint flags, int stream_id) { switch_status_t ret = (switch_status_t)freeswitchPINVOKE.switch_core_session_read_video_frame(SWIGTYPE_p_switch_core_session.getCPtr(session), SWIGTYPE_p_p_switch_frame.getCPtr(frame), flags, stream_id); return ret; @@ -9559,6 +9568,18 @@ class freeswitchPINVOKE { [DllImport("mod_managed", EntryPoint="CSharp_switch_vid_params_t_fps_get")] public static extern uint switch_vid_params_t_fps_get(HandleRef jarg1); + [DllImport("mod_managed", EntryPoint="CSharp_switch_vid_params_t_d_width_set")] + public static extern void switch_vid_params_t_d_width_set(HandleRef jarg1, uint jarg2); + + [DllImport("mod_managed", EntryPoint="CSharp_switch_vid_params_t_d_width_get")] + public static extern uint switch_vid_params_t_d_width_get(HandleRef jarg1); + + [DllImport("mod_managed", EntryPoint="CSharp_switch_vid_params_t_d_height_set")] + public static extern void switch_vid_params_t_d_height_set(HandleRef jarg1, uint jarg2); + + [DllImport("mod_managed", EntryPoint="CSharp_switch_vid_params_t_d_height_get")] + public static extern uint switch_vid_params_t_d_height_get(HandleRef jarg1); + [DllImport("mod_managed", EntryPoint="CSharp_new_switch_vid_params_t")] public static extern IntPtr new_switch_vid_params_t(); @@ -11338,6 +11359,12 @@ class freeswitchPINVOKE { [DllImport("mod_managed", EntryPoint="CSharp_switch_core_session_read_frame")] public static extern int switch_core_session_read_frame(HandleRef jarg1, HandleRef jarg2, uint jarg3, int jarg4); + [DllImport("mod_managed", EntryPoint="CSharp_switch_core_session_transcoding")] + public static extern int switch_core_session_transcoding(HandleRef jarg1, HandleRef jarg2, int jarg3); + + [DllImport("mod_managed", EntryPoint="CSharp_switch_core_session_passthru")] + public static extern void switch_core_session_passthru(HandleRef jarg1, int jarg2, int jarg3); + [DllImport("mod_managed", EntryPoint="CSharp_switch_core_session_read_video_frame")] public static extern int switch_core_session_read_video_frame(HandleRef jarg1, HandleRef jarg2, uint jarg3, int jarg4); @@ -14065,6 +14092,12 @@ class freeswitchPINVOKE { [DllImport("mod_managed", EntryPoint="CSharp_switch_timer_samplecount_get")] public static extern uint switch_timer_samplecount_get(HandleRef jarg1); + [DllImport("mod_managed", EntryPoint="CSharp_switch_timer_last_samplecount_set")] + public static extern void switch_timer_last_samplecount_set(HandleRef jarg1, uint jarg2); + + [DllImport("mod_managed", EntryPoint="CSharp_switch_timer_last_samplecount_get")] + public static extern uint switch_timer_last_samplecount_get(HandleRef jarg1); + [DllImport("mod_managed", EntryPoint="CSharp_switch_timer_timer_interface_set")] public static extern void switch_timer_timer_interface_set(HandleRef jarg1, HandleRef jarg2); @@ -35952,7 +35985,8 @@ namespace FreeSWITCH.Native { SFF_USE_VIDEO_TIMESTAMP = (1 << 16), SFF_ENCODED = (1 << 17), SFF_TEXT_LINE_BREAK = (1 << 18), - SFF_IS_KEYFRAME = (1 << 19) + SFF_IS_KEYFRAME = (1 << 19), + SFF_EXTERNAL = (1 << 20) } } @@ -40382,11 +40416,10 @@ public enum switch_rtp_flag_t { SWITCH_RTP_FLAG_MUTE, SWITCH_RTP_FLAG_NACK, SWITCH_RTP_FLAG_TMMBR, - SWITCH_RTP_FLAG_GEN_TS_DELTA, - SWITCH_RTP_FLAG_GEN_TS_MANUAL, SWITCH_RTP_FLAG_DETECT_SSRC, SWITCH_RTP_FLAG_TEXT, SWITCH_RTP_FLAG_OLD_FIR, + SWITCH_RTP_FLAG_PASSTHRU, SWITCH_RTP_FLAG_INVALID } @@ -43410,6 +43443,16 @@ public class switch_timer : IDisposable { } } + public uint last_samplecount { + set { + freeswitchPINVOKE.switch_timer_last_samplecount_set(swigCPtr, value); + } + get { + uint ret = freeswitchPINVOKE.switch_timer_last_samplecount_get(swigCPtr); + return ret; + } + } + public switch_timer_interface timer_interface { set { freeswitchPINVOKE.switch_timer_timer_interface_set(swigCPtr, switch_timer_interface.getCPtr(value)); @@ -44192,6 +44235,26 @@ public class switch_vid_params_t : IDisposable { } } + public uint d_width { + set { + freeswitchPINVOKE.switch_vid_params_t_d_width_set(swigCPtr, value); + } + get { + uint ret = freeswitchPINVOKE.switch_vid_params_t_d_width_get(swigCPtr); + return ret; + } + } + + public uint d_height { + set { + freeswitchPINVOKE.switch_vid_params_t_d_height_set(swigCPtr, value); + } + get { + uint ret = freeswitchPINVOKE.switch_vid_params_t_d_height_get(swigCPtr); + return ret; + } + } + public switch_vid_params_t() : this(freeswitchPINVOKE.new_switch_vid_params_t(), true) { } diff --git a/src/switch_core_media.c b/src/switch_core_media.c index 2205fe2534..fbe54c63b0 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -60,7 +60,8 @@ static void gen_ice(switch_core_session_t *session, switch_media_type_t type, co typedef enum { SMF_INIT = (1 << 0), SMF_READY = (1 << 1), - SMF_JB_PAUSED = (1 << 2) + SMF_JB_PAUSED = (1 << 2), + SMF_VB_PAUSED = (1 << 3) } smh_flag_t; @@ -198,6 +199,7 @@ typedef struct switch_rtp_engine_s { switch_media_flow_t rmode; switch_media_flow_t smode; switch_thread_id_t thread_id; + switch_thread_id_t thread_write_lock; uint8_t new_ice; uint8_t new_dtls; uint32_t sdp_bw; @@ -3017,6 +3019,11 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_write_frame(switch_core_sessio engine = &smh->engines[type]; + if (type == SWITCH_MEDIA_TYPE_VIDEO) { + if (engine->thread_write_lock && engine->thread_write_lock != switch_thread_self()) { + return SWITCH_STATUS_SUCCESS; + } + } if (switch_channel_test_flag(session->channel, CF_VIDEO_ONLY) && type == SWITCH_MEDIA_TYPE_AUDIO) { return SWITCH_STATUS_SUCCESS; @@ -6188,7 +6195,7 @@ SWITCH_DECLARE(switch_file_handle_t *) switch_core_media_get_video_file(switch_c return fh; } -static void switch_core_session_write_blank_video(switch_core_session_t *session, uint32_t ms) +SWITCH_DECLARE(void) switch_core_session_write_blank_video(switch_core_session_t *session, uint32_t ms) { switch_frame_t fr = { 0 }; int i = 0; @@ -6244,6 +6251,41 @@ static void switch_core_session_write_blank_video(switch_core_session_t *session } +typedef struct core_fps_s { + float fps; + int ms; + int samples; +} core_fps_t; + +static struct core_fps_s FPS_VALS[] = { + {1.0f, 1000, 90}, + {5.0f, 200, 450}, + {10.0f, 100, 900}, + {15.0f, 66, 1364}, + {16.60f, 60, 1500}, + {20.0f, 50, 4500}, + {25.0f, 40, 2250}, + {30.0f, 33, 2700}, + {33.0f, 30, 2790}, + {66.60f, 15, 6000}, + {100.0f, 10, 9000}, + {0,0,0} +}; + + +static int video_get_fps(core_fps_t *fpsP, float fps) +{ + uint32_t i = 0; + + for (i = 0; FPS_VALS[i].ms; i++) { + if (FPS_VALS[i].fps == fps) { + *fpsP = FPS_VALS[i]; + return 1; + } + } + + return 0; +} static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void *obj) { @@ -6255,9 +6297,11 @@ static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void int buflen = SWITCH_RTP_MAX_BUF_LEN; switch_timer_t timer = { 0 }; int fps; - switch_video_read_flag_t read_flags = SVR_FLUSH|SVR_BLOCK; + switch_video_read_flag_t read_flags = SVR_FLUSH; switch_core_session_t *b_session = NULL; - + core_fps_t fps_data = { 0 }; + switch_image_t *last_frame = NULL; + if (switch_core_session_read_lock(session) != SWITCH_STATUS_SUCCESS) { return NULL; } @@ -6276,7 +6320,10 @@ static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO]; + switch_mutex_lock(smh->write_mutex[SWITCH_MEDIA_TYPE_VIDEO]); + v_engine->thread_write_lock = switch_thread_self(); + buf = switch_core_session_alloc(session, buflen); fr.packet = buf; fr.packetlen = buflen; @@ -6296,7 +6343,8 @@ static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void } - switch_core_timer_init(&timer, "soft", (int)(1000 / fps) , 1, switch_core_session_get_pool(session)); + video_get_fps(&fps_data, fps); + switch_core_timer_init(&timer, "soft", (int)(1000 / fps) , fps_data.samples, switch_core_session_get_pool(session)); while (smh->video_write_thread_running > 0 && switch_channel_up_nosig(session->channel) && smh->video_write_fh && switch_test_flag(smh->video_write_fh, SWITCH_FILE_OPEN)) { @@ -6307,15 +6355,27 @@ static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void if (smh->video_write_fh && smh->video_write_fh->mm.source_fps && smh->video_write_fh->mm.source_fps != fps) { switch_core_timer_destroy(&timer); - switch_core_timer_init(&timer, "soft", (int)(1000 / fps) , 1, switch_core_session_get_pool(session)); + video_get_fps(&fps_data, fps); + switch_core_timer_init(&timer, "soft", (int)(1000 / fps) , fps_data.samples, switch_core_session_get_pool(session)); } if (smh->video_write_fh && !switch_test_flag(smh->video_write_fh, SWITCH_FILE_FLAG_VIDEO_EOF)) { wstatus = switch_core_file_read_video(smh->video_write_fh, &fr, read_flags); if (wstatus == SWITCH_STATUS_SUCCESS) { - switch_core_session_write_video_frame(session, &fr, SWITCH_IO_FLAG_FORCE, SVR_FLUSH); - switch_img_free(&fr.img); + fr.timestamp = timer.samplecount; + fr.flags = SFF_USE_VIDEO_TIMESTAMP|SFF_RAW_RTP|SFF_RAW_RTP_PARSE_FRAME; + + if (smh->vid_params.d_width && smh->vid_params.d_height) { + switch_img_fit(&fr.img, smh->vid_params.d_width, smh->vid_params.d_height, SWITCH_FIT_SIZE); + } + + switch_core_session_write_video_frame(session, &fr, SWITCH_IO_FLAG_FORCE, 0); + + switch_img_free(&last_frame); + last_frame = fr.img; + fr.img = NULL; + } else if (wstatus != SWITCH_STATUS_BREAK && wstatus != SWITCH_STATUS_IGNORE) { switch_set_flag_locked(smh->video_write_fh, SWITCH_FILE_FLAG_VIDEO_EOF); } @@ -6323,6 +6383,25 @@ static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void switch_mutex_unlock(v_engine->mh.file_write_mutex); } + if (last_frame) { + int x = 0; + switch_rgb_color_t bgcolor; + switch_color_set_rgb(&bgcolor, "#000000"); + switch_img_fill(last_frame, 0, 0, last_frame->d_w, last_frame->d_h, &bgcolor); + fr.img = last_frame; + + for (x = 0; x < fps / 2; x++) { + switch_core_timer_next(&timer); + fr.timestamp = timer.samplecount; + fr.flags = SFF_USE_VIDEO_TIMESTAMP|SFF_RAW_RTP|SFF_RAW_RTP_PARSE_FRAME; + switch_core_session_write_video_frame(session, &fr, SWITCH_IO_FLAG_FORCE, 0); + } + switch_core_media_gen_key_frame(session); + switch_core_session_request_video_refresh(session); + switch_img_free(&last_frame); + } + + switch_core_timer_destroy(&timer); switch_core_session_rwunlock(session); @@ -6332,6 +6411,10 @@ static void *SWITCH_THREAD_FUNC video_write_thread(switch_thread_t *thread, void switch_core_session_rwunlock(b_session); } + + v_engine->thread_write_lock = 0; + switch_mutex_unlock(smh->write_mutex[SWITCH_MEDIA_TYPE_VIDEO]); + switch_channel_clear_flag(session->channel, CF_VIDEO_WRITING); smh->video_write_thread_running = 0; @@ -6473,7 +6556,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_file(switch_core_ses if (fh) { switch_threadattr_t *thd_attr = NULL; - switch_core_session_write_blank_video(session, 500); + //switch_core_session_write_blank_video(session, 500); switch_threadattr_create(&thd_attr, switch_core_session_get_pool(session)); switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); smh->video_write_thread_running = 1; @@ -6490,7 +6573,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_file(switch_core_ses switch_thread_join(&st, smh->video_write_thread); switch_mutex_lock(v_engine->mh.file_write_mutex); smh->video_write_thread = NULL; - switch_core_session_write_blank_video(session, 500); + //switch_core_session_write_blank_video(session, 500); } smh->video_write_fh = fh; @@ -8159,11 +8242,6 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_activate_rtp(switch_core_sessi } - if ((val = switch_channel_get_variable(session->channel, "rtp_gen_ts_delta_audio")) && switch_true(val)) { - flags[SWITCH_RTP_FLAG_GEN_TS_MANUAL] = 1; - flags[SWITCH_RTP_FLAG_GEN_TS_DELTA] = 1; - } - if (switch_channel_up(session->channel)) { switch_channel_set_variable(session->channel, "rtp_use_timer_name", timer_name); @@ -8872,11 +8950,6 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_activate_rtp(switch_core_sessi flags[SWITCH_RTP_FLAG_AUTOADJ]++; } - if ((val = switch_channel_get_variable(session->channel, "rtp_gen_ts_delta_video")) && switch_true(val)) { - flags[SWITCH_RTP_FLAG_GEN_TS_MANUAL] = 1; - flags[SWITCH_RTP_FLAG_GEN_TS_DELTA] = 1; - } - if (switch_channel_test_flag(session->channel, CF_PROXY_MEDIA)) { flags[SWITCH_RTP_FLAG_PROXY_MEDIA]++; } @@ -12101,12 +12174,26 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_receive_message(switch_core_se case SWITCH_MESSAGE_INDICATE_BRIDGE: { +#if 0 + if (switch_rtp_ready(v_engine->rtp_session)) { + const char *val; + + if ((!(val = switch_channel_get_variable(session->channel, "rtp_jitter_buffer_during_bridge")) || switch_false(val))) { + if (switch_rtp_get_jitter_buffer(v_engine->rtp_session) && switch_channel_test_cap_partner(session->channel, CC_FS_RTP)) { + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, + "%s PAUSE Jitterbuffer\n", switch_channel_get_name(session->channel)); + switch_rtp_pause_jitter_buffer(v_engine->rtp_session, SWITCH_TRUE); + switch_set_flag(smh, SMF_VB_PAUSED); + } + } + } +#endif + if (switch_rtp_ready(a_engine->rtp_session)) { const char *val; int ok = 0; - if (!switch_channel_test_flag(session->channel, CF_VIDEO) && - (!(val = switch_channel_get_variable(session->channel, "rtp_jitter_buffer_during_bridge")) || switch_false(val))) { + if ((!(val = switch_channel_get_variable(session->channel, "rtp_jitter_buffer_during_bridge")) || switch_false(val))) { if (switch_channel_test_flag(session->channel, CF_JITTERBUFFER) && switch_channel_test_cap_partner(session->channel, CC_FS_RTP)) { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "%s PAUSE Jitterbuffer\n", switch_channel_get_name(session->channel)); @@ -12160,6 +12247,20 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_receive_message(switch_core_se } goto end; case SWITCH_MESSAGE_INDICATE_UNBRIDGE: + +#if 0 + if (switch_rtp_ready(v_engine->rtp_session)) { + + if (switch_test_flag(smh, SMF_VB_PAUSED)) { + switch_clear_flag(smh, SMF_VB_PAUSED); + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, + "%s RESUME Video Jitterbuffer\n", switch_channel_get_name(session->channel)); + switch_rtp_pause_jitter_buffer(v_engine->rtp_session, SWITCH_FALSE); + + } + } +#endif + if (switch_rtp_ready(a_engine->rtp_session)) { if (switch_test_flag(smh, SMF_JB_PAUSED)) { @@ -13811,8 +13912,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor switch_image_t *dup_img = NULL, *img = frame->img; switch_status_t encode_status; switch_frame_t write_frame = {0}; - switch_rtp_engine_t *v_engine; - + switch_rtp_engine_t *v_engine = NULL; switch_assert(session); if (!(smh = session->media_handle)) { @@ -13828,9 +13928,6 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor return SWITCH_STATUS_FALSE; } - if (switch_channel_test_flag(session->channel, CF_VIDEO_WRITING) && !(flags & SWITCH_IO_FLAG_FORCE)) { - return SWITCH_STATUS_SUCCESS; - } if (switch_core_session_media_flow(session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_RECVONLY || switch_core_session_media_flow(session, SWITCH_MEDIA_TYPE_VIDEO) == SWITCH_MEDIA_FLOW_INACTIVE) { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG3, "Writing video to RECVONLY/INACTIVE session\n"); @@ -13845,7 +13942,6 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor return SWITCH_STATUS_SUCCESS; } - v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO]; if (smh->write_mutex[SWITCH_MEDIA_TYPE_VIDEO] && switch_mutex_trylock(smh->write_mutex[SWITCH_MEDIA_TYPE_VIDEO]) != SWITCH_STATUS_SUCCESS) { /* return CNG, another thread is already writing */ switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG1, "%s is already being written to for %s\n", @@ -13853,6 +13949,11 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor return SWITCH_STATUS_INUSE; } + v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO]; + if (v_engine->thread_write_lock && v_engine->thread_write_lock != switch_thread_self()) { + return SWITCH_STATUS_SUCCESS; + } + if (!smh->video_init && smh->mparams->video_key_first && (now - smh->video_last_key_time) > smh->mparams->video_key_first) { switch_core_media_gen_key_frame(session); @@ -13871,16 +13972,8 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor if (!img) { switch_status_t vstatus; - if (!switch_rtp_test_flag(v_engine->rtp_session, SWITCH_RTP_FLAG_GEN_TS_MANUAL)) { - switch_rtp_set_flag(v_engine->rtp_session, SWITCH_RTP_FLAG_GEN_TS_DELTA); - } - vstatus = switch_core_session_write_encoded_video_frame(session, frame, flags, stream_id); switch_goto_status(vstatus, done); - } else { - if (!switch_rtp_test_flag(v_engine->rtp_session, SWITCH_RTP_FLAG_GEN_TS_MANUAL)) { - switch_rtp_clear_flag(v_engine->rtp_session, SWITCH_RTP_FLAG_GEN_TS_DELTA); - } } @@ -13894,6 +13987,11 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_cor img = dup_img; } + if (!switch_channel_test_flag(session->channel, CF_VIDEO_WRITING)) { + smh->vid_params.d_width = img->d_w; + smh->vid_params.d_height = img->d_h; + } + if (session->bugs) { switch_media_bug_t *bp; int prune = 0; @@ -14066,6 +14164,53 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_wait_for_video_input_params( } +SWITCH_DECLARE(switch_bool_t) switch_core_session_transcoding(switch_core_session_t *session_a, switch_core_session_t *session_b, switch_media_type_t type) +{ + switch_bool_t transcoding = SWITCH_FALSE; + + switch(type) { + case SWITCH_MEDIA_TYPE_AUDIO: + transcoding = (session_a->read_codec->implementation->impl_id != session_b->read_codec->implementation->impl_id || session_a->read_impl.decoded_bytes_per_packet != session_b->read_impl.decoded_bytes_per_packet); + break; + case SWITCH_MEDIA_TYPE_VIDEO: + transcoding = (switch_channel_test_flag(session_a->channel, CF_VIDEO_DECODED_READ) || + switch_channel_test_flag(session_b->channel, CF_VIDEO_DECODED_READ)); + break; + default: + break; + } + + return transcoding; + +} + +SWITCH_DECLARE(void) switch_core_session_passthru(switch_core_session_t *session, switch_media_type_t type, switch_bool_t on) +{ + switch_rtp_engine_t *engine; + + if (!session->media_handle) return; + + engine = &session->media_handle->engines[type]; + + + if (switch_rtp_ready(engine->rtp_session)) { + if (on) { + switch_rtp_set_flag(engine->rtp_session, SWITCH_RTP_FLAG_PASSTHRU); + } else { + switch_rtp_clear_flag(engine->rtp_session, SWITCH_RTP_FLAG_PASSTHRU); + } + + if (type == SWITCH_MEDIA_TYPE_VIDEO) { + switch_core_session_request_video_refresh(session); + if (!on) { + switch_core_media_gen_key_frame(session); + } + } + + } + +} + SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags, int stream_id) { diff --git a/src/switch_ivr_bridge.c b/src/switch_ivr_bridge.c index 98d446ad8a..477ee899b2 100644 --- a/src/switch_ivr_bridge.c +++ b/src/switch_ivr_bridge.c @@ -155,6 +155,7 @@ static void video_bridge_thread(switch_core_session_t *session, void *obj) switch_frame_t *read_frame = 0; int set_decoded_read = 0, refresh_timer = 0; int refresh_cnt = 300; + int pass_val = 0, last_pass_val = 0; vh->up = 1; @@ -178,8 +179,18 @@ static void video_bridge_thread(switch_core_session_t *session, void *obj) if (switch_channel_media_up(channel)) { switch_codec_t *a_codec = switch_core_session_get_video_read_codec(vh->session_a); switch_codec_t *b_codec = switch_core_session_get_video_write_codec(vh->session_b); + + if (switch_core_session_transcoding(vh->session_a, vh->session_b, SWITCH_MEDIA_TYPE_VIDEO)) { + pass_val = 1; + } else { + pass_val = 2; + } - + if (pass_val != last_pass_val) { + switch_core_session_passthru(session, SWITCH_MEDIA_TYPE_VIDEO, pass_val == 2 ? SWITCH_TRUE : SWITCH_FALSE); + last_pass_val = pass_val; + } + if (switch_channel_test_flag(channel, CF_VIDEO_REFRESH_REQ)) { switch_channel_clear_flag(channel, CF_VIDEO_REFRESH_REQ); refresh_timer = refresh_cnt; @@ -246,6 +257,8 @@ static void video_bridge_thread(switch_core_session_t *session, void *obj) switch_core_session_request_video_refresh(vh->session_a); switch_core_session_request_video_refresh(vh->session_b); + switch_core_session_passthru(vh->session_a, SWITCH_MEDIA_TYPE_VIDEO, SWITCH_FALSE); + switch_core_session_rwunlock(vh->session_a); switch_core_session_rwunlock(vh->session_b); @@ -370,6 +383,7 @@ static void *audio_bridge_thread(switch_thread_t *thread, void *obj) struct vid_helper th = { 0 }; const char *banner_file = NULL; int played_banner = 0, banner_counter = 0; + int pass_val = 0, last_pass_val = 0; #ifdef SWITCH_VIDEO_IN_THREADS struct vid_helper vh = { 0 }; @@ -501,6 +515,17 @@ static void *audio_bridge_thread(switch_thread_t *thread, void *obj) switch_status_t status; switch_event_t *event; + if (switch_core_session_transcoding(session_a, session_b, SWITCH_MEDIA_TYPE_AUDIO)) { + pass_val = 1; + } else { + pass_val = 2; + } + + if (pass_val != last_pass_val) { + switch_core_session_passthru(session_a, SWITCH_MEDIA_TYPE_AUDIO, pass_val == 2 ? SWITCH_TRUE : SWITCH_FALSE); + last_pass_val = pass_val; + } + if (switch_channel_test_flag(chan_a, CF_TRANSFER)) { data->clean_exit = 1; } @@ -810,6 +835,8 @@ static void *audio_bridge_thread(switch_thread_t *thread, void *obj) end_of_bridge_loop: + switch_core_session_passthru(session_a, SWITCH_MEDIA_TYPE_AUDIO, SWITCH_FALSE); + #ifdef SWITCH_VIDEO_IN_THREADS if (vh.up > 0) { diff --git a/src/switch_jitterbuffer.c b/src/switch_jitterbuffer.c index 585e9890c4..9e37a5f99a 100644 --- a/src/switch_jitterbuffer.c +++ b/src/switch_jitterbuffer.c @@ -58,9 +58,10 @@ struct switch_jb_s { struct switch_jb_node_s *node_list; uint32_t last_target_seq; uint32_t highest_read_ts; + uint32_t highest_dropped_ts; uint32_t highest_read_seq; uint32_t highest_wrote_ts; - uint32_t highest_wrote_seq; + uint16_t highest_wrote_seq; uint16_t target_seq; uint32_t target_ts; uint32_t last_target_ts; @@ -104,6 +105,8 @@ struct switch_jb_s { switch_channel_t *channel; uint32_t buffer_lag; uint32_t flush; + uint32_t packet_count; + uint32_t max_packet_len; }; @@ -211,6 +214,22 @@ static inline switch_jb_node_t *new_node(switch_jb_t *jb) } if (!np) { + int mult = 25; + + if (jb->type != SJB_VIDEO) { + mult = 2; + } else { + if (jb->max_packet_len > mult) { + mult = jb->max_packet_len; + } + } + + if (jb->allocated_nodes > jb->max_frame_len * mult) { + jb_debug(jb, 2, "ALLOCATED FRAMES TOO HIGH! %d\n", jb->allocated_nodes); + switch_jb_reset(jb); + switch_mutex_unlock(jb->list_mutex); + return NULL; + } np = switch_core_alloc(jb->pool, sizeof(*np)); jb->allocated_nodes++; @@ -279,7 +298,7 @@ static inline void hide_node(switch_jb_node_t *node, switch_bool_t pop) } if (switch_core_inthash_delete(jb->node_hash, node->packet.header.seq)) { - if (node->packet.header.m && jb->type == SJB_VIDEO) { + if (node->packet.header.version == 1 && jb->type == SJB_VIDEO) { jb->complete_frames--; } } @@ -573,20 +592,45 @@ static inline void drop_second_newest_frame(switch_jb_t *jb) } #endif +static inline int check_seq(uint16_t a, uint16_t b) +{ + a = ntohs(a); + b = ntohs(b); + + if (a >= b || (b > a && b > USHRT_MAX / 2 && a < USHRT_MAX / 2)) { + return 1; + } + + return 0; +} + +static inline int check_ts(uint32_t a, uint32_t b) +{ + a = ntohl(a); + b = ntohl(b); + + if (a > b || (b > a && b > UINT_MAX / 2 && a < UINT_MAX / 2)) { + return 1; + } + + return 0; +} + static inline void add_node(switch_jb_t *jb, switch_rtp_packet_t *packet, switch_size_t len) { switch_jb_node_t *node = new_node(jb); + if (!node) { + return; + } + + node->packet = *packet; node->len = len; memcpy(node->packet.body, packet->body, len); switch_core_inthash_insert(jb->node_hash, node->packet.header.seq, node); - if (packet->header.m && jb->type == SJB_VIDEO) { - jb->complete_frames++; - } - if (jb->node_hash_ts) { switch_core_inthash_insert(jb->node_hash_ts, node->packet.header.ts, node); } @@ -621,11 +665,23 @@ static inline void add_node(switch_jb_t *jb, switch_rtp_packet_t *packet, switch } if (jb->type == SJB_VIDEO) { - if (jb->write_init && ((htons(packet->header.seq) >= htons(jb->highest_wrote_seq) && (ntohl(node->packet.header.ts) > ntohl(jb->highest_wrote_ts))) || - (ntohl(jb->highest_wrote_ts) > (UINT_MAX - 1000) && ntohl(node->packet.header.ts) < 1000))) { + if (!switch_test_flag(jb, SJB_QUEUE_ONLY)) { + jb->packet_count++; + } + + if (jb->write_init && check_seq(packet->header.seq, jb->highest_wrote_seq) && check_ts(node->packet.header.ts, jb->highest_wrote_ts)) { jb_debug(jb, 2, "WRITE frame ts: %u complete=%u/%u n:%u\n", ntohl(node->packet.header.ts), jb->complete_frames , jb->frame_len, jb->visible_nodes); jb->highest_wrote_ts = packet->header.ts; - //verify_oldest_frame(jb); + jb->complete_frames++; + + if (!switch_test_flag(jb, SJB_QUEUE_ONLY)) { + if (jb->packet_count > jb->max_packet_len) { + jb->max_packet_len = jb->packet_count; + } + + jb->packet_count = 0; + } + node->packet.header.version = 1; } else if (!jb->write_init) { jb->highest_wrote_ts = packet->header.ts; } @@ -716,9 +772,9 @@ static inline switch_status_t jb_next_packet_by_seq(switch_jb_t *jb, switch_jb_n jb_frame_inc(jb, 1); } - //if (jb->session) { - // switch_core_session_request_video_refresh(jb->session); - //} + if (jb->session) { + switch_core_session_request_video_refresh(jb->session); + } for (x = 0; x < 10; x++) { increment_seq(jb); @@ -729,6 +785,7 @@ static inline switch_status_t jb_next_packet_by_seq(switch_jb_t *jb, switch_jb_n jb_debug(jb, 2, "%s", "SAME FRAME DROPPING\n"); jb->dropped++; drop_ts(jb, node->packet.header.ts); + jb->highest_dropped_ts = ntohl(node->packet.header.ts); node = NULL; goto top; } @@ -737,6 +794,7 @@ static inline switch_status_t jb_next_packet_by_seq(switch_jb_t *jb, switch_jb_n jb_debug(jb, 2, "MISSING incremental seq: %u\n", ntohs(jb->target_seq)); } } + } else { increment_seq(jb); } @@ -816,10 +874,11 @@ SWITCH_DECLARE(void) switch_jb_set_session(switch_jb_t *jb, switch_core_session_ jb->session = session; jb->channel = switch_core_session_get_channel(session); - if (jb->type == SJB_VIDEO && (var = switch_channel_get_variable_dup(jb->channel, "jb_video_low_bitrate", SWITCH_FALSE, -1))) { + if (jb->type == SJB_VIDEO && !switch_test_flag(jb, SJB_QUEUE_ONLY) && + (var = switch_channel_get_variable_dup(jb->channel, "jb_video_low_bitrate", SWITCH_FALSE, -1))) { int tmp = atoi(var); - if (tmp > 128 && tmp < 10240) { + if (tmp >= 128 && tmp <= 10240) { jb->video_low_bitrate = (uint32_t)tmp; } } @@ -1133,6 +1192,16 @@ SWITCH_DECLARE(switch_status_t) switch_jb_put_packet(switch_jb_t *jb, switch_rtp switch_mutex_lock(jb->mutex); + if (jb->highest_dropped_ts) { + if (ntohl(packet->header.ts) < jb->highest_dropped_ts) { + jb_debug(jb, 2, "%s", "TS ALREADY DROPPED, DROPPING PACKET\n"); + switch_mutex_unlock(jb->mutex); + return SWITCH_STATUS_SUCCESS; + } + jb->highest_dropped_ts = 0; + } + + if (!want) want = got; if (switch_test_flag(jb, SJB_QUEUE_ONLY) || jb->type == SJB_AUDIO || jb->type == SJB_TEXT) { @@ -1197,6 +1266,7 @@ SWITCH_DECLARE(switch_status_t) switch_jb_get_packet_by_seq(switch_jb_t *jb, uin *packet = node->packet; *len = node->len; memcpy(packet->body, node->packet.body, node->len); + packet->header.version = 2; status = SWITCH_STATUS_SUCCESS; } else { jb_debug(jb, 2, "Missing buffered seq: %u\n", ntohs(seq)); @@ -1211,7 +1281,6 @@ SWITCH_DECLARE(switch_size_t) switch_jb_get_last_read_len(switch_jb_t *jb) return jb->last_len; } - SWITCH_DECLARE(switch_status_t) switch_jb_get_packet(switch_jb_t *jb, switch_rtp_packet_t *packet, switch_size_t *len) { switch_jb_node_t *node = NULL; @@ -1261,16 +1330,17 @@ SWITCH_DECLARE(switch_status_t) switch_jb_get_packet(switch_jb_t *jb, switch_rtp if (jb->session) { switch_core_session_request_video_refresh(jb->session); } - } else if (!switch_channel_test_flag(jb->channel, CF_VIDEO_BITRATE_UNMANAGABLE) && jb->frame_len > jb->min_frame_len * 2) { + } else if (!switch_channel_test_flag(jb->channel, CF_VIDEO_BITRATE_UNMANAGABLE) && jb->frame_len > jb->max_frame_len / 2) { switch_core_session_message_t msg = { 0 }; jb->bitrate_control = jb->video_low_bitrate; - + msg.message_id = SWITCH_MESSAGE_INDICATE_BITRATE_REQ; msg.numeric_arg = jb->bitrate_control * 1024; msg.from = __FILE__; jb_debug(jb, 2, "Force BITRATE to %d\n", jb->bitrate_control); + switch_core_session_receive_message(jb->session, &msg); switch_channel_set_flag(jb->channel, CF_VIDEO_BITRATE_UNMANAGABLE); if (jb->session) { @@ -1291,15 +1361,14 @@ SWITCH_DECLARE(switch_status_t) switch_jb_get_packet(switch_jb_t *jb, switch_rtp if ((status = jb_next_packet(jb, &node)) == SWITCH_STATUS_SUCCESS) { jb_debug(jb, 2, "Found next frame cur ts: %u seq: %u\n", htonl(node->packet.header.ts), htons(node->packet.header.seq)); - if (!jb->read_init || ntohs(node->packet.header.seq) > ntohs(jb->highest_read_seq) || - (ntohs(jb->highest_read_seq) > USHRT_MAX - 10 && ntohs(node->packet.header.seq) <= 10) ) { + if (!jb->read_init || check_seq(node->packet.header.seq, jb->highest_read_seq)) { jb->highest_read_seq = node->packet.header.seq; } - if (jb->type == SJB_TEXT || jb->type == SJB_AUDIO || - (jb->read_init && htons(node->packet.header.seq) >= htons(jb->highest_read_seq) && (ntohl(node->packet.header.ts) > ntohl(jb->highest_read_ts)))) { + if (jb->type != SJB_VIDEO || + (jb->read_init && check_seq(node->packet.header.seq, jb->highest_read_seq) && check_ts(node->packet.header.ts, jb->highest_read_ts))) { - if (jb->type == SJB_TEXT || jb->type == SJB_AUDIO) { + if (jb->type != SJB_VIDEO) { jb->complete_frames--; } jb_debug(jb, 2, "READ frame ts: %u complete=%u/%u n:%u\n", ntohl(node->packet.header.ts), jb->complete_frames , jb->frame_len, jb->visible_nodes); @@ -1351,6 +1420,7 @@ SWITCH_DECLARE(switch_status_t) switch_jb_get_packet(switch_jb_t *jb, switch_rtp *len = node->len; jb->last_len = *len; memcpy(packet->body, node->packet.body, node->len); + packet->header.version = 2; hide_node(node, SWITCH_TRUE); jb_debug(jb, 1, "GET packet ts:%u seq:%u %s\n", ntohl(packet->header.ts), ntohs(packet->header.seq), packet->header.m ? " " : ""); diff --git a/src/switch_rtp.c b/src/switch_rtp.c index 219080cf28..52ef067ded 100644 --- a/src/switch_rtp.c +++ b/src/switch_rtp.c @@ -302,8 +302,8 @@ typedef struct ts_normalize_s { uint32_t last_frame; uint32_t ts; uint32_t delta; - uint32_t delta_ct; uint32_t delta_ttl; + int last_external; } ts_normalize_t; struct switch_rtp { @@ -453,6 +453,7 @@ struct switch_rtp { switch_byte_t rtcp_auto_adj_used; uint8_t pause_jb; uint16_t last_seq; + uint16_t last_write_seq; switch_time_t last_read_time; switch_size_t last_flush_packet_count; uint32_t interdigit_delay; @@ -1656,6 +1657,10 @@ static void check_jitter(switch_rtp_t *rtp_session) (rtp_session->stats.inbound.last_processed_seq + 1), lost); rtp_session->stats.inbound.last_loss++; + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { + switch_core_session_request_video_refresh(rtp_session->session); + } + if (rtp_session->stats.inbound.last_loss > 0 && rtp_session->stats.inbound.last_loss < LOST_BURST_CAPTURE) { rtp_session->stats.inbound.loss[rtp_session->stats.inbound.last_loss] += lost; } @@ -4119,21 +4124,27 @@ SWITCH_DECLARE(switch_jb_t *) switch_rtp_get_jitter_buffer(switch_rtp_t *rtp_ses SWITCH_DECLARE(switch_status_t) switch_rtp_pause_jitter_buffer(switch_rtp_t *rtp_session, switch_bool_t pause) { - - if (!switch_rtp_ready(rtp_session) || !rtp_session->jb) { - return SWITCH_STATUS_FALSE; - } - - if (!!pause == !!rtp_session->pause_jb) { - return SWITCH_STATUS_FALSE; - } + int new_val; if (rtp_session->pause_jb && !pause) { - switch_jb_reset(rtp_session->jb); + if (rtp_session->jb) { + switch_jb_reset(rtp_session->jb); + } + + if (rtp_session->vb) { + switch_jb_reset(rtp_session->vb); + } } - rtp_session->pause_jb = pause ? 1 : 0; + new_val = pause ? 1 : -1; + if (rtp_session->pause_jb + new_val > -1) { + rtp_session->pause_jb += new_val; + } + + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG1, + "Jitterbuffer %s is %s\n", rtp_type(rtp_session), rtp_session->pause_jb ? "paused" : "enabled"); + return SWITCH_STATUS_SUCCESS; } @@ -4730,37 +4741,7 @@ SWITCH_DECLARE(void) switch_rtp_set_flags(switch_rtp_t *rtp_session, switch_rtp_ for(i = 0; i < SWITCH_RTP_FLAG_INVALID; i++) { if (flags[i]) { - rtp_session->flags[i] = flags[i]; - - if (i == SWITCH_RTP_FLAG_AUTOADJ) { - rtp_session->autoadj_window = 20; - rtp_session->autoadj_threshold = 10; - rtp_session->autoadj_tally = 0; - if (rtp_session->session) { - switch_channel_t *channel = switch_core_session_get_channel(rtp_session->session); - const char *x = switch_channel_get_variable(channel, "rtp_auto_adjust_threshold"); - if (x && *x) { - int xn = atoi(x); - if (xn > 0 && xn <= 65535) { - rtp_session->autoadj_window = xn*2; - rtp_session->autoadj_threshold = xn; - } - } - } - - - rtp_session->flags[SWITCH_RTP_FLAG_RTCP_AUTOADJ] = 1; - - - rtp_session->rtcp_autoadj_window = 20; - rtp_session->rtcp_autoadj_threshold = 1; - rtp_session->rtcp_autoadj_tally = 0; - - - rtp_flush_read_buffer(rtp_session, SWITCH_RTP_FLUSH_ONCE); - } else if (i == SWITCH_RTP_FLAG_NOBLOCK && rtp_session->sock_input) { - switch_socket_opt_set(rtp_session->sock_input, SWITCH_SO_NONBLOCK, TRUE); - } + switch_rtp_set_flag(rtp_session, i); } } } @@ -4771,19 +4752,24 @@ SWITCH_DECLARE(void) switch_rtp_clear_flags(switch_rtp_t *rtp_session, switch_rt for(i = 0; i < SWITCH_RTP_FLAG_INVALID; i++) { if (flags[i]) { - rtp_session->flags[i] = 0; + switch_rtp_clear_flag(rtp_session, i); } } } SWITCH_DECLARE(void) switch_rtp_set_flag(switch_rtp_t *rtp_session, switch_rtp_flag_t flag) { + int old_flag = rtp_session->flags[flag]; switch_mutex_lock(rtp_session->flag_mutex); rtp_session->flags[flag] = 1; switch_mutex_unlock(rtp_session->flag_mutex); - if (flag == SWITCH_RTP_FLAG_DTMF_ON) { + if (flag == SWITCH_RTP_FLAG_PASSTHRU) { + if (!old_flag) { + switch_rtp_pause_jitter_buffer(rtp_session, SWITCH_TRUE); + } + } else if (flag == SWITCH_RTP_FLAG_DTMF_ON) { rtp_session->stats.inbound.last_processed_seq = 0; } else if (flag == SWITCH_RTP_FLAG_FLUSH) { reset_jitter_seq(rtp_session); @@ -4832,12 +4818,17 @@ SWITCH_DECLARE(uint32_t) switch_rtp_test_flag(switch_rtp_t *rtp_session, switch_ SWITCH_DECLARE(void) switch_rtp_clear_flag(switch_rtp_t *rtp_session, switch_rtp_flag_t flag) { + int old_flag = rtp_session->flags[flag]; switch_mutex_lock(rtp_session->flag_mutex); rtp_session->flags[flag] = 0; switch_mutex_unlock(rtp_session->flag_mutex); - if (flag == SWITCH_RTP_FLAG_DTMF_ON) { + if (flag == SWITCH_RTP_FLAG_PASSTHRU) { + if (old_flag) { + switch_rtp_pause_jitter_buffer(rtp_session, SWITCH_FALSE); + } + } else if (flag == SWITCH_RTP_FLAG_DTMF_ON) { rtp_session->stats.inbound.last_processed_seq = 0; } else if (flag == SWITCH_RTP_FLAG_PAUSE) { reset_jitter_seq(rtp_session); @@ -5803,7 +5794,7 @@ static switch_status_t read_rtp_packet(switch_rtp_t *rtp_session, switch_size_t return SWITCH_STATUS_SUCCESS; } - if (rtp_session->vb && jb_valid(rtp_session)) { + if (rtp_session->vb && !rtp_session->pause_jb && jb_valid(rtp_session)) { status = switch_jb_put_packet(rtp_session->vb, (switch_rtp_packet_t *) &rtp_session->recv_msg, *bytes); if (status == SWITCH_STATUS_TOO_LATE) { @@ -5900,7 +5891,7 @@ static switch_status_t read_rtp_packet(switch_rtp_t *rtp_session, switch_size_t } } - if (rtp_session->vb && jb_valid(rtp_session)) { + if (rtp_session->vb && !rtp_session->pause_jb && jb_valid(rtp_session)) { switch_status_t vstatus = switch_jb_get_packet(rtp_session->vb, (switch_rtp_packet_t *) &rtp_session->recv_msg, bytes); status = vstatus; @@ -6700,7 +6691,7 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_ pt = 100000; } - if (rtp_session->vb) { + if (rtp_session->vb && !rtp_session->pause_jb) { if (switch_jb_poll(rtp_session->vb)) { pt = 1000; } @@ -6726,7 +6717,7 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_ } if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { - got_jb = (rtp_session->vb && switch_jb_poll(rtp_session->vb)); + got_jb = (rtp_session->vb && !rtp_session->pause_jb && switch_jb_poll(rtp_session->vb)); } else { got_jb = SWITCH_TRUE; } @@ -6922,7 +6913,7 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_ process_rtcp_packet(rtp_session, &rtcp_bytes); ret = 1; - if (!rtp_session->flags[SWITCH_RTP_FLAG_USE_TIMER] && rtp_session->timer.interval) { + if (!rtp_session->flags[SWITCH_RTP_FLAG_USE_TIMER] && rtp_session->timer.interval && !rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { switch_core_timer_sync(&rtp_session->timer); reset_jitter_seq(rtp_session); } @@ -7422,6 +7413,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_zerocopy_read_frame(switch_rtp_t *rtp frame->source = __FILE__; switch_set_flag(frame, SFF_RAW_RTP); + switch_set_flag(frame, SFF_EXTERNAL); if (frame->payload == rtp_session->recv_te) { switch_set_flag(frame, SFF_RFC2833); } @@ -7649,59 +7641,65 @@ static int rtp_common_write(switch_rtp_t *rtp_session, } } - - if (switch_rtp_test_flag(rtp_session, SWITCH_RTP_FLAG_GEN_TS_DELTA) || switch_rtp_test_flag(rtp_session, SWITCH_RTP_FLAG_VIDEO)) { + if (switch_rtp_test_flag(rtp_session, SWITCH_RTP_FLAG_VIDEO)) { + int external = (*flags & SFF_EXTERNAL); /* Normalize the timestamps to our own base by generating a made up starting point then adding the measured deltas to that base so if the timestamps and ssrc of the source change, it will not break the other end's jitter bufffer / decoder etc *cough* CHROME *cough* */ if (!rtp_session->ts_norm.ts) { - if (switch_rtp_test_flag(rtp_session, SWITCH_RTP_FLAG_GEN_TS_DELTA)) { - rtp_session->ts_norm.ts = (uint32_t) rand() % 1000000 + 1; - } else { - switch_core_timer_sync(&rtp_session->timer); - rtp_session->ts_norm.ts = rtp_session->timer.samplecount; - } + rtp_session->ts_norm.ts = (uint32_t) rand() % 1000000 + 1; } - if (!rtp_session->ts_norm.last_ssrc || send_msg->header.ssrc != rtp_session->ts_norm.last_ssrc) { - if (switch_rtp_test_flag(rtp_session, SWITCH_RTP_FLAG_GEN_TS_DELTA)) { - if (rtp_session->ts_norm.last_ssrc) { - rtp_session->ts_norm.delta_ct = 1; - rtp_session->ts_norm.delta_ttl = 0; - if (rtp_session->ts_norm.delta) { - rtp_session->ts_norm.ts += rtp_session->ts_norm.delta; - } - } + if (!rtp_session->ts_norm.last_ssrc || send_msg->header.ssrc != rtp_session->ts_norm.last_ssrc || rtp_session->ts_norm.last_external != external) { + switch_core_session_t *other_session; + + switch_core_session_request_video_refresh(rtp_session->session); + switch_core_media_gen_key_frame(rtp_session->session); + + if (switch_core_session_get_partner(rtp_session->session, &other_session) == SWITCH_STATUS_SUCCESS) { + switch_core_session_request_video_refresh(other_session); + switch_core_media_gen_key_frame(other_session); + switch_core_session_rwunlock(other_session); + } + + if (rtp_session->ts_norm.last_ssrc) { + rtp_session->ts_norm.delta_ttl = 0; + rtp_session->ts_norm.ts++; } rtp_session->ts_norm.last_ssrc = send_msg->header.ssrc; rtp_session->ts_norm.last_frame = ntohl(send_msg->header.ts); } + rtp_session->ts_norm.last_external = external; if (ntohl(send_msg->header.ts) != rtp_session->ts_norm.last_frame) { - int32_t delta = (int32_t) (ntohl(send_msg->header.ts) - rtp_session->ts_norm.last_frame); + int32_t delta; + int64_t x, y; - if (switch_rtp_test_flag(rtp_session, SWITCH_RTP_FLAG_VIDEO) && delta > 0 && delta < 90000) { + x = rtp_session->ts_norm.last_frame; + y = ntohl(send_msg->header.ts); + + if (x > UINT32_MAX / 2 && y < UINT32_MAX / 2) { + x -= (int64_t)UINT32_MAX+1; + } + + delta = (int32_t)y-x; + + if (delta < 0 || delta > 90000) { + switch_core_media_gen_key_frame(rtp_session->session); + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG1, + "Timestamp shift detected last: %d this: %d delta: %d stick with prev delta: %d\n", + rtp_session->ts_norm.last_frame, ntohl(send_msg->header.ts), delta, rtp_session->ts_norm.delta); + } else { rtp_session->ts_norm.delta = delta; } - if (switch_rtp_test_flag(rtp_session, SWITCH_RTP_FLAG_GEN_TS_DELTA)) { - rtp_session->ts_norm.ts += rtp_session->ts_norm.delta; - } else { - switch_core_timer_sync(&rtp_session->timer); - if (rtp_session->ts_norm.ts == rtp_session->timer.samplecount) { - rtp_session->ts_norm.ts = rtp_session->timer.samplecount + 1; - } else { - rtp_session->ts_norm.ts = rtp_session->timer.samplecount; - } - if (send_msg->header.m) { - rtp_session->ts_norm.last_frame++; - } - } - } + rtp_session->ts_norm.ts += rtp_session->ts_norm.delta; + } + rtp_session->ts_norm.last_frame = ntohl(send_msg->header.ts); send_msg->header.ts = htonl(rtp_session->ts_norm.ts); } @@ -7844,8 +7842,27 @@ static int rtp_common_write(switch_rtp_t *rtp_session, } if (send) { - send_msg->header.seq = htons(++rtp_session->seq); + int delta = 1; + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && (*flags & SFF_EXTERNAL) && rtp_session->stats.outbound.packet_count && rtp_session->flags[SWITCH_RTP_FLAG_PASSTHRU]) { + int32_t x; + int32_t y; + + x = rtp_session->last_write_seq; + y = ntohs(send_msg->header.seq); + + if (x > UINT16_MAX / 2 && y < UINT16_MAX / 2) { + x -= (int32_t)UINT16_MAX+1; + } + + delta = y-x; + } + + rtp_session->seq += delta; + + send_msg->header.seq = htons(rtp_session->seq); + rtp_session->last_write_seq = rtp_session->seq; + if (rtp_session->flags[SWITCH_RTP_FLAG_BYTESWAP] && send_msg->header.pt == rtp_session->payload) { switch_swap_linear((int16_t *)send_msg->body, (int) datalen); } @@ -7993,7 +8010,8 @@ static int rtp_common_write(switch_rtp_t *rtp_session, // //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "SEND %u\n", ntohs(send_msg->header.seq)); //} if (switch_socket_sendto(rtp_session->sock_output, rtp_session->remote_addr, 0, (void *) send_msg, &bytes) != SWITCH_STATUS_SUCCESS) { - rtp_session->seq--; + rtp_session->seq -= delta; + ret = -1; goto end; } @@ -8239,7 +8257,7 @@ SWITCH_DECLARE(int) switch_rtp_write_frame(switch_rtp_t *rtp_session, switch_fra fwd = (rtp_session->flags[SWITCH_RTP_FLAG_RAW_WRITE] && (switch_test_flag(frame, SFF_RAW_RTP) || switch_test_flag(frame, SFF_RAW_RTP_PARSE_FRAME))) ? 1 : 0; - if (!fwd && !rtp_session->sending_dtmf && !rtp_session->queue_delay && + if (!fwd && !rtp_session->sending_dtmf && !rtp_session->queue_delay && !rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && rtp_session->flags[SWITCH_RTP_FLAG_RAW_WRITE] && (rtp_session->rtp_bugs & RTP_BUG_GEN_ONE_GEN_ALL)) { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_WARNING, "Generating RTP locally but timestamp passthru is configured, disabling....\n"); @@ -8308,6 +8326,7 @@ SWITCH_DECLARE(int) switch_rtp_write_frame(switch_rtp_t *rtp_session, switch_fra if (switch_test_flag(frame, SFF_RAW_RTP_PARSE_FRAME)) { send_msg->header.version = 2; send_msg->header.m = frame->m; + send_msg->header.ts = htonl(frame->timestamp); if (frame->ssrc) { send_msg->header.ssrc = htonl(frame->ssrc); diff --git a/src/switch_time.c b/src/switch_time.c index efa7205b4e..2847158861 100644 --- a/src/switch_time.c +++ b/src/switch_time.c @@ -386,6 +386,14 @@ static switch_status_t timer_generic_sync(switch_timer_t *timer) timer->tick = (elapsed / timer->interval) / 1000; timer->samplecount = (uint32_t)(timer->tick * timer->samples); + if (timer->interval == 1 && timer->samplecount == timer->last_samplecount) { + timer->samplecount++; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Timer sync too often\n"); + } + timer->last_samplecount = timer->samplecount; + + + return SWITCH_STATUS_SUCCESS; }