diff --git a/src/include/private/switch_core_pvt.h b/src/include/private/switch_core_pvt.h index 3ff631453b..e166f9fa5a 100644 --- a/src/include/private/switch_core_pvt.h +++ b/src/include/private/switch_core_pvt.h @@ -190,6 +190,10 @@ struct switch_core_session { uint32_t decoder_errors; switch_core_video_thread_callback_func_t *_video_thread_callback; void *_video_thread_user_data; + //switch_time_t last_video_write_time; + + switch_image_write_callback_t image_write_callback; + void *image_write_callback_user_data; }; struct switch_media_bug { diff --git a/src/include/switch_core.h b/src/include/switch_core.h index 83cc10aa4d..ca48d8118c 100644 --- a/src/include/switch_core.h +++ b/src/include/switch_core.h @@ -1275,6 +1275,26 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_frame(_In_ switch_core_ SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(_In_ switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags, int stream_id); +/*! + \brief Write a video image to a session using a video frame + \param session the session to write to + \param frame a pointer to a frame to use for write with proper codec + \param img the image structure with the image data + \param the size for packetization + \param flag pointer to frame flags to pass in / out + \return SWITCH_STATUS_SUCCESS a if the image was written +*/ +SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_image(switch_core_session_t *session, switch_frame_t *frame, + switch_image_t *img, switch_size_t size, uint32_t *flag); +/*! + \brief set a callback to be called after each frame of an image is written + \param session the session to write to + \param callback the function to call + \param user_data the user data pointer to pass as an arguement to the callback + \return void +*/ +SWITCH_DECLARE(void) switch_core_session_set_image_write_callback(switch_core_session_t *session, switch_image_write_callback_t callback, void *user_data); + /*! \brief Write a video frame to a session \param session the session to write to diff --git a/src/include/switch_core_media.h b/src/include/switch_core_media.h index 056a5abfce..b0f88acf49 100644 --- a/src/include/switch_core_media.h +++ b/src/include/switch_core_media.h @@ -305,6 +305,8 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess switch_codec_control_type_t *rtype, void **ret_data); +SWITCH_DECLARE(switch_timer_t *) switch_core_media_get_timer(switch_core_session_t *session, switch_media_type_t mtype); + SWITCH_END_EXTERN_C #endif /* For Emacs: diff --git a/src/include/switch_module_interfaces.h b/src/include/switch_module_interfaces.h index 60298f0f66..f989b2808c 100644 --- a/src/include/switch_module_interfaces.h +++ b/src/include/switch_module_interfaces.h @@ -217,6 +217,7 @@ struct switch_timer { switch_size_t diff; switch_time_t start; uint64_t tick; + }; typedef enum { diff --git a/src/include/switch_rtp.h b/src/include/switch_rtp.h index 0c2002d506..3046946657 100644 --- a/src/include/switch_rtp.h +++ b/src/include/switch_rtp.h @@ -268,6 +268,9 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_activate_ice(switch_rtp_t *rtp_sessio */ SWITCH_DECLARE(switch_status_t) switch_rtp_activate_rtcp(switch_rtp_t *rtp_session, int send_rate, switch_port_t remote_port, switch_bool_t mux); + +SWITCH_DECLARE(switch_timer_t *) switch_rtp_get_media_timer(switch_rtp_t *rtp_session); + /*! \brief Acvite a jitter buffer on an RTP session \param rtp_session the rtp session @@ -285,6 +288,9 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_deactivate_jitter_buffer(switch_rtp_t SWITCH_DECLARE(switch_status_t) switch_rtp_pause_jitter_buffer(switch_rtp_t *rtp_session, switch_bool_t pause); SWITCH_DECLARE(stfu_instance_t *) switch_rtp_get_jitter_buffer(switch_rtp_t *rtp_session); + + + /*! \brief Set an RTP Flag \param rtp_session the RTP session diff --git a/src/include/switch_types.h b/src/include/switch_types.h index 56093011c3..2dbc4a20a6 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -226,6 +226,7 @@ SWITCH_BEGIN_EXTERN_C #define SWITCH_DTMF_LOG_LEN 1000 #define SWITCH_MAX_TRANS 2000 #define SWITCH_CORE_SESSION_MAX_PRIVATES 2 +#define SWITCH_DEFAULT_VIDEO_SIZE 1500 /* Jitter */ #define JITTER_VARIANCE_THRESHOLD 400.0 @@ -1494,7 +1495,8 @@ typedef enum { SFF_NOT_AUDIO = (1 << 9), SFF_RTCP = (1 << 10), SFF_MARKER = (1 << 11), - SFF_WAIT_KEY_FRAME = (1 << 12) + SFF_WAIT_KEY_FRAME = (1 << 12), + SFF_RAW_RTP_PARSE_FRAME = (1 << 13) } switch_frame_flag_enum_t; typedef uint32_t switch_frame_flag_t; @@ -2196,6 +2198,8 @@ typedef switch_status_t (*switch_core_codec_control_func_t) (switch_codec_t *cod void **ret_data); +typedef switch_status_t (*switch_image_write_callback_t) (switch_core_session_t *session, switch_frame_t *frame, switch_image_t *img, void *user_data); + typedef switch_status_t (*switch_core_codec_init_func_t) (switch_codec_t *, switch_codec_flag_t, const switch_codec_settings_t *codec_settings); typedef switch_status_t (*switch_core_codec_fmtp_parse_func_t) (const char *fmtp, switch_codec_fmtp_t *codec_fmtp); typedef switch_status_t (*switch_core_codec_destroy_func_t) (switch_codec_t *); diff --git a/src/mod/formats/mod_vlc/mod_vlc.c b/src/mod/formats/mod_vlc/mod_vlc.c index ea760f1838..9c50f6de92 100644 --- a/src/mod/formats/mod_vlc/mod_vlc.c +++ b/src/mod/formats/mod_vlc/mod_vlc.c @@ -107,7 +107,6 @@ struct vlc_video_context { uint8_t video_packet[1500 + 12]; void *raw_yuyv_data; switch_image_t *img; - switch_time_t last_video_ts; switch_payload_t pt; uint32_t seq; int width; @@ -241,68 +240,19 @@ static void vlc_video_unlock_dummy_callback(void *data, void *id, void *const *p static void vlc_video_unlock_callback(void *data, void *id, void *const *p_pixels) { - vlc_video_context_t *context = (vlc_video_context_t *)data; + vlc_video_context_t *context = (vlc_video_context_t *) data; switch_frame_t *frame = context->vid_frame; - uint32_t flag = 0; - uint32_t encoded_data_len = 1500; - switch_time_t now = (switch_time_t)(switch_micro_time_now() / 1000); - switch_codec_t *codec = switch_core_session_get_video_write_codec(context->session); - long delta; switch_assert(id == NULL); /* picture identifier, not needed here */ - switch_assert(codec); - - if (now - context->last_video_ts < 60) goto end; if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0); - if (!context->img) goto end; + + switch_assert(context->img); yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height); - delta = now - context->last_video_ts; + switch_core_session_write_video_image(context->session, frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, NULL); - if (delta > 0) { - frame->timestamp += delta * 90; - context->last_video_ts = now; - } - //printf("WTF VLC d_w=%d d_h=%d w=%d h=%d\n", context->img->d_w, context->img->d_h, context->img->w, context->img->h); - - //context->img->d_w = context->img->w; - //context->img->d_h = context->img->h; - - switch_core_codec_encode_video(codec, context->img, frame->data, &encoded_data_len, &flag); - - while(encoded_data_len) { - //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%ld\n", codec->implementation->iananame, encoded_data_len, flag, context->last_video_ts); - - frame->datalen = encoded_data_len; - frame->packetlen = frame->datalen + 12; - frame->m = flag & SFF_MARKER ? 1 : 0; - - if (1) { - /* set correct mark and ts */ - switch_rtp_hdr_t *rtp = (switch_rtp_hdr_t *)frame->packet; - - memset(rtp, 0, 12); - rtp->version = 2; - rtp->m = frame->m; - rtp->ts = htonl(frame->timestamp); - rtp->ssrc = (uint32_t) ((intptr_t) rtp + (uint32_t) switch_epoch_time_now(NULL)); - - switch_set_flag(frame, SFF_RAW_RTP); - } - - switch_set_flag(frame, SFF_RAW_RTP); - // switch_set_flag(frame, SFF_PROXY_PACKET); - - switch_core_session_write_video_frame(context->session, frame, SWITCH_IO_FLAG_NONE, 0); - - encoded_data_len = 1500; - - switch_core_codec_encode_video(codec, NULL, frame->data, &encoded_data_len, &flag); - } - -end: switch_mutex_unlock(context->video_mutex); } @@ -323,72 +273,28 @@ static void do_buffer_frame(vlc_video_context_t *context) switch_mutex_unlock(context->video_mutex); } + static void vlc_video_channel_unlock_callback(void *data, void *id, void *const *p_pixels) { vlc_video_context_t *context = (vlc_video_context_t *)data; uint32_t flag = 0; - uint32_t encoded_data_len = 1500; - switch_codec_t *codec = switch_core_session_get_video_write_codec(context->session); switch_frame_t *frame = context->vid_frame; - switch_time_t now = (switch_time_t)(switch_micro_time_now() / 1000); - long delta; switch_assert(id == NULL); /* picture identifier, not needed here */ - switch_assert(codec); if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0); - if (!context->img) goto end; + switch_assert(context->img); yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height); - encoded_data_len = 1500; - - frame->packet = context->video_packet; - frame->data = context->video_packet + 12; - delta = now - context->last_video_ts; - - if (delta > 0) { - frame->timestamp += delta * 90; - context->last_video_ts = now; - } if (context->video_refresh_req > 0) { flag |= SFF_WAIT_KEY_FRAME; context->video_refresh_req--; } - switch_core_codec_encode_video(codec, context->img, frame->data, &encoded_data_len, &flag); + switch_core_session_write_video_image(context->session, frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, &flag); - while(encoded_data_len) { - // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%u\n", codec->implementation->iananame, encoded_data_len, flag, context->ts); - - frame->datalen = encoded_data_len; - frame->packetlen = frame->datalen + 12; - frame->m = flag & SFF_MARKER ? 1 : 0; - - if (1) { - /* set correct mark and ts */ - switch_rtp_hdr_t *rtp = (switch_rtp_hdr_t *)frame->packet; - - memset(rtp, 0, 12); - rtp->version = 2; - rtp->m = frame->m; - rtp->ts = htonl(frame->timestamp); - rtp->ssrc = (uint32_t) ((intptr_t) rtp + (uint32_t) switch_epoch_time_now(NULL)); - - switch_set_flag(frame, SFF_RAW_RTP); - } - - switch_set_flag(frame, SFF_RAW_RTP); - switch_set_flag(frame, SFF_PROXY_PACKET); - - do_buffer_frame(context); - - encoded_data_len = 1500; - switch_core_codec_encode_video(codec, NULL, frame->data, &encoded_data_len, &flag); - } - -end: switch_mutex_unlock(context->video_mutex); } @@ -720,6 +626,7 @@ SWITCH_STANDARD_APP(play_video_function) switch_size_t audio_datalen; + switch_channel_clear_flag(channel, CF_VIDEO_ECHO); context = switch_core_session_alloc(session, sizeof(vlc_video_context_t)); @@ -974,6 +881,7 @@ static switch_status_t vlc_write_frame(switch_core_session_t *session, switch_fr static switch_status_t vlc_read_video_frame(switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags, int stream_id); static switch_status_t vlc_receive_message(switch_core_session_t *session, switch_core_session_message_t *msg); static switch_status_t vlc_kill_channel(switch_core_session_t *session, int sig); +static switch_status_t vlc_state_change(switch_core_session_t *session); typedef struct { switch_core_session_t *session; @@ -1014,7 +922,7 @@ switch_io_routines_t vlc_io_routines = { /*send_dtmf*/ NULL, /*receive_message*/ vlc_receive_message, /*receive_event*/ NULL, - /*state_change*/ NULL, + /*state_change*/ vlc_state_change, /*read_video_frame*/ vlc_read_video_frame, /*write_video_frame*/ NULL, /*state_run*/ NULL @@ -1118,12 +1026,25 @@ fail: return status; } +static switch_status_t vlc_channel_img_callback(switch_core_session_t *session, switch_frame_t *frame, switch_image_t *img, void *user_data) +{ + vlc_video_context_t *context = (vlc_video_context_t *) user_data; + + do_buffer_frame(context); + + return SWITCH_STATUS_SUCCESS; +} + + static switch_status_t channel_on_init(switch_core_session_t *session) { switch_channel_t *channel = switch_core_session_get_channel(session); + vlc_private_t *tech_pvt = switch_core_session_get_private(session); switch_channel_set_state(channel, CS_CONSUME_MEDIA); + switch_core_session_set_image_write_callback(session, vlc_channel_img_callback, tech_pvt->context); + return SWITCH_STATUS_SUCCESS; } @@ -1488,6 +1409,19 @@ static switch_status_t vlc_receive_message(switch_core_session_t *session, switc return SWITCH_STATUS_SUCCESS; } +static switch_status_t vlc_state_change(switch_core_session_t *session) +{ + switch_channel_t *channel = switch_core_session_get_channel(session); + switch_channel_state_t state = switch_channel_get_state(channel); + + if (state == CS_HANGUP || state == CS_ROUTING) { + switch_core_session_video_reset(session); + } + + return SWITCH_STATUS_SUCCESS; +} + + static switch_status_t vlc_kill_channel(switch_core_session_t *session, int sig) { vlc_private_t *tech_pvt = switch_core_session_get_private(session); diff --git a/src/switch_core_io.c b/src/switch_core_io.c index 7bda3d05d3..f7515b9264 100644 --- a/src/switch_core_io.c +++ b/src/switch_core_io.c @@ -36,6 +36,53 @@ #include #include "private/switch_core_pvt.h" +SWITCH_DECLARE(void) switch_core_session_set_image_write_callback(switch_core_session_t *session, switch_image_write_callback_t callback, void *user_data) +{ + session->image_write_callback = callback; + session->image_write_callback_user_data = user_data; +} + +SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_image(switch_core_session_t *session, switch_frame_t *frame, + switch_image_t *img, switch_size_t size, uint32_t *flag) +{ + uint32_t encoded_data_len = size, lflag = 0, *flagp = flag; + switch_codec_t *codec = switch_core_session_get_video_write_codec(session); + switch_timer_t *timer; + + switch_assert(session); + + if (!flag) { + flagp = &lflag; + } + + timer = switch_core_media_get_timer(session, SWITCH_MEDIA_TYPE_VIDEO); + switch_assert(timer); + + switch_core_codec_encode_video(codec, img, frame->data, &encoded_data_len, flagp); + + while(encoded_data_len) { + + frame->datalen = encoded_data_len; + frame->packetlen = frame->datalen + 12; + frame->m = (*flagp & SFF_MARKER) ? 1 : 0; + frame->timestamp = timer->samplecount; + + switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME); + + switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0); + + if (session->image_write_callback) { + session->image_write_callback(session, frame, img, session->image_write_callback_user_data); + } + + encoded_data_len = size; + switch_core_codec_encode_video(codec, NULL, frame->data, &encoded_data_len, flagp); + } + + return SWITCH_STATUS_SUCCESS; +} + + SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags, int stream_id) { diff --git a/src/switch_core_media.c b/src/switch_core_media.c index 8387eaf372..5b620548c1 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -7211,6 +7211,8 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess nack++; } + nack = v_engine->nack = pli = v_engine->pli = 0; + if (vp8) { if (v_engine->fir || fir) { @@ -9445,6 +9447,25 @@ SWITCH_DECLARE(char *) switch_core_media_process_sdp_filter(const char *sdp, con } +SWITCH_DECLARE(switch_timer_t *) switch_core_media_get_timer(switch_core_session_t *session, switch_media_type_t mtype) +{ + switch_rtp_engine_t *engine = NULL; + switch_media_handle_t *smh = NULL; + + switch_assert(session); + + if (!(smh = session->media_handle)) { + return NULL; + } + + if (!(engine = &smh->engines[mtype])) { + return NULL; + } + + return switch_rtp_get_media_timer(engine->rtp_session); + +} + SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_session_t *session, switch_media_type_t mtype, switch_io_type_t iotype, @@ -9481,8 +9502,6 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess return SWITCH_STATUS_FALSE; } - - /* For Emacs: * Local Variables: * mode:c diff --git a/src/switch_core_session.c b/src/switch_core_session.c index 76ca5a9e81..3381852565 100644 --- a/src/switch_core_session.c +++ b/src/switch_core_session.c @@ -2677,6 +2677,8 @@ SWITCH_DECLARE(void) switch_core_session_video_reset(switch_core_session_t *sess switch_channel_set_flag(session->channel, CF_VIDEO_ECHO); switch_channel_clear_flag(session->channel, CF_VIDEO_PASSIVE); switch_core_session_refresh_video(session); + session->image_write_callback = NULL; + session->image_write_callback_user_data = NULL; } SWITCH_DECLARE(switch_status_t) switch_core_session_execute_application_get_flags(switch_core_session_t *session, const char *app, diff --git a/src/switch_rtp.c b/src/switch_rtp.c index d7b46ac213..f272eecd15 100644 --- a/src/switch_rtp.c +++ b/src/switch_rtp.c @@ -3575,7 +3575,14 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_create(switch_rtp_t **new_rtp_session switch_rtp_clear_flag(rtp_session, SWITCH_RTP_FLAG_USE_TIMER); } } else { - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Not using a timer\n"); + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { + if (switch_core_timer_init(&rtp_session->timer, "soft", 1, 90, pool) == SWITCH_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Starting video timer.\n"); + } + } else { + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Not using a timer\n"); + } + switch_rtp_clear_flag(rtp_session, SWITCH_RTP_FLAG_USE_TIMER); switch_rtp_clear_flag(rtp_session, SWITCH_RTP_FLAG_NOBLOCK); } @@ -3781,6 +3788,19 @@ static void jb_callback(stfu_instance_t *i, void *udata) } +SWITCH_DECLARE(switch_timer_t *) switch_rtp_get_media_timer(switch_rtp_t *rtp_session) +{ + if (rtp_session->timer.timer_interface) { + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { + switch_core_timer_sync(&rtp_session->timer); + } + return &rtp_session->timer; + } + + return NULL; +} + + SWITCH_DECLARE(stfu_instance_t *) switch_rtp_get_jitter_buffer(switch_rtp_t *rtp_session) { if (!switch_rtp_ready(rtp_session) || !rtp_session->jb) { @@ -6953,7 +6973,8 @@ SWITCH_DECLARE(int) switch_rtp_write_frame(switch_rtp_t *rtp_session, switch_fra } #endif - fwd = (rtp_session->flags[SWITCH_RTP_FLAG_RAW_WRITE] && switch_test_flag(frame, SFF_RAW_RTP)) ? 1 : 0; + fwd = (rtp_session->flags[SWITCH_RTP_FLAG_RAW_WRITE] && + (switch_test_flag(frame, SFF_RAW_RTP) || switch_test_flag(frame, SFF_RAW_RTP_PARSE_FRAME))) ? 1 : 0; if (!fwd && !rtp_session->sending_dtmf && !rtp_session->queue_delay && rtp_session->flags[SWITCH_RTP_FLAG_RAW_WRITE] && (rtp_session->rtp_bugs & RTP_BUG_GEN_ONE_GEN_ALL)) { @@ -7012,11 +7033,16 @@ SWITCH_DECLARE(int) switch_rtp_write_frame(switch_rtp_t *rtp_session, switch_fra send_msg = frame->packet; len = frame->packetlen; ts = 0; - // Trying this based on http://jira.freeswitch.org/browse/MODSOFIA-90 - //if (frame->codec && frame->codec->agreed_pt == frame->payload) { send_msg->header.pt = payload; - //} + + if (switch_test_flag(frame, SFF_RAW_RTP_PARSE_FRAME)) { + send_msg->header.version = 2; + send_msg->header.m = frame->m; + send_msg->header.ts = htonl(frame->timestamp); + send_msg->header.ssrc = htonl(rtp_session->ssrc); + } + } else { data = frame->data; len = frame->datalen;