retronetworking
/
freeswitch
13
0
Fork 0
Code Pull Requests Releases Activity

FS-7500: major refactoring pass. Push concepts from mod_vlc as deep as possible and flesh out api to use everywhere else. Round 2 will be to convert the bridge and other places using the same code

This commit is contained in:
Anthony Minessale 2014-11-13 20:52:57 -06:00 committed by Michael Jerris
parent 765fff3d75
commit 365a5dd820
11 changed files with 175 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/include/private/switch_core_pvt.h
View File

@ -190,6 +190,10 @@ struct switch_core_session {
uint32_t decoder_errors;
switch_core_video_thread_callback_func_t *_video_thread_callback;
void *_video_thread_user_data;
//switch_time_t last_video_write_time;
switch_image_write_callback_t image_write_callback;
void *image_write_callback_user_data;
};
struct switch_media_bug {

20
src/include/switch_core.h
View File

@ -1275,6 +1275,26 @@ SWITCH_DECLARE(switch_status_t) switch_core_session_read_frame(_In_ switch_core_
SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(_In_ switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags,
int stream_id);
/*!
\brief Write a video image to a session using a video frame
\param session the session to write to
\param frame a pointer to a frame to use for write with proper codec
\param img the image structure with the image data
\param the size for packetization
\param flag pointer to frame flags to pass in / out
\return SWITCH_STATUS_SUCCESS a if the image was written
*/
SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_image(switch_core_session_t *session, switch_frame_t *frame,
switch_image_t *img, switch_size_t size, uint32_t *flag);
/*!
\brief set a callback to be called after each frame of an image is written
\param session the session to write to
\param callback the function to call
\param user_data the user data pointer to pass as an arguement to the callback
\return void
*/
SWITCH_DECLARE(void) switch_core_session_set_image_write_callback(switch_core_session_t *session, switch_image_write_callback_t callback, void *user_data);
/*!
\brief Write a video frame to a session
\param session the session to write to

2
src/include/switch_core_media.h
View File

@ -305,6 +305,8 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess
switch_codec_control_type_t *rtype,
void **ret_data);
SWITCH_DECLARE(switch_timer_t *) switch_core_media_get_timer(switch_core_session_t *session, switch_media_type_t mtype);
SWITCH_END_EXTERN_C
#endif
/* For Emacs:

1
src/include/switch_module_interfaces.h
View File

@ -217,6 +217,7 @@ struct switch_timer {
switch_size_t diff;
switch_time_t start;
uint64_t tick;
};
typedef enum {

6
src/include/switch_rtp.h
View File

@ -268,6 +268,9 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_activate_ice(switch_rtp_t *rtp_sessio
*/
SWITCH_DECLARE(switch_status_t) switch_rtp_activate_rtcp(switch_rtp_t *rtp_session, int send_rate, switch_port_t remote_port, switch_bool_t mux);
SWITCH_DECLARE(switch_timer_t *) switch_rtp_get_media_timer(switch_rtp_t *rtp_session);
/*!
\brief Acvite a jitter buffer on an RTP session
\param rtp_session the rtp session
@ -285,6 +288,9 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_deactivate_jitter_buffer(switch_rtp_t
SWITCH_DECLARE(switch_status_t) switch_rtp_pause_jitter_buffer(switch_rtp_t *rtp_session, switch_bool_t pause);
SWITCH_DECLARE(stfu_instance_t *) switch_rtp_get_jitter_buffer(switch_rtp_t *rtp_session);
/*!
\brief Set an RTP Flag
\param rtp_session the RTP session

6
src/include/switch_types.h
View File

@ -226,6 +226,7 @@ SWITCH_BEGIN_EXTERN_C
#define SWITCH_DTMF_LOG_LEN 1000
#define SWITCH_MAX_TRANS 2000
#define SWITCH_CORE_SESSION_MAX_PRIVATES 2
#define SWITCH_DEFAULT_VIDEO_SIZE 1500
/* Jitter */
#define JITTER_VARIANCE_THRESHOLD 400.0
@ -1494,7 +1495,8 @@ typedef enum {
SFF_NOT_AUDIO = (1 << 9),
SFF_RTCP = (1 << 10),
SFF_MARKER = (1 << 11),
SFF_WAIT_KEY_FRAME = (1 << 12)
SFF_WAIT_KEY_FRAME = (1 << 12),
SFF_RAW_RTP_PARSE_FRAME = (1 << 13)
} switch_frame_flag_enum_t;
typedef uint32_t switch_frame_flag_t;
@ -2196,6 +2198,8 @@ typedef switch_status_t (*switch_core_codec_control_func_t) (switch_codec_t *cod
void **ret_data);
typedef switch_status_t (*switch_image_write_callback_t) (switch_core_session_t *session, switch_frame_t *frame, switch_image_t *img, void *user_data);
typedef switch_status_t (*switch_core_codec_init_func_t) (switch_codec_t *, switch_codec_flag_t, const switch_codec_settings_t *codec_settings);
typedef switch_status_t (*switch_core_codec_fmtp_parse_func_t) (const char *fmtp, switch_codec_fmtp_t *codec_fmtp);
typedef switch_status_t (*switch_core_codec_destroy_func_t) (switch_codec_t *);

138
src/mod/formats/mod_vlc/mod_vlc.c
View File

@ -107,7 +107,6 @@ struct vlc_video_context {
uint8_t video_packet[1500 + 12];
void *raw_yuyv_data;
switch_image_t *img;
switch_time_t last_video_ts;
switch_payload_t pt;
uint32_t seq;
int width;
@ -241,68 +240,19 @@ static void vlc_video_unlock_dummy_callback(void *data, void *id, void *const *p
static void vlc_video_unlock_callback(void *data, void *id, void *const *p_pixels)
{
vlc_video_context_t *context = (vlc_video_context_t *)data;
vlc_video_context_t *context = (vlc_video_context_t *) data;
switch_frame_t *frame = context->vid_frame;
uint32_t flag = 0;
uint32_t encoded_data_len = 1500;
switch_time_t now = (switch_time_t)(switch_micro_time_now() / 1000);
switch_codec_t *codec = switch_core_session_get_video_write_codec(context->session);
long delta;
switch_assert(id == NULL); /* picture identifier, not needed here */
switch_assert(codec);
if (now - context->last_video_ts < 60) goto end;
if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0);
if (!context->img) goto end;
switch_assert(context->img);
yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height);
delta = now - context->last_video_ts;
switch_core_session_write_video_image(context->session, frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, NULL);
if (delta > 0) {
frame->timestamp += delta * 90;
context->last_video_ts = now;
}
//printf("WTF VLC d_w=%d d_h=%d w=%d h=%d\n", context->img->d_w, context->img->d_h, context->img->w, context->img->h);
//context->img->d_w = context->img->w;
//context->img->d_h = context->img->h;
switch_core_codec_encode_video(codec, context->img, frame->data, &encoded_data_len, &flag);
while(encoded_data_len) {
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%ld\n", codec->implementation->iananame, encoded_data_len, flag, context->last_video_ts);
frame->datalen = encoded_data_len;
frame->packetlen = frame->datalen + 12;
frame->m = flag & SFF_MARKER ? 1 : 0;
if (1) {
/* set correct mark and ts */
switch_rtp_hdr_t *rtp = (switch_rtp_hdr_t *)frame->packet;
memset(rtp, 0, 12);
rtp->version = 2;
rtp->m = frame->m;
rtp->ts = htonl(frame->timestamp);
rtp->ssrc = (uint32_t) ((intptr_t) rtp + (uint32_t) switch_epoch_time_now(NULL));
switch_set_flag(frame, SFF_RAW_RTP);
}
switch_set_flag(frame, SFF_RAW_RTP);
// switch_set_flag(frame, SFF_PROXY_PACKET);
switch_core_session_write_video_frame(context->session, frame, SWITCH_IO_FLAG_NONE, 0);
encoded_data_len = 1500;
switch_core_codec_encode_video(codec, NULL, frame->data, &encoded_data_len, &flag);
}
end:
switch_mutex_unlock(context->video_mutex);
}
@ -323,72 +273,28 @@ static void do_buffer_frame(vlc_video_context_t *context)
switch_mutex_unlock(context->video_mutex);
}
static void vlc_video_channel_unlock_callback(void *data, void *id, void *const *p_pixels)
{
vlc_video_context_t *context = (vlc_video_context_t *)data;
uint32_t flag = 0;
uint32_t encoded_data_len = 1500;
switch_codec_t *codec = switch_core_session_get_video_write_codec(context->session);
switch_frame_t *frame = context->vid_frame;
switch_time_t now = (switch_time_t)(switch_micro_time_now() / 1000);
long delta;
switch_assert(id == NULL); /* picture identifier, not needed here */
switch_assert(codec);
if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0);
if (!context->img) goto end;
switch_assert(context->img);
yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height);
encoded_data_len = 1500;
frame->packet = context->video_packet;
frame->data = context->video_packet + 12;
delta = now - context->last_video_ts;
if (delta > 0) {
frame->timestamp += delta * 90;
context->last_video_ts = now;
}
if (context->video_refresh_req > 0) {
flag |= SFF_WAIT_KEY_FRAME;
context->video_refresh_req--;
}
switch_core_codec_encode_video(codec, context->img, frame->data, &encoded_data_len, &flag);
switch_core_session_write_video_image(context->session, frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, &flag);
while(encoded_data_len) {
// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%u\n", codec->implementation->iananame, encoded_data_len, flag, context->ts);
frame->datalen = encoded_data_len;
frame->packetlen = frame->datalen + 12;
frame->m = flag & SFF_MARKER ? 1 : 0;
if (1) {
/* set correct mark and ts */
switch_rtp_hdr_t *rtp = (switch_rtp_hdr_t *)frame->packet;
memset(rtp, 0, 12);
rtp->version = 2;
rtp->m = frame->m;
rtp->ts = htonl(frame->timestamp);
rtp->ssrc = (uint32_t) ((intptr_t) rtp + (uint32_t) switch_epoch_time_now(NULL));
switch_set_flag(frame, SFF_RAW_RTP);
}
switch_set_flag(frame, SFF_RAW_RTP);
switch_set_flag(frame, SFF_PROXY_PACKET);
do_buffer_frame(context);
encoded_data_len = 1500;
switch_core_codec_encode_video(codec, NULL, frame->data, &encoded_data_len, &flag);
}
end:
switch_mutex_unlock(context->video_mutex);
}
@ -720,6 +626,7 @@ SWITCH_STANDARD_APP(play_video_function)
switch_size_t audio_datalen;
switch_channel_clear_flag(channel, CF_VIDEO_ECHO);
context = switch_core_session_alloc(session, sizeof(vlc_video_context_t));
@ -974,6 +881,7 @@ static switch_status_t vlc_write_frame(switch_core_session_t *session, switch_fr
static switch_status_t vlc_read_video_frame(switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags, int stream_id);
static switch_status_t vlc_receive_message(switch_core_session_t *session, switch_core_session_message_t *msg);
static switch_status_t vlc_kill_channel(switch_core_session_t *session, int sig);
static switch_status_t vlc_state_change(switch_core_session_t *session);
typedef struct {
switch_core_session_t *session;
@ -1014,7 +922,7 @@ switch_io_routines_t vlc_io_routines = {
/*send_dtmf*/ NULL,
/*receive_message*/ vlc_receive_message,
/*receive_event*/ NULL,
/*state_change*/ NULL,
/*state_change*/ vlc_state_change,
/*read_video_frame*/ vlc_read_video_frame,
/*write_video_frame*/ NULL,
/*state_run*/ NULL
@ -1118,12 +1026,25 @@ fail:
return status;
}
static switch_status_t vlc_channel_img_callback(switch_core_session_t *session, switch_frame_t *frame, switch_image_t *img, void *user_data)
{
vlc_video_context_t *context = (vlc_video_context_t *) user_data;
do_buffer_frame(context);
return SWITCH_STATUS_SUCCESS;
}
static switch_status_t channel_on_init(switch_core_session_t *session)
{
switch_channel_t *channel = switch_core_session_get_channel(session);
vlc_private_t *tech_pvt = switch_core_session_get_private(session);
switch_channel_set_state(channel, CS_CONSUME_MEDIA);
switch_core_session_set_image_write_callback(session, vlc_channel_img_callback, tech_pvt->context);
return SWITCH_STATUS_SUCCESS;
}
@ -1488,6 +1409,19 @@ static switch_status_t vlc_receive_message(switch_core_session_t *session, switc
return SWITCH_STATUS_SUCCESS;
}
static switch_status_t vlc_state_change(switch_core_session_t *session)
{
switch_channel_t *channel = switch_core_session_get_channel(session);
switch_channel_state_t state = switch_channel_get_state(channel);
if (state == CS_HANGUP || state == CS_ROUTING) {
switch_core_session_video_reset(session);
}
return SWITCH_STATUS_SUCCESS;
}
static switch_status_t vlc_kill_channel(switch_core_session_t *session, int sig)
{
vlc_private_t *tech_pvt = switch_core_session_get_private(session);

47
src/switch_core_io.c
View File

@ -36,6 +36,53 @@
#include <switch.h>
#include "private/switch_core_pvt.h"
SWITCH_DECLARE(void) switch_core_session_set_image_write_callback(switch_core_session_t *session, switch_image_write_callback_t callback, void *user_data)
{
session->image_write_callback = callback;
session->image_write_callback_user_data = user_data;
}
SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_image(switch_core_session_t *session, switch_frame_t *frame,
switch_image_t *img, switch_size_t size, uint32_t *flag)
{
uint32_t encoded_data_len = size, lflag = 0, *flagp = flag;
switch_codec_t *codec = switch_core_session_get_video_write_codec(session);
switch_timer_t *timer;
switch_assert(session);
if (!flag) {
flagp = &lflag;
}
timer = switch_core_media_get_timer(session, SWITCH_MEDIA_TYPE_VIDEO);
switch_assert(timer);
switch_core_codec_encode_video(codec, img, frame->data, &encoded_data_len, flagp);
while(encoded_data_len) {
frame->datalen = encoded_data_len;
frame->packetlen = frame->datalen + 12;
frame->m = (*flagp & SFF_MARKER) ? 1 : 0;
frame->timestamp = timer->samplecount;
switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME);
switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
if (session->image_write_callback) {
session->image_write_callback(session, frame, img, session->image_write_callback_user_data);
}
encoded_data_len = size;
switch_core_codec_encode_video(codec, NULL, frame->data, &encoded_data_len, flagp);
}
return SWITCH_STATUS_SUCCESS;
}
SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags,
int stream_id)
{

23
src/switch_core_media.c
View File

@ -7211,6 +7211,8 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess
nack++;
}
nack = v_engine->nack = pli = v_engine->pli = 0;
if (vp8) {
if (v_engine->fir || fir) {
@ -9445,6 +9447,25 @@ SWITCH_DECLARE(char *) switch_core_media_process_sdp_filter(const char *sdp, con
}
SWITCH_DECLARE(switch_timer_t *) switch_core_media_get_timer(switch_core_session_t *session, switch_media_type_t mtype)
{
switch_rtp_engine_t *engine = NULL;
switch_media_handle_t *smh = NULL;
switch_assert(session);
if (!(smh = session->media_handle)) {
return NULL;
}
if (!(engine = &smh->engines[mtype])) {
return NULL;
}
return switch_rtp_get_media_timer(engine->rtp_session);
}
SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_session_t *session,
switch_media_type_t mtype,
switch_io_type_t iotype,
@ -9481,8 +9502,6 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess
return SWITCH_STATUS_FALSE;
}
/* For Emacs:
* Local Variables:
* mode:c

2
src/switch_core_session.c
View File

@ -2677,6 +2677,8 @@ SWITCH_DECLARE(void) switch_core_session_video_reset(switch_core_session_t *sess
switch_channel_set_flag(session->channel, CF_VIDEO_ECHO);
switch_channel_clear_flag(session->channel, CF_VIDEO_PASSIVE);
switch_core_session_refresh_video(session);
session->image_write_callback = NULL;
session->image_write_callback_user_data = NULL;
}
SWITCH_DECLARE(switch_status_t) switch_core_session_execute_application_get_flags(switch_core_session_t *session, const char *app,

36
src/switch_rtp.c
View File

@ -3575,7 +3575,14 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_create(switch_rtp_t **new_rtp_session
switch_rtp_clear_flag(rtp_session, SWITCH_RTP_FLAG_USE_TIMER);
}
} else {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Not using a timer\n");
if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) {
if (switch_core_timer_init(&rtp_session->timer, "soft", 1, 90, pool) == SWITCH_STATUS_SUCCESS) {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Starting video timer.\n");
}
} else {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Not using a timer\n");
}
switch_rtp_clear_flag(rtp_session, SWITCH_RTP_FLAG_USE_TIMER);
switch_rtp_clear_flag(rtp_session, SWITCH_RTP_FLAG_NOBLOCK);
}
@ -3781,6 +3788,19 @@ static void jb_callback(stfu_instance_t *i, void *udata)
}
SWITCH_DECLARE(switch_timer_t *) switch_rtp_get_media_timer(switch_rtp_t *rtp_session)
{
if (rtp_session->timer.timer_interface) {
if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) {
switch_core_timer_sync(&rtp_session->timer);
}
return &rtp_session->timer;
}
return NULL;
}
SWITCH_DECLARE(stfu_instance_t *) switch_rtp_get_jitter_buffer(switch_rtp_t *rtp_session)
{
if (!switch_rtp_ready(rtp_session) || !rtp_session->jb) {
@ -6953,7 +6973,8 @@ SWITCH_DECLARE(int) switch_rtp_write_frame(switch_rtp_t *rtp_session, switch_fra
}
#endif
fwd = (rtp_session->flags[SWITCH_RTP_FLAG_RAW_WRITE] && switch_test_flag(frame, SFF_RAW_RTP)) ? 1 : 0;
fwd = (rtp_session->flags[SWITCH_RTP_FLAG_RAW_WRITE] &&
(switch_test_flag(frame, SFF_RAW_RTP) || switch_test_flag(frame, SFF_RAW_RTP_PARSE_FRAME))) ? 1 : 0;
if (!fwd && !rtp_session->sending_dtmf && !rtp_session->queue_delay &&
rtp_session->flags[SWITCH_RTP_FLAG_RAW_WRITE] && (rtp_session->rtp_bugs & RTP_BUG_GEN_ONE_GEN_ALL)) {
@ -7012,11 +7033,16 @@ SWITCH_DECLARE(int) switch_rtp_write_frame(switch_rtp_t *rtp_session, switch_fra
send_msg = frame->packet;
len = frame->packetlen;
ts = 0;
// Trying this based on http://jira.freeswitch.org/browse/MODSOFIA-90
//if (frame->codec && frame->codec->agreed_pt == frame->payload) {
send_msg->header.pt = payload;
//}
if (switch_test_flag(frame, SFF_RAW_RTP_PARSE_FRAME)) {
send_msg->header.version = 2;
send_msg->header.m = frame->m;
send_msg->header.ts = htonl(frame->timestamp);
send_msg->header.ssrc = htonl(rtp_session->ssrc);
}
} else {
data = frame->data;
len = frame->datalen;