Fix media update issues with sip2sip.info (loosing AVPF profile info which disable RTCP-FB)

This commit is contained in:
bossiel 2015-06-11 06:53:46 +00:00
parent e150f2eab6
commit 2834ba3d41
11 changed files with 521 additions and 455 deletions

View File

@ -932,11 +932,11 @@ int tdav_codec_h263_open_encoder(tdav_codec_h263_t* self)
self->encoder.context->mb_qmax = self->encoder.context->qmax;
#endif
self->encoder.context->mb_decision = FF_MB_DECISION_RD;
max_bw_kpbs = TSK_CLAMP(
0,
tmedia_get_video_bandwidth_kbps_2(TMEDIA_CODEC_VIDEO(self)->out.width, TMEDIA_CODEC_VIDEO(self)->out.height, TMEDIA_CODEC_VIDEO(self)->out.fps),
self->encoder.max_bw_kpbs
);
max_bw_kpbs = TSK_CLAMP(
0,
tmedia_get_video_bandwidth_kbps_2(TMEDIA_CODEC_VIDEO(self)->out.width, TMEDIA_CODEC_VIDEO(self)->out.height, TMEDIA_CODEC_VIDEO(self)->out.fps),
self->encoder.max_bw_kpbs
);
self->encoder.context->bit_rate = (max_bw_kpbs * 1024);// bps
//self->encoder.context->rc_lookahead = 0;
self->encoder.context->rtp_payload_size = RTP_PAYLOAD_SIZE;
@ -1046,6 +1046,8 @@ int tdav_codec_h263_open_decoder(tdav_codec_h263_t* self)
TSK_DEBUG_ERROR("Failed to open [%s] codec", TMEDIA_CODEC(self)->plugin->desc);
return ret;
}
self->decoder.last_seq = 0;
return ret;
}

View File

@ -804,11 +804,14 @@ int tdav_codec_h264_open_encoder(tdav_codec_h264_t* self)
TSK_DEBUG_ERROR("Failed to open [%s] codec", TMEDIA_CODEC(self)->plugin->desc);
return ret;
}
self->encoder.frame_count = 0;
TSK_DEBUG_INFO("[H.264] bitrate=%d bps", self->encoder.context->bit_rate);
return ret;
#elif HAVE_H264_PASSTHROUGH
self->encoder.frame_count = 0;
return 0;
#endif
@ -870,6 +873,7 @@ int tdav_codec_h264_open_decoder(tdav_codec_h264_t* self)
TSK_DEBUG_ERROR("Failed to open [%s] codec", TMEDIA_CODEC(self)->plugin->desc);
return ret;
}
self->decoder.last_seq = 0;
return ret;

View File

@ -699,6 +699,8 @@ static int tdav_codec_h264_cisco_open_encoder(tdav_codec_h264_cisco_t* self)
TSK_DEBUG_ERROR("Failed to create mutex for the encoder");
goto bail;
}
self->encoder.frame_count = 0;
ret = 0;
@ -755,6 +757,7 @@ int tdav_codec_h264_cisco_open_decoder(tdav_codec_h264_cisco_t* self)
TSK_DEBUG_ERROR("Failed to initialize decoder: %ld", err);
goto bail;
}
self->decoder.last_seq = 0;
TSK_DEBUG_INFO("[OpenH264 Decoder] neg_width=%d, neg_height=%d, neg_fps=%d",
TMEDIA_CODEC_VIDEO(self)->in.width,
TMEDIA_CODEC_VIDEO(self)->in.height,

View File

@ -568,6 +568,8 @@ int tdav_codec_mp4ves_open_decoder(tdav_codec_mp4ves_t* self)
TSK_DEBUG_ERROR("Failed to open MP4V-ES decoder");
return ret;
}
self->decoder.last_seq = 0;
return ret;
}

View File

@ -95,6 +95,7 @@ static int tdav_codec_opus_open(tmedia_codec_t* self)
return -2;
}
}
opus->decoder.last_seq = 0;
// Initialize the encoder
if(!opus->encoder.inst){

View File

@ -776,6 +776,8 @@ int tdav_codec_vp8_open_encoder(tdav_codec_vp8_t* self)
TSK_DEBUG_ERROR("Failed to create mutex");
return -4;
}
self->encoder.frame_count = 0;
self->encoder.initialized = tsk_true;

View File

@ -1,23 +1,23 @@
/*
* Copyright (C) 2011-2015 Mamadou DIOP
* Copyright (C) 2011-2015 Doubango Telecom <http://www.doubango.org>
*
* This file is part of Open Source Doubango Framework.
*
* DOUBANGO is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* DOUBANGO is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with DOUBANGO.
*
*/
* Copyright (C) 2011-2015 Mamadou DIOP
* Copyright (C) 2011-2015 Doubango Telecom <http://www.doubango.org>
*
* This file is part of Open Source Doubango Framework.
*
* DOUBANGO is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* DOUBANGO is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with DOUBANGO.
*
*/
/**@file tdav_video_jb.c
* @brief Video Jitter Buffer
@ -61,162 +61,154 @@
#define TDAV_VIDEO_JB_LATENCY_MIN 2 /* Must be > 0 */
#define TDAV_VIDEO_JB_LATENCY_MAX 15 /* Default, will be updated using fps */
static int _tdav_video_jb_set_defaults(struct tdav_video_jb_s* self);
static const tdav_video_frame_t* _tdav_video_jb_get_frame(struct tdav_video_jb_s* self, uint32_t timestamp, uint8_t pt, tsk_bool_t *pt_matched);
static void* TSK_STDCALL _tdav_video_jb_decode_thread_func(void *arg);
typedef struct tdav_video_jb_s
{
TSK_DECLARE_OBJECT;
tsk_bool_t started;
int32_t fps;
int32_t fps_prob;
int32_t avg_duration;
int32_t rate; // in Khz
uint32_t last_timestamp;
int32_t conseq_frame_drop;
int32_t tail_max;
tdav_video_frames_L_t *frames;
int64_t frames_count;
tsk_size_t latency_min;
tsk_size_t latency_max;
uint32_t decode_last_timestamp;
int32_t decode_last_seq_num_with_mark; // -1 = unset
uint64_t decode_last_time;
tsk_thread_handle_t* decode_thread[1];
tsk_condwait_handle_t* decode_thread_cond;
uint16_t seq_nums[0xFF];
tdav_video_jb_cb_f callback;
const void* callback_data;
// to avoid locking use different cb_data
tdav_video_jb_cb_data_xt cb_data_rtp;
tdav_video_jb_cb_data_xt cb_data_fdd;
tdav_video_jb_cb_data_xt cb_data_any;
struct{
void* ptr;
tsk_size_t size;
} buffer;
TSK_DECLARE_SAFEOBJ;
TSK_DECLARE_OBJECT;
tsk_bool_t started;
int32_t fps;
int32_t fps_prob;
int32_t avg_duration;
int32_t rate; // in Khz
uint32_t last_timestamp;
int32_t conseq_frame_drop;
int32_t tail_max;
tdav_video_frames_L_t *frames;
int64_t frames_count;
tsk_size_t latency_min;
tsk_size_t latency_max;
uint32_t decode_last_timestamp;
int32_t decode_last_seq_num_with_mark; // -1 = unset
uint64_t decode_last_time;
tsk_thread_handle_t* decode_thread[1];
tsk_condwait_handle_t* decode_thread_cond;
uint16_t seq_nums[0xFF];
tdav_video_jb_cb_f callback;
const void* callback_data;
// to avoid locking use different cb_data
tdav_video_jb_cb_data_xt cb_data_rtp;
tdav_video_jb_cb_data_xt cb_data_fdd;
tdav_video_jb_cb_data_xt cb_data_any;
struct{
void* ptr;
tsk_size_t size;
} buffer;
TSK_DECLARE_SAFEOBJ;
}
tdav_video_jb_t;
static tsk_object_t* tdav_video_jb_ctor(tsk_object_t * self, va_list * app)
{
tdav_video_jb_t *jb = self;
if(jb){
if(!(jb->frames = tsk_list_create())){
TSK_DEBUG_ERROR("Failed to create list");
return tsk_null;
}
if(!(jb->decode_thread_cond = tsk_condwait_create())){
TSK_DEBUG_ERROR("Failed to create condition var");
return tsk_null;
}
jb->cb_data_fdd.type = tdav_video_jb_cb_data_type_fdd;
jb->cb_data_rtp.type = tdav_video_jb_cb_data_type_rtp;
jb->decode_last_seq_num_with_mark = -1;
jb->fps = TDAV_VIDEO_JB_FPS_MAX;
jb->rate = TDAV_VIDEO_JB_RATE;
jb->latency_min = TDAV_VIDEO_JB_LATENCY_MIN;
jb->latency_max = TDAV_VIDEO_JB_LATENCY_MAX;
tsk_safeobj_init(jb);
}
return self;
tdav_video_jb_t *jb = self;
if(jb){
if(!(jb->frames = tsk_list_create())){
TSK_DEBUG_ERROR("Failed to create list");
return tsk_null;
}
if(!(jb->decode_thread_cond = tsk_condwait_create())){
TSK_DEBUG_ERROR("Failed to create condition var");
return tsk_null;
}
jb->cb_data_fdd.type = tdav_video_jb_cb_data_type_fdd;
jb->cb_data_rtp.type = tdav_video_jb_cb_data_type_rtp;
tsk_safeobj_init(jb);
}
return self;
}
static tsk_object_t* tdav_video_jb_dtor(tsk_object_t * self)
{
tdav_video_jb_t *jb = self;
if(jb){
if(jb->started){
tdav_video_jb_stop(jb);
}
TSK_OBJECT_SAFE_FREE(jb->frames);
if(jb->decode_thread_cond){
tsk_condwait_destroy(&jb->decode_thread_cond);
}
TSK_SAFE_FREE(jb->buffer.ptr);
tsk_safeobj_deinit(jb);
}
return self;
}
static const tsk_object_def_t tdav_video_jb_def_s =
{
sizeof(tdav_video_jb_t),
tdav_video_jb_ctor,
tdav_video_jb_dtor,
tsk_null,
tdav_video_jb_t *jb = self;
if(jb){
if(jb->started){
tdav_video_jb_stop(jb);
}
TSK_OBJECT_SAFE_FREE(jb->frames);
if(jb->decode_thread_cond){
tsk_condwait_destroy(&jb->decode_thread_cond);
}
TSK_SAFE_FREE(jb->buffer.ptr);
tsk_safeobj_deinit(jb);
}
return self;
}
static const tsk_object_def_t tdav_video_jb_def_s =
{
sizeof(tdav_video_jb_t),
tdav_video_jb_ctor,
tdav_video_jb_dtor,
tsk_null,
};
tdav_video_jb_t* tdav_video_jb_create()
{
tdav_video_jb_t* jb;
if((jb = tsk_object_new(&tdav_video_jb_def_s))){
jb->fps = TDAV_VIDEO_JB_FPS;
jb->fps_prob = TDAV_VIDEO_JB_FPS_PROB;
jb->tail_max = TDAV_VIDEO_JB_TAIL_MAX;
}
return jb;
tdav_video_jb_t* jb;
if ((jb = tsk_object_new(&tdav_video_jb_def_s))) {
if (_tdav_video_jb_set_defaults(jb) != 0) {
TSK_OBJECT_SAFE_FREE(jb);
}
}
return jb;
}
#define tdav_video_jb_reset_fps_prob(self) {\
(self)->fps_prob = TDAV_VIDEO_JB_FPS_PROB; \
(self)->last_timestamp = 0; \
(self)->avg_duration = 0; \
(self)->fps_prob = TDAV_VIDEO_JB_FPS_PROB; \
(self)->last_timestamp = 0; \
(self)->avg_duration = 0; \
}
#define tdav_video_jb_reset_tail_min_prob(self) {\
(self)->tail_prob = TDAV_VIDEO_JB_TAIL_MIN_PROB; \
(self)->tail_min = TDAV_VIDEO_JB_TAIL_MIN_MAX; \
(self)->tail_prob = TDAV_VIDEO_JB_TAIL_MIN_PROB; \
(self)->tail_min = TDAV_VIDEO_JB_TAIL_MIN_MAX; \
}
int tdav_video_jb_set_callback(tdav_video_jb_t* self, tdav_video_jb_cb_f callback, const void* usr_data)
{
if(!self){
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
self->callback = callback;
self->cb_data_any.usr_data = usr_data;
self->cb_data_fdd.usr_data = usr_data;
self->cb_data_rtp.usr_data = usr_data;
return 0;
if(!self){
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
self->callback = callback;
self->cb_data_any.usr_data = usr_data;
self->cb_data_fdd.usr_data = usr_data;
self->cb_data_rtp.usr_data = usr_data;
return 0;
}
int tdav_video_jb_start(tdav_video_jb_t* self)
{
int ret = 0;
if(!self){
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
if(self->started){
return 0;
}
self->started = tsk_true;
if(!self->decode_thread[0]){
ret = tsk_thread_create(&self->decode_thread[0], _tdav_video_jb_decode_thread_func, self);
if(ret != 0 || !self->decode_thread[0]){
TSK_DEBUG_ERROR("Failed to create new thread");
}
ret = tsk_thread_set_priority(self->decode_thread[0], TSK_THREAD_PRIORITY_TIME_CRITICAL);
}
return ret;
int ret = 0;
if(!self){
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
if(self->started){
return 0;
}
self->started = tsk_true;
if(!self->decode_thread[0]){
ret = tsk_thread_create(&self->decode_thread[0], _tdav_video_jb_decode_thread_func, self);
if(ret != 0 || !self->decode_thread[0]){
TSK_DEBUG_ERROR("Failed to create new thread");
}
ret = tsk_thread_set_priority(self->decode_thread[0], TSK_THREAD_PRIORITY_TIME_CRITICAL);
}
return ret;
}
int tdav_video_jb_put(tdav_video_jb_t* self, trtp_rtp_packet_t* rtp_pkt)
@ -225,313 +217,347 @@ int tdav_video_jb_put(tdav_video_jb_t* self, trtp_rtp_packet_t* rtp_pkt)
self->cb_data_rtp.rtp.pkt = rtp_pkt;
self->callback(&self->cb_data_rtp);
#else
const tdav_video_frame_t* old_frame;
tsk_bool_t pt_matched = tsk_false, is_frame_late_or_dup = tsk_false, is_restarted = tsk_false;
uint16_t* seq_num;
if(!self || !rtp_pkt || !rtp_pkt->header){
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
if(!self->started){
TSK_DEBUG_INFO("Video jitter buffer not started");
return 0;
}
seq_num = &self->seq_nums[rtp_pkt->header->payload_type];
tsk_safeobj_lock(self);
//TSK_DEBUG_INFO("receive seqnum=%u", rtp_pkt->header->seq_num);
if(self->decode_last_timestamp && (self->decode_last_timestamp > rtp_pkt->header->timestamp)){
if((self->decode_last_timestamp - rtp_pkt->header->timestamp) < TDAV_VIDEO_JB_MAX_DROPOUT){
TSK_DEBUG_INFO("--------Frame already Decoded [seqnum=%u]------------", rtp_pkt->header->seq_num);
tsk_safeobj_unlock(self);
return 0;
}
}
old_frame = _tdav_video_jb_get_frame(self, rtp_pkt->header->timestamp, rtp_pkt->header->payload_type, &pt_matched);
if((*seq_num && *seq_num != 0xFFFF) && (*seq_num + 1) != rtp_pkt->header->seq_num){
int32_t diff = ((int32_t)rtp_pkt->header->seq_num - (int32_t)*seq_num);
tsk_bool_t is_frame_loss = (diff > 0);
is_restarted = (TSK_ABS(diff) > TDAV_VIDEO_JB_MAX_DROPOUT);
is_frame_late_or_dup = !is_frame_loss;
tdav_video_jb_reset_fps_prob(self);
TSK_DEBUG_INFO("Packet %s (from JB) [%hu - %hu]", is_frame_loss ? "loss" : "late/duplicated/nack", *seq_num, rtp_pkt->header->seq_num);
if(is_frame_loss && !is_restarted){
if(self->callback){
self->cb_data_any.type = tdav_video_jb_cb_data_type_fl;
self->cb_data_any.ssrc = rtp_pkt->header->ssrc;
self->cb_data_any.fl.seq_num = (*seq_num + 1);
self->cb_data_any.fl.count = diff - 1;
self->callback(&self->cb_data_any);
}
}
}
if(!old_frame){
tdav_video_frame_t* new_frame;
if(pt_matched){
// if we have a frame with the same payload type but without this timestamp this means that we moved to a new frame
// this happens if the frame is waiting to be decoded or the marker is lost
}
if((new_frame = tdav_video_frame_create(rtp_pkt))){
// compute avg frame duration
if(self->last_timestamp && self->last_timestamp < rtp_pkt->header->timestamp){
uint32_t duration = (rtp_pkt->header->timestamp - self->last_timestamp)/self->rate;
self->avg_duration = self->avg_duration ? ((self->avg_duration + duration) >> 1) : duration;
--self->fps_prob;
}
self->last_timestamp = rtp_pkt->header->timestamp;
tsk_list_lock(self->frames);
if(self->frames_count >= self->tail_max){
if(++self->conseq_frame_drop >= self->tail_max){
TSK_DEBUG_ERROR("Too many frames dropped and fps=%d", self->fps);
tsk_list_clear_items(self->frames);
self->conseq_frame_drop = 0;
self->frames_count = 1;
if(self->callback){
self->cb_data_any.type = tdav_video_jb_cb_data_type_tmfr;
self->cb_data_any.ssrc = rtp_pkt->header->ssrc;
self->callback(&self->cb_data_any);
}
}
else{
TSK_DEBUG_INFO("Dropping video frame because frames_count(%lld)>=tail_max(%d)", self->frames_count, self->tail_max);
tsk_list_remove_first_item(self->frames);
}
tdav_video_jb_reset_fps_prob(self);
}
else{
++self->frames_count;
}
tsk_list_push_ascending_data(self->frames, (void**)&new_frame);
tsk_list_unlock(self->frames);
}
if(self->fps_prob <= 0 && self->avg_duration){
// compute FPS using timestamp values
int32_t fps_new = (1000 / self->avg_duration);
int32_t fps_old = self->fps;
self->fps = TSK_CLAMP(TDAV_VIDEO_JB_FPS_MIN, fps_new, TDAV_VIDEO_JB_FPS_MAX);
self->tail_max = (self->fps << TDAV_VIDEO_JB_TAIL_MAX_LOG2); // maximum delay = 2 seconds
self->latency_max = self->fps; // maximum = 1 second
TSK_DEBUG_INFO("According to rtp-timestamps ...FPS = %d (clipped to %d) tail_max=%d, latency_max=%u", fps_new, self->fps, self->tail_max, (unsigned)self->latency_max);
tdav_video_jb_reset_fps_prob(self);
if(self->callback && (fps_old != self->fps)){
self->cb_data_any.type = tdav_video_jb_cb_data_type_fps_changed;
self->cb_data_any.ssrc = rtp_pkt->header->ssrc;
self->cb_data_any.fps.new = self->fps; // clipped value
self->cb_data_any.fps.old = fps_old;
self->callback(&self->cb_data_any);
}
}
}
else{
tdav_video_frame_put((tdav_video_frame_t*)old_frame, rtp_pkt);
}
tsk_safeobj_unlock(self);
if(!is_frame_late_or_dup || is_restarted){
*seq_num = rtp_pkt->header->seq_num;
}
const tdav_video_frame_t* old_frame;
tsk_bool_t pt_matched = tsk_false, is_frame_late_or_dup = tsk_false, is_restarted = tsk_false;
uint16_t* seq_num;
if(!self || !rtp_pkt || !rtp_pkt->header){
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
if(!self->started){
TSK_DEBUG_INFO("Video jitter buffer not started");
return 0;
}
seq_num = &self->seq_nums[rtp_pkt->header->payload_type];
tsk_safeobj_lock(self);
//TSK_DEBUG_INFO("receive seqnum=%u", rtp_pkt->header->seq_num);
if(self->decode_last_timestamp && (self->decode_last_timestamp > rtp_pkt->header->timestamp)){
if((self->decode_last_timestamp - rtp_pkt->header->timestamp) < TDAV_VIDEO_JB_MAX_DROPOUT){
TSK_DEBUG_INFO("--------Frame already Decoded [seqnum=%u]------------", rtp_pkt->header->seq_num);
tsk_safeobj_unlock(self);
return 0;
}
}
old_frame = _tdav_video_jb_get_frame(self, rtp_pkt->header->timestamp, rtp_pkt->header->payload_type, &pt_matched);
if((*seq_num && *seq_num != 0xFFFF) && (*seq_num + 1) != rtp_pkt->header->seq_num){
int32_t diff = ((int32_t)rtp_pkt->header->seq_num - (int32_t)*seq_num);
tsk_bool_t is_frame_loss = (diff > 0);
is_restarted = (TSK_ABS(diff) > TDAV_VIDEO_JB_MAX_DROPOUT);
is_frame_late_or_dup = !is_frame_loss;
tdav_video_jb_reset_fps_prob(self);
TSK_DEBUG_INFO("Packet %s (from JB) [%hu - %hu]", is_frame_loss ? "loss" : "late/duplicated/nack", *seq_num, rtp_pkt->header->seq_num);
if(is_frame_loss && !is_restarted){
if(self->callback){
self->cb_data_any.type = tdav_video_jb_cb_data_type_fl;
self->cb_data_any.ssrc = rtp_pkt->header->ssrc;
self->cb_data_any.fl.seq_num = (*seq_num + 1);
self->cb_data_any.fl.count = diff - 1;
self->callback(&self->cb_data_any);
}
}
}
if(!old_frame){
tdav_video_frame_t* new_frame;
if(pt_matched){
// if we have a frame with the same payload type but without this timestamp this means that we moved to a new frame
// this happens if the frame is waiting to be decoded or the marker is lost
}
if((new_frame = tdav_video_frame_create(rtp_pkt))){
// compute avg frame duration
if(self->last_timestamp && self->last_timestamp < rtp_pkt->header->timestamp){
uint32_t duration = (rtp_pkt->header->timestamp - self->last_timestamp)/self->rate;
self->avg_duration = self->avg_duration ? ((self->avg_duration + duration) >> 1) : duration;
--self->fps_prob;
}
self->last_timestamp = rtp_pkt->header->timestamp;
tsk_list_lock(self->frames);
if(self->frames_count >= self->tail_max){
if(++self->conseq_frame_drop >= self->tail_max){
TSK_DEBUG_ERROR("Too many frames dropped and fps=%d", self->fps);
tsk_list_clear_items(self->frames);
self->conseq_frame_drop = 0;
self->frames_count = 1;
if(self->callback){
self->cb_data_any.type = tdav_video_jb_cb_data_type_tmfr;
self->cb_data_any.ssrc = rtp_pkt->header->ssrc;
self->callback(&self->cb_data_any);
}
}
else{
TSK_DEBUG_INFO("Dropping video frame because frames_count(%lld)>=tail_max(%d)", self->frames_count, self->tail_max);
tsk_list_remove_first_item(self->frames);
}
tdav_video_jb_reset_fps_prob(self);
}
else{
++self->frames_count;
}
tsk_list_push_ascending_data(self->frames, (void**)&new_frame);
tsk_list_unlock(self->frames);
}
if(self->fps_prob <= 0 && self->avg_duration){
// compute FPS using timestamp values
int32_t fps_new = (1000 / self->avg_duration);
int32_t fps_old = self->fps;
self->fps = TSK_CLAMP(TDAV_VIDEO_JB_FPS_MIN, fps_new, TDAV_VIDEO_JB_FPS_MAX);
self->tail_max = (self->fps << TDAV_VIDEO_JB_TAIL_MAX_LOG2); // maximum delay = 2 seconds
self->latency_max = self->fps; // maximum = 1 second
TSK_DEBUG_INFO("According to rtp-timestamps ...FPS = %d (clipped to %d) tail_max=%d, latency_max=%u", fps_new, self->fps, self->tail_max, (unsigned)self->latency_max);
tdav_video_jb_reset_fps_prob(self);
if(self->callback && (fps_old != self->fps)){
self->cb_data_any.type = tdav_video_jb_cb_data_type_fps_changed;
self->cb_data_any.ssrc = rtp_pkt->header->ssrc;
self->cb_data_any.fps.new = self->fps; // clipped value
self->cb_data_any.fps.old = fps_old;
self->callback(&self->cb_data_any);
}
}
}
else{
tdav_video_frame_put((tdav_video_frame_t*)old_frame, rtp_pkt);
}
tsk_safeobj_unlock(self);
if(!is_frame_late_or_dup || is_restarted){
*seq_num = rtp_pkt->header->seq_num;
}
#endif
return 0;
return 0;
}
int tdav_video_jb_stop(tdav_video_jb_t* self)
{
int ret;
if(!self){
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
if(!self->started){
return 0;
}
int ret;
if(!self){
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
if(!self->started){
return 0;
}
TSK_DEBUG_INFO("tdav_video_jb_stop()");
self->started = tsk_false;
ret = tsk_condwait_broadcast(self->decode_thread_cond);
if (self->decode_thread[0]) {
ret = tsk_thread_join(&self->decode_thread[0]);
}
// clear pending frames
tsk_list_lock(self->frames);
tsk_list_clear_items(self->frames);
self->frames_count = 0;
tsk_list_unlock(self->frames);
// reset default values to make sure next start will be called with right defaults
// do not call this function in start to avoid overriding values defined between prepare() and start()
_tdav_video_jb_set_defaults(self);
return ret;
}
TSK_DEBUG_INFO("tdav_video_jb_stop()");
self->started = tsk_false;
ret = tsk_condwait_broadcast(self->decode_thread_cond);
if(self->decode_thread[0]){
ret = tsk_thread_join(&self->decode_thread[0]);
}
return ret;
static int _tdav_video_jb_set_defaults(struct tdav_video_jb_s* self)
{
if (!self) {
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
self->fps = TDAV_VIDEO_JB_FPS;
self->fps_prob = TDAV_VIDEO_JB_FPS_PROB;
self->tail_max = TDAV_VIDEO_JB_TAIL_MAX;
self->avg_duration = 0;
self->rate = TDAV_VIDEO_JB_RATE;
self->conseq_frame_drop = 0;
self->frames_count = 0;
self->decode_last_timestamp = 0;
self->decode_last_seq_num_with_mark = -1;
self->decode_last_time = 0;
self->latency_min = TDAV_VIDEO_JB_LATENCY_MIN;
self->latency_max = TDAV_VIDEO_JB_LATENCY_MAX;
return 0;
}
static const tdav_video_frame_t* _tdav_video_jb_get_frame(tdav_video_jb_t* self, uint32_t timestamp, uint8_t pt, tsk_bool_t *pt_matched)
{
const tdav_video_frame_t* ret = tsk_null;
const tsk_list_item_t *item;
*pt_matched =tsk_false;
tsk_list_lock(self->frames);
tsk_list_foreach(item, self->frames){
if(TDAV_VIDEO_FRAME(item->data)->payload_type == pt){
if(!(*pt_matched)) *pt_matched = tsk_true;
if(TDAV_VIDEO_FRAME(item->data)->timestamp == timestamp){
ret = item->data;
break;
}
}
}
tsk_list_unlock(self->frames);
return ret;
const tdav_video_frame_t* ret = tsk_null;
const tsk_list_item_t *item;
*pt_matched =tsk_false;
tsk_list_lock(self->frames);
tsk_list_foreach(item, self->frames){
if(TDAV_VIDEO_FRAME(item->data)->payload_type == pt){
if(!(*pt_matched)) *pt_matched = tsk_true;
if(TDAV_VIDEO_FRAME(item->data)->timestamp == timestamp){
ret = item->data;
break;
}
}
}
tsk_list_unlock(self->frames);
return ret;
}
static void* TSK_STDCALL _tdav_video_jb_decode_thread_func(void *arg)
{
tdav_video_jb_t* jb = (tdav_video_jb_t*)arg;
//uint64_t delay;
int32_t missing_seq_num_start = 0, prev_missing_seq_num_start = 0;
int32_t missing_seq_num_count = 0, prev_lasted_missing_seq_num_count = 0;
const tdav_video_frame_t* frame;
tsk_list_item_t* item;
uint64_t next_decode_duration = 0, now, _now;
//uint64_t x_decode_duration = (1000 / jb->fps); // expected
//uint64_t x_decode_time = tsk_time_now();//expected
tsk_bool_t postpone, cleaning_delay = tsk_false;
static const uint64_t __toomuch_delay_to_be_valid = 10000; // guard against systems with buggy "tsk_time_now()" -Won't say Windows ...but :)-
jb->decode_last_seq_num_with_mark = -1; // -1 -> unset
jb->decode_last_time = tsk_time_now();
(void)(now);
//(void)(delay);
TSK_DEBUG_INFO("Video jitter buffer thread - ENTER");
while(jb->started){
now = tsk_time_now();
if (next_decode_duration > 0) {
tsk_condwait_timedwait(jb->decode_thread_cond, next_decode_duration);
}
if(!jb->started){
break;
}
// TSK_DEBUG_INFO("Frames count = %d", jb->frames_count);
if(jb->frames_count >= (int64_t)jb->latency_min){
item = tsk_null;
postpone = tsk_false;
tdav_video_jb_t* jb = (tdav_video_jb_t*)arg;
//uint64_t delay;
int32_t missing_seq_num_start = 0, prev_missing_seq_num_start = 0;
int32_t missing_seq_num_count = 0, prev_lasted_missing_seq_num_count = 0;
const tdav_video_frame_t* frame;
tsk_list_item_t* item;
uint64_t next_decode_duration = 0, now, _now;
//uint64_t x_decode_duration = (1000 / jb->fps); // expected
//uint64_t x_decode_time = tsk_time_now();//expected
tsk_bool_t postpone, cleaning_delay = tsk_false;
static const uint64_t __toomuch_delay_to_be_valid = 10000; // guard against systems with buggy "tsk_time_now()" -Won't say Windows ...but :)-
jb->decode_last_seq_num_with_mark = -1; // -1 -> unset
jb->decode_last_time = tsk_time_now();
(void)(now);
//(void)(delay);
TSK_DEBUG_INFO("Video jitter buffer thread - ENTER");
while(jb->started){
now = tsk_time_now();
if (next_decode_duration > 0) {
tsk_condwait_timedwait(jb->decode_thread_cond, next_decode_duration);
}
if(!jb->started){
break;
}
// TSK_DEBUG_INFO("Frames count = %d", jb->frames_count);
if(jb->frames_count >= (int64_t)jb->latency_min){
item = tsk_null;
postpone = tsk_false;
tsk_safeobj_lock(jb); // against get_frame()
tsk_list_lock(jb->frames); // against put()
// is it still acceptable to wait for missing packets?
if (jb->frames_count < (int64_t)jb->latency_max){
frame = (const tdav_video_frame_t*)jb->frames->head->data;
if(!tdav_video_frame_is_complete(frame, jb->decode_last_seq_num_with_mark, &missing_seq_num_start, &missing_seq_num_count)){
TSK_DEBUG_INFO("Time to decode frame...but some RTP packets are missing (missing_seq_num_start=%d, missing_seq_num_count=%d, last_seq_num_with_mark=%d). Postpone :(", missing_seq_num_start, missing_seq_num_count, jb->decode_last_seq_num_with_mark);
// signal to the session that a sequence number is missing (will send a NACK)
// the missing seqnum has been already requested in jb_put() and here we request it again only ONE time
if(jb->callback){
if(prev_missing_seq_num_start != missing_seq_num_start || prev_lasted_missing_seq_num_count != missing_seq_num_count){ // guard to request it only once
jb->cb_data_any.type = tdav_video_jb_cb_data_type_fl;
jb->cb_data_any.ssrc = frame->ssrc;
jb->cb_data_any.fl.seq_num = prev_missing_seq_num_start = missing_seq_num_start;
jb->cb_data_any.fl.count = prev_lasted_missing_seq_num_count = missing_seq_num_count;
jb->callback(&jb->cb_data_any);
}
postpone = tsk_true;
}
}
}
else{
TSK_DEBUG_INFO("frames_count(%lld)>=latency_max(%u)...decoding video frame even if pkts are missing :(", jb->frames_count, (unsigned)jb->latency_max);
jb->decode_last_seq_num_with_mark = -1; // unset()
}
if(!postpone){
item = tsk_list_pop_first_item(jb->frames);
--jb->frames_count;
}
tsk_list_unlock(jb->frames);
tsk_safeobj_unlock(jb);
tsk_safeobj_lock(jb); // against get_frame()
tsk_list_lock(jb->frames); // against put()
if(item){
jb->decode_last_timestamp = ((const tdav_video_frame_t*)item->data)->timestamp;
if(jb->callback){
trtp_rtp_packet_t* pkt;
const tsk_list_item_t* _item = item; // save memory address as "tsk_list_foreach() will change it for each loop"
int32_t last_seq_num = -1; // guard against duplicated packets
frame = _item->data;
tsk_list_foreach(_item, frame->pkts){
if(!(pkt = _item->data) || !pkt->payload.size || !pkt->header || pkt->header->seq_num == last_seq_num || !jb->started){
TSK_DEBUG_ERROR("Skipping invalid rtp packet (do not decode!)");
continue;
}
jb->cb_data_rtp.rtp.pkt = pkt;
jb->callback(&jb->cb_data_rtp);
if(pkt->header->marker){
jb->decode_last_seq_num_with_mark = pkt->header->seq_num;
}
}
}
TSK_OBJECT_SAFE_FREE(item);
}
}
// is it still acceptable to wait for missing packets?
if (jb->frames_count < (int64_t)jb->latency_max){
frame = (const tdav_video_frame_t*)jb->frames->head->data;
if(!tdav_video_frame_is_complete(frame, jb->decode_last_seq_num_with_mark, &missing_seq_num_start, &missing_seq_num_count)){
TSK_DEBUG_INFO("Time to decode frame...but some RTP packets are missing (missing_seq_num_start=%d, missing_seq_num_count=%d, last_seq_num_with_mark=%d). Postpone :(", missing_seq_num_start, missing_seq_num_count, jb->decode_last_seq_num_with_mark);
// signal to the session that a sequence number is missing (will send a NACK)
// the missing seqnum has been already requested in jb_put() and here we request it again only ONE time
if(jb->callback){
if(prev_missing_seq_num_start != missing_seq_num_start || prev_lasted_missing_seq_num_count != missing_seq_num_count){ // guard to request it only once
jb->cb_data_any.type = tdav_video_jb_cb_data_type_fl;
jb->cb_data_any.ssrc = frame->ssrc;
jb->cb_data_any.fl.seq_num = prev_missing_seq_num_start = missing_seq_num_start;
jb->cb_data_any.fl.count = prev_lasted_missing_seq_num_count = missing_seq_num_count;
jb->callback(&jb->cb_data_any);
}
postpone = tsk_true;
}
}
}
else{
TSK_DEBUG_INFO("frames_count(%lld)>=latency_max(%u)...decoding video frame even if pkts are missing :(", jb->frames_count, (unsigned)jb->latency_max);
jb->decode_last_seq_num_with_mark = -1; // unset()
}
if(!postpone){
item = tsk_list_pop_first_item(jb->frames);
--jb->frames_count;
}
tsk_list_unlock(jb->frames);
tsk_safeobj_unlock(jb);
if(item){
jb->decode_last_timestamp = ((const tdav_video_frame_t*)item->data)->timestamp;
if(jb->callback){
trtp_rtp_packet_t* pkt;
const tsk_list_item_t* _item = item; // save memory address as "tsk_list_foreach() will change it for each loop"
int32_t last_seq_num = -1; // guard against duplicated packets
frame = _item->data;
tsk_list_foreach(_item, frame->pkts){
if(!(pkt = _item->data) || !pkt->payload.size || !pkt->header || pkt->header->seq_num == last_seq_num || !jb->started){
TSK_DEBUG_ERROR("Skipping invalid rtp packet (do not decode!)");
continue;
}
jb->cb_data_rtp.rtp.pkt = pkt;
jb->callback(&jb->cb_data_rtp);
if(pkt->header->marker){
jb->decode_last_seq_num_with_mark = pkt->header->seq_num;
}
}
}
TSK_OBJECT_SAFE_FREE(item);
}
}
#if 1
if (cleaning_delay || jb->frames_count > (int64_t)jb->latency_max){
//x_decode_time = now;
next_decode_duration = 0;
cleaning_delay = ((jb->frames_count << 1) > (int64_t)jb->latency_max); // cleanup up2 half
}
else{
next_decode_duration = (1000 / jb->fps);
_now = tsk_time_now();
if (_now > now) {
if ((_now - now) > next_decode_duration){
next_decode_duration = 0;
}
else {
next_decode_duration -= (_now - now);
}
}
//delay = ( (now > x_decode_time) ? (now - x_decode_time) : (x_decode_duration >> 1)/* do not use zero to avoid endless loop when there is no frame to display */ );
//next_decode_duration = (delay > x_decode_duration) ? 0 : (x_decode_duration - delay);
//x_decode_duration = (1000 / jb->fps);
//x_decode_time += x_decode_duration;
}
//delay = /*(now - x_decode_time);*/(now > x_decode_time) ? (now - x_decode_time) : ( (jb->frames_count >= jb->latency_max) ? 0 : (x_decode_duration >> 1) )/* do not use zero to avoid endless loop when there is no frame to display */;
// delay = (jb->frames_count > jb->latency_max) ? 0 : ( (now > x_decode_time) ? (now - x_decode_time) : (x_decode_duration >> 1)/* do not use zero to avoid endless loop when there is no frame to display */ );
// comparison used as guard against time wrapping
/*if(delay > __toomuch_delay_to_be_valid){
TSK_DEBUG_INFO("Too much delay (%llu) in video jb. Reseting...", delay);
x_decode_time = now;
next_decode_duration = 0;
}
else*/{
//next_decode_duration = (delay > x_decode_duration) ? 0 : (x_decode_duration - delay);
//x_decode_duration = (1000 / jb->fps);
//x_decode_time += x_decode_duration;
}
//TSK_DEBUG_INFO("next_decode_timeout=%llu, delay = %llu", next_decode_duration, delay);
if (cleaning_delay || jb->frames_count > (int64_t)jb->latency_max){
//x_decode_time = now;
next_decode_duration = 0;
cleaning_delay = ((jb->frames_count << 1) > (int64_t)jb->latency_max); // cleanup up2 half
}
else{
next_decode_duration = (1000 / jb->fps);
_now = tsk_time_now();
if (_now > now) {
if ((_now - now) > next_decode_duration){
next_decode_duration = 0;
}
else {
next_decode_duration -= (_now - now);
}
}
//delay = ( (now > x_decode_time) ? (now - x_decode_time) : (x_decode_duration >> 1)/* do not use zero to avoid endless loop when there is no frame to display */ );
//next_decode_duration = (delay > x_decode_duration) ? 0 : (x_decode_duration - delay);
//x_decode_duration = (1000 / jb->fps);
//x_decode_time += x_decode_duration;
}
//delay = /*(now - x_decode_time);*/(now > x_decode_time) ? (now - x_decode_time) : ( (jb->frames_count >= jb->latency_max) ? 0 : (x_decode_duration >> 1) )/* do not use zero to avoid endless loop when there is no frame to display */;
// delay = (jb->frames_count > jb->latency_max) ? 0 : ( (now > x_decode_time) ? (now - x_decode_time) : (x_decode_duration >> 1)/* do not use zero to avoid endless loop when there is no frame to display */ );
// comparison used as guard against time wrapping
/*if(delay > __toomuch_delay_to_be_valid){
TSK_DEBUG_INFO("Too much delay (%llu) in video jb. Reseting...", delay);
x_decode_time = now;
next_decode_duration = 0;
}
else*/{
//next_decode_duration = (delay > x_decode_duration) ? 0 : (x_decode_duration - delay);
//x_decode_duration = (1000 / jb->fps);
//x_decode_time += x_decode_duration;
}
//TSK_DEBUG_INFO("next_decode_timeout=%llu, delay = %llu", next_decode_duration, delay);
#else
next_decode_duration = (1000 / jb->fps);
next_decode_duration = (1000 / jb->fps);
#endif
}
TSK_DEBUG_INFO("Video jitter buffer thread - EXIT");
return tsk_null;
}
TSK_DEBUG_INFO("Video jitter buffer thread - EXIT");
return tsk_null;
}

View File

@ -134,6 +134,8 @@ static const tmedia_codec_action_t __action_encode_bw_down = tmedia_codec_action
(__self)->encoder.pkt_loss_prob_bad = TDAV_SESSION_VIDEO_PKT_LOSS_PROB_BAD; \
(__self)->encoder.pkt_loss_prob_good = TDAV_SESSION_VIDEO_PKT_LOSS_PROB_GOOD; \
}
static int _tdav_session_video_set_defaults(tdav_session_video_t* self);
static int _tdav_session_video_jb_cb(const tdav_video_jb_cb_data_xt* data);
static int _tdav_session_video_open_decoder(tdav_session_video_t* self, uint8_t payload_type);
static int _tdav_session_video_decode(tdav_session_video_t* self, const trtp_rtp_packet_t* packet);
@ -681,6 +683,28 @@ static int tdav_session_video_rtcp_cb(const void* callback_data, const trtp_rtcp
return 0;
}
static int _tdav_session_video_set_defaults(tdav_session_video_t* self)
{
if (!self) {
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
self->jb_enabled = tmedia_defaults_get_videojb_enabled();
self->zero_artifacts = tmedia_defaults_get_video_zeroartifacts_enabled();
self->avpf.max = tmedia_defaults_get_avpf_tail_min();
self->encoder.pkt_loss_level = tdav_session_video_pkt_loss_level_low;
self->encoder.pkt_loss_prob_bad = 0; // honor first report
self->encoder.pkt_loss_prob_good = TDAV_SESSION_VIDEO_PKT_LOSS_PROB_GOOD;
self->encoder.last_frame_time = 0;
// reset rotation info (MUST for reINVITE when mobile device in portrait[90 degrees])
self->encoder.rotation = 0;
TSK_DEBUG_INFO("Video 'zero-artifacts' option = %s", self->zero_artifacts ? "yes" : "no");
return 0;
}
// From jitter buffer to codec
static int _tdav_session_video_jb_cb(const tdav_video_jb_cb_data_xt* data)
{
@ -1044,6 +1068,7 @@ static int tdav_session_video_get(tmedia_session_t* self, tmedia_param_t* param)
static int tdav_session_video_prepare(tmedia_session_t* self)
{
tdav_session_av_t* base = (tdav_session_av_t*)(self);
tdav_session_video_t* video = (tdav_session_video_t*)self;
int ret;
if((ret = tdav_session_av_prepare(base))){
@ -1150,9 +1175,10 @@ static int tdav_session_video_stop(tmedia_session_t* self)
TSK_OBJECT_SAFE_FREE(video->encoder.codec);
tsk_mutex_unlock(video->encoder.h_mutex);
TSK_OBJECT_SAFE_FREE(video->decoder.codec);
// reset rotation info (MUST for reINVITE when mobile device in portrait[90 degrees])
video->encoder.rotation = 0;
// reset default values to make sure next start will be called with right defaults
// do not call this function in start to avoid overriding values defined between prepare() and start()
_tdav_session_video_set_defaults(video);
return ret;
}
@ -1348,10 +1374,8 @@ static int _tdav_session_video_init(tdav_session_video_t *p_self, tmedia_type_t
}
/* init() self */
p_self->jb_enabled = tmedia_defaults_get_videojb_enabled();
p_self->zero_artifacts = tmedia_defaults_get_video_zeroartifacts_enabled();
TSK_DEBUG_INFO("Video 'zero-artifacts' option = %s", p_self->zero_artifacts ? "yes" : "no");
if (!(p_self->encoder.h_mutex = tsk_mutex_create())) {
_tdav_session_video_set_defaults(p_self);
if (!p_self->encoder.h_mutex && !(p_self->encoder.h_mutex = tsk_mutex_create())) {
TSK_DEBUG_ERROR("Failed to create encode mutex");
return -4;
}
@ -1371,10 +1395,6 @@ static int _tdav_session_video_init(tdav_session_video_t *p_self, tmedia_type_t
tmedia_producer_set_enc_callback(p_base->producer, tdav_session_video_producer_enc_cb, p_self);
tmedia_producer_set_raw_callback(p_base->producer, tdav_session_video_raw_cb, p_self);
}
p_self->avpf.max = tmedia_defaults_get_avpf_tail_min();
p_self->encoder.pkt_loss_level = tdav_session_video_pkt_loss_level_low;
p_self->encoder.pkt_loss_prob_bad = 0; // honor first report
p_self->encoder.pkt_loss_prob_good = TDAV_SESSION_VIDEO_PKT_LOSS_PROB_GOOD;
return 0;
}

View File

@ -1997,6 +1997,7 @@ int tmedia_session_mgr_set_onerror_cbfn(tmedia_session_mgr_t* self, const void*
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
// save for sessions created later
self->onerror_cb.fun = fun;
self->onerror_cb.usrdata = usrdata;
@ -2021,6 +2022,7 @@ int tmedia_session_mgr_set_rfc5168_cbfn(tmedia_session_mgr_t* self, const void*
TSK_DEBUG_ERROR("Invalid parameter");
return -1;
}
// save for functions created later
self->rfc5168_cb.fun = fun;
self->rfc5168_cb.usrdata = usrdata;
@ -2121,9 +2123,6 @@ static int _tmedia_session_mgr_load_sessions(tmedia_session_mgr_t* self)
/* set other default values */
// set callback functions
tmedia_session_set_onerror_cbfn(session, self->onerror_cb.usrdata, self->onerror_cb.fun);
/* push session */
tsk_list_push_back_data(self->sessions, (void**)(&session));
}
@ -2147,6 +2146,9 @@ static int _tmedia_session_mgr_load_sessions(tmedia_session_mgr_t* self)
TMEDIA_SESSION_SET_STR(self->type, "local-ipver", self->ipv6 ? "ipv6" : "ipv4"),
TMEDIA_SESSION_SET_INT32(self->type, "bandwidth-level", self->bl),
TMEDIA_SESSION_SET_NULL());
// set callback functions
tmedia_session_mgr_set_onerror_cbfn(self, self->onerror_cb.usrdata, self->onerror_cb.fun);
tmedia_session_mgr_set_rfc5168_cbfn(self, self->rfc5168_cb.usrdata, self->rfc5168_cb.fun);
}
#undef has_media

View File

@ -1688,8 +1688,12 @@ int trtp_manager_stop(trtp_manager_t* self)
TSK_OBJECT_SAFE_FREE(master_copy);
TSK_OBJECT_SAFE_FREE(self->transport);
}
// Free RTCP socket
// Free RTCP info to make sure these values will be updated in next start()
TSK_OBJECT_SAFE_FREE(self->rtcp.local_socket);
TSK_OBJECT_SAFE_FREE(self->rtcp.session);
self->rtcp.public_port = self->rtcp.remote_port = 0;
TSK_FREE(self->rtcp.public_ip);
TSK_FREE(self->rtcp.remote_ip);
// reset default values
self->is_symetric_rtp_checked = self->is_symetric_rtcp_checked = tsk_false;

View File

@ -878,7 +878,7 @@ int tsdp_header_M_diff(const tsdp_header_M_t* M_old, const tsdp_header_M_t* M_ne
diff |= tsdp_header_M_diff_sdes_crypto;
}
}
else if (index == 0) { // (A1 && !AO) means "more" crypto lines, otherwise "less". In all cases if the first matched we're ok
else if (index == 0 && (A0 || A1)) { // (A1 && !AO) means "more" crypto lines, otherwise "less". In all cases if the first matched we're ok
diff |= tsdp_header_M_diff_sdes_crypto;
}
++index;