diff --git a/Makefile.am b/Makefile.am index 8f681a58e5..c8d07eab09 100644 --- a/Makefile.am +++ b/Makefile.am @@ -188,6 +188,7 @@ library_include_HEADERS = \ src/include/switch_scheduler.h \ src/include/switch_core.h \ src/include/switch_core_media.h \ + src/include/switch_core_video.h \ src/include/switch_core_db.h \ src/include/switch_mprintf.h \ src/include/switch_config.h \ @@ -254,6 +255,7 @@ libfreeswitch_la_SOURCES = \ src/switch_core.c \ src/switch_version.c \ src/switch_core_media.c \ + src/switch_core_video.c \ src/switch_sdp.c \ src/switch_scheduler.c \ src/switch_core_db.c \ diff --git a/src/include/switch.h b/src/include/switch.h index ca95045248..2c9245da1b 100644 --- a/src/include/switch.h +++ b/src/include/switch.h @@ -143,6 +143,7 @@ #include "switch_json.h" #include "switch_limit.h" #include "switch_core_media.h" +#include "switch_core_video.h" #include diff --git a/src/include/switch_core.h b/src/include/switch_core.h index c3b31d42a2..b123284670 100644 --- a/src/include/switch_core.h +++ b/src/include/switch_core.h @@ -1621,6 +1621,33 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_decode(switch_codec_t *codec, uint32_t encoded_rate, void *decoded_data, uint32_t *decoded_data_len, uint32_t *decoded_rate, unsigned int *flag); +/*! + \brief Encode video data using a codec handle + \param codec the codec handle to use + \param img the img in I420 format + \param encoded_data the buffer to write the encoded data to + \param encoded_data_len the size of the encoded_data buffer + \param flag flags to exchange + \return SWITCH_STATUS_SUCCESS if the data was encoded + \note encoded_data_len will be rewritten to the in-use size of encoded_data +*/ +SWITCH_DECLARE(switch_status_t) switch_core_codec_encode_video(switch_codec_t *codec, + switch_image_t *img, + void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag); + +/*! + \brief Decode video data using a codec handle + \param codec the codec handle to use + \param frame the frame to be decoded + \param img the new image in I420 format, allocated by the codec + \param flag flags to exchange + \return SWITCH_STATUS_SUCCESS if the data was decoded, and a non-NULL img +*/ +SWITCH_DECLARE(switch_status_t) switch_core_codec_decode_video(switch_codec_t *codec, + switch_frame_t *frame, + switch_image_t **img, unsigned int *flag); + + /*! \brief Destroy an initalized codec handle \param codec the codec handle to destroy diff --git a/src/include/switch_loadable_module.h b/src/include/switch_loadable_module.h index 1dd8fb7b55..514ad45534 100644 --- a/src/include/switch_loadable_module.h +++ b/src/include/switch_loadable_module.h @@ -532,6 +532,45 @@ static inline void switch_core_codec_add_implementation(switch_memory_pool_t *po ///\} +static inline void switch_core_codec_add_video_implementation(switch_memory_pool_t *pool, switch_codec_interface_t *codec_interface, + /*! the IANA code number */ + switch_payload_t ianacode, + /*! the IANA code name */ + const char *iananame, + /*! default fmtp to send (can be overridden by the init function) */ + char *fmtp, + switch_core_codec_init_func_t init, + /*! function to encode raw data into encoded data */ + switch_core_codec_video_encode_func_t encode, + /*! function to decode encoded data into raw data */ + switch_core_codec_video_decode_func_t decode, + /*! deinitalize a codec handle using this implementation */ + switch_core_codec_destroy_func_t destroy) +{ + + switch_codec_implementation_t *impl = (switch_codec_implementation_t *) switch_core_alloc(pool, sizeof(*impl)); + memset(impl, 0, sizeof(*impl)); + impl->codec_type = SWITCH_CODEC_TYPE_VIDEO; + impl->ianacode = ianacode; + impl->iananame = switch_core_strdup(pool, iananame); + impl->fmtp = switch_core_strdup(pool, fmtp); + impl->samples_per_second = 90000; + impl->actual_samples_per_second = 90000; + impl->bits_per_second = 0; + impl->microseconds_per_packet = 0; + impl->samples_per_packet = 0; + impl->number_of_channels = 1; + impl->codec_frames_per_packet = 1; + impl->init = init; + impl->encode_video = encode; + impl->decode_video = decode; + impl->destroy = destroy; + impl->codec_id = codec_interface->codec_id; + impl->next = codec_interface->implementations; + impl->impl_id = switch_core_codec_next_id(); + codec_interface->implementations = impl; +} + #define SWITCH_DECLARE_STATIC_MODULE(init, load, run, shut) void init(void) { \ switch_loadable_module_build_dynamic(__FILE__, load, run, shut, SWITCH_FALSE); \ } diff --git a/src/include/switch_module_interfaces.h b/src/include/switch_module_interfaces.h index 84c4b135d6..20f110ab5f 100644 --- a/src/include/switch_module_interfaces.h +++ b/src/include/switch_module_interfaces.h @@ -639,10 +639,6 @@ struct switch_codec { struct switch_codec *next; switch_core_session_t *session; switch_frame_t *cur_frame; - /*! raw picture for encode */ - switch_picture_t enc_picture; - /*! decoded picture */ - switch_picture_t dec_picture; }; /*! \brief A table of settings and callbacks that define a paticular implementation of a codec */ @@ -679,6 +675,10 @@ struct switch_codec_implementation { switch_core_codec_encode_func_t encode; /*! function to decode encoded data into raw data */ switch_core_codec_decode_func_t decode; + /*! function to encode video raw data into encoded data */ + switch_core_codec_video_encode_func_t encode_video; + /*! function to decode video encoded data into raw data */ + switch_core_codec_video_decode_func_t decode_video; /*! deinitalize a codec handle using this implementation */ switch_core_codec_destroy_func_t destroy; uint32_t codec_id; diff --git a/src/include/switch_types.h b/src/include/switch_types.h index 50c841ae44..65d25d1188 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -541,7 +541,6 @@ SWITCH_DECLARE_DATA extern switch_filenames SWITCH_GLOBAL_filenames; #define SWITCH_MAX_SAMPLE_LEN 48 #define SWITCH_BYTES_PER_SAMPLE 2 /* slin is 2 bytes per sample */ #define SWITCH_RECOMMENDED_BUFFER_SIZE 8192 -#define SWITCH_RECOMMENDED_VIDEO_BUFFER_SIZE 4096 * 1024 /* Fixme: Just Make sure it's big enough for now */ #define SWITCH_MAX_CODECS 50 #define SWITCH_MAX_STATE_HANDLERS 30 #define SWITCH_CORE_QUEUE_LEN 100000 @@ -2095,7 +2094,7 @@ typedef struct switch_caller_extension switch_caller_extension_t; typedef struct switch_caller_application switch_caller_application_t; typedef struct switch_state_handler_table switch_state_handler_table_t; typedef struct switch_timer switch_timer_t; -typedef struct switch_picture switch_picture_t; +typedef struct switch_image switch_image_t; typedef struct switch_codec switch_codec_t; typedef struct switch_core_thread_session switch_core_thread_session_t; typedef struct switch_codec_implementation switch_codec_implementation_t; @@ -2168,6 +2167,14 @@ typedef switch_status_t (*switch_core_codec_decode_func_t) (switch_codec_t *code uint32_t encoded_rate, void *decoded_data, uint32_t *decoded_data_len, uint32_t *decoded_rate, unsigned int *flag); +typedef switch_status_t (*switch_core_codec_video_encode_func_t) (switch_codec_t *codec, + switch_image_t *img, + void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag); + +typedef switch_status_t (*switch_core_codec_video_decode_func_t) (switch_codec_t *codec, + switch_frame_t *frame, + switch_image_t **img, unsigned int *flag); + typedef switch_status_t (*switch_core_codec_init_func_t) (switch_codec_t *, switch_codec_flag_t, const switch_codec_settings_t *codec_settings); typedef switch_status_t (*switch_core_codec_fmtp_parse_func_t) (const char *fmtp, switch_codec_fmtp_t *codec_fmtp); typedef switch_status_t (*switch_core_codec_destroy_func_t) (switch_codec_t *); diff --git a/src/mod/applications/mod_fsv/mod_fsv.c b/src/mod/applications/mod_fsv/mod_fsv.c index c569de1c0e..589ff32a31 100644 --- a/src/mod/applications/mod_fsv/mod_fsv.c +++ b/src/mod/applications/mod_fsv/mod_fsv.c @@ -574,12 +574,18 @@ SWITCH_STANDARD_APP(play_yuv_function) switch_dtmf_t dtmf = { 0 }; switch_frame_t *read_frame; uint32_t width = 0, height = 0, size; - switch_byte_t *yuv; + switch_image_t *img = NULL; + switch_byte_t *yuv = NULL; switch_time_t last_video_ts = 0; int argc; char *argv[3] = { 0 }; char *mydata = switch_core_session_strdup(session, data); + if (!switch_channel_test_flag(channel, CF_VIDEO)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Channel %s has no video\n", switch_channel_get_name(channel)); + goto done; + } + argc = switch_separate_string(mydata, ' ', argv, (sizeof(argv) / sizeof(argv[0]))); if (argc == 0) { @@ -593,13 +599,16 @@ SWITCH_STANDARD_APP(play_yuv_function) width = width ? width : 352; height = height ? height : 288; size = width * height * 3 / 2; - yuv = malloc(size); - if (!yuv) { + img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, width, height, 0); + + if (!img) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Memory Error\n"); goto end; } + yuv = img->planes[SWITCH_PLANE_PACKED]; + // switch_channel_set_flag(channel, CF_VIDEO_PASSIVE); switch_channel_clear_flag(channel, CF_VIDEO_ECHO); @@ -625,7 +634,6 @@ SWITCH_STANDARD_APP(play_yuv_function) switch_channel_answer(channel); codec = switch_core_session_get_video_write_codec(session); - switch_assert(codec); vid_frame.codec = codec; vid_frame.packet = vid_buffer; @@ -664,18 +672,13 @@ SWITCH_STANDARD_APP(play_yuv_function) if (read_frame) switch_core_session_write_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0); { /* video part */ - uint32_t decoded_data_len; uint32_t flag = 0; uint32_t encoded_data_len = 1500; - uint32_t encoded_rate = 0; switch_frame_t *frame = &vid_frame; switch_time_t now = switch_micro_time_now() / 1000; char ts_str[33]; - int delta; + long delta; - codec->enc_picture.width = width; - codec->enc_picture.height = height; - decoded_data_len = width * height * 3 / 2; delta = now - last_video_ts; if (delta > 0) { @@ -684,11 +687,11 @@ SWITCH_STANDARD_APP(play_yuv_function) } sprintf(ts_str, "%u", (uint32_t)frame->timestamp); - text(yuv, width, 20, 20, ts_str); - switch_core_codec_encode(codec, NULL, yuv, decoded_data_len, 0, vid_frame.data, &encoded_data_len, &encoded_rate, &flag); + text(img->planes[SWITCH_PLANE_PACKED], width, 20, 20, ts_str); + switch_core_codec_encode_video(codec, img, vid_frame.data, &encoded_data_len, &flag); while(encoded_data_len) { - // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%u\n", codec->implementation->iananame, encoded_data_len, flag, last_video_ts); + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%lld\n", codec->implementation->iananame, encoded_data_len, flag, last_video_ts); frame->datalen = encoded_data_len; frame->packetlen = frame->datalen + 12; @@ -709,7 +712,7 @@ SWITCH_STANDARD_APP(play_yuv_function) switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0); encoded_data_len = 1500; - switch_core_codec_encode(codec, NULL, NULL, 0, 0, frame->data, &encoded_data_len, &encoded_rate, &flag); + switch_core_codec_encode_video(codec, NULL, vid_frame.data, &encoded_data_len, &flag); } } } @@ -723,7 +726,7 @@ SWITCH_STANDARD_APP(play_yuv_function) close(fd); } - switch_safe_free(yuv); + switch_img_free(img); done: // switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE); @@ -737,34 +740,23 @@ SWITCH_STANDARD_APP(decode_video_function) switch_codec_t *codec = NULL; switch_dtmf_t dtmf = { 0 }; switch_frame_t *frame; - uint32_t size; uint32_t width = 0, height = 0; - switch_byte_t *yuv; switch_time_t last = switch_micro_time_now(); uint32_t max_pictures = 0; uint32_t decoded_pictures = 0; if (!zstr(data)) max_pictures = atoi(data); - switch_channel_set_flag(channel, CF_VIDEO_PASSIVE); - switch_channel_clear_flag(channel, CF_VIDEO_ECHO); - switch_channel_answer(channel); - if (!switch_channel_test_flag(channel, CF_VIDEO)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Channel %s has no video\n", switch_channel_get_name(channel)); goto done; } + switch_channel_set_flag(channel, CF_VIDEO_PASSIVE); + switch_channel_clear_flag(channel, CF_VIDEO_ECHO); + switch_channel_answer(channel); switch_core_session_refresh_video(session); - size = 1920 * 1080 * 3 / 2; - yuv = malloc(size); - - if (!yuv) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Memory Error\n"); - goto done; - } - switch_channel_set_variable(channel, SWITCH_PLAYBACK_TERMINATOR_USED, ""); @@ -801,7 +793,7 @@ SWITCH_STANDARD_APP(decode_video_function) } } - if (frame) { + if (frame && frame->datalen > 0) { switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0); } else { continue; @@ -816,19 +808,17 @@ SWITCH_STANDARD_APP(decode_video_function) *((uint8_t *)frame->data + 8), *((uint8_t *)frame->data + 9), *((uint8_t *)frame->data + 10), frame->m, switch_test_flag(frame, SFF_CNG) ? " CNG" : ""); - if (switch_test_flag(frame, SFF_CNG)) { + if (switch_test_flag(frame, SFF_CNG) || frame->datalen < 3) { continue; } if ( 1 ) { /* video part */ - uint32_t decoded_rate = 0; - uint32_t decoded_data_len = size; uint32_t flag = 0; + switch_image_t *img = NULL; - codec->cur_frame = frame; - switch_core_codec_decode(codec, NULL, frame->data, frame->datalen, 0, yuv, &decoded_data_len, &decoded_rate, &flag); + switch_core_codec_decode_video(codec, frame, &img, &flag); - if (switch_test_flag(codec->cur_frame, SFF_WAIT_KEY_FRAME)) { + if ((switch_test_flag(frame, SFF_WAIT_KEY_FRAME))) { switch_time_t now = switch_micro_time_now(); if (now - last > 3000000) { switch_core_session_refresh_video(session); @@ -837,19 +827,19 @@ SWITCH_STANDARD_APP(decode_video_function) continue; } - if (decoded_data_len > 0) { - if (codec->dec_picture.width > 0 && !width) { - width = codec->dec_picture.width; + if (img) { + if (img->d_w > 0 && !width) { + width = img->d_w; switch_channel_set_variable_printf(channel, "video_width", "%d", width); } - if (codec->dec_picture.height > 0 && !height) { - height = codec->dec_picture.height; + if (img->d_h > 0 && !height) { + height = img->d_h; switch_channel_set_variable_printf(channel, "video_height", "%d", height); } decoded_pictures++; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "picture#%d %dx%d len: %d \n", decoded_pictures, codec->dec_picture.width, codec->dec_picture.height, decoded_data_len); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "picture#%d %dx%d\n", decoded_pictures, img->d_w, img->d_h); if (max_pictures && (decoded_pictures >= max_pictures)) { break; @@ -861,8 +851,6 @@ SWITCH_STANDARD_APP(decode_video_function) switch_core_thread_session_end(session); switch_channel_set_variable(channel, SWITCH_CURRENT_APPLICATION_RESPONSE_VARIABLE, "OK"); - switch_safe_free(yuv); - done: switch_channel_clear_flag(channel, CF_VIDEO_PASSIVE); switch_channel_set_flag(channel, CF_VIDEO_ECHO); diff --git a/src/switch_core_codec.c b/src/switch_core_codec.c index 20fdaf2bd1..0800db506c 100644 --- a/src/switch_core_codec.c +++ b/src/switch_core_codec.c @@ -784,6 +784,65 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_decode(switch_codec_t *codec, return status; } +SWITCH_DECLARE(switch_status_t) switch_core_codec_encode_video(switch_codec_t *codec, + switch_image_t *img, + void *encoded_data, uint32_t *encoded_data_len, unsigned int *flag) +{ + switch_status_t status = SWITCH_STATUS_FALSE; + + switch_assert(codec != NULL); + + if (!codec->implementation || !switch_core_codec_ready(codec)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec is not initialized!\n"); + return SWITCH_STATUS_NOT_INITALIZED; + } + + if (!switch_test_flag(codec, SWITCH_CODEC_FLAG_ENCODE)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec encoder is not initialized!\n"); + return SWITCH_STATUS_NOT_INITALIZED; + } + + if (codec->mutex) switch_mutex_lock(codec->mutex); + + if (codec->implementation->encode_video) { + status = codec->implementation->encode_video(codec, img, encoded_data, encoded_data_len, flag); + } + + if (codec->mutex) switch_mutex_unlock(codec->mutex); + + return status; + +} + +SWITCH_DECLARE(switch_status_t) switch_core_codec_decode_video(switch_codec_t *codec, + switch_frame_t *frame, + switch_image_t **img, unsigned int *flag) +{ + switch_status_t status = SWITCH_STATUS_FALSE; + + switch_assert(codec != NULL); + switch_assert(frame != NULL); + + if (!codec->implementation || !switch_core_codec_ready(codec)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Decode Codec is not initialized!\n"); + return SWITCH_STATUS_NOT_INITALIZED; + } + + if (!switch_test_flag(codec, SWITCH_CODEC_FLAG_DECODE)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec decoder is not initialized!\n"); + return SWITCH_STATUS_NOT_INITALIZED; + } + + if (codec->mutex) switch_mutex_lock(codec->mutex); + + if (codec->implementation->decode_video) { + status = codec->implementation->decode_video(codec, frame, img, flag); + } + if (codec->mutex) switch_mutex_unlock(codec->mutex); + + return status; +} + SWITCH_DECLARE(switch_status_t) switch_core_codec_destroy(switch_codec_t *codec) { switch_mutex_t *mutex = codec->mutex; diff --git a/src/switch_core_media.c b/src/switch_core_media.c index d4e46673be..b8ceaffc01 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -4387,14 +4387,11 @@ static switch_status_t video_bridge_callback(switch_core_session_t *session, swi switch_core_session_write_video_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0); return status; } else { - uint8_t raw_buff[BUF_SIZE]; uint8_t rtp_buff[1500] = { 0 }; - uint32_t decoded_rate = 0; - uint32_t decoded_data_len = BUF_SIZE; uint32_t flag = 0; uint32_t encoded_data_len = 1500; - uint32_t encoded_rate = 0; switch_frame_t write_frame = { 0 }; + switch_image_t *img = NULL; #if 0 switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%d/%s != %d/%s need transcoding!!!\n", @@ -4405,23 +4402,20 @@ static switch_status_t video_bridge_callback(switch_core_session_t *session, swi // if (!strcmp(codec->implementation->iananame, "VP8")) return status; // if (!strcmp(codec->implementation->iananame, "H264")) return status; - codec->cur_frame = read_frame; - switch_core_codec_decode(codec, NULL, read_frame->data, read_frame->datalen, 0, raw_buff, &decoded_data_len, &decoded_rate, &flag); + switch_core_codec_decode_video(codec, read_frame, &img, &flag); - if (decoded_data_len < 3) return SWITCH_STATUS_SUCCESS; + if (!img) return SWITCH_STATUS_SUCCESS; - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%s decoded_data_len: %d size: %dx%d\n", codec->implementation->iananame, decoded_data_len, codec->dec_picture.width, codec->dec_picture.height); + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%s decoded_data_len: %d size: %dx%d\n", codec->implementation->iananame, decoded_data_len, codec->dec_picture.width, codec->dec_picture.height); write_frame.packet = rtp_buff; write_frame.data = rtp_buff + 12; - other_codec->enc_picture.width = codec->dec_picture.width; - other_codec->enc_picture.height = codec->dec_picture.height; encoded_data_len = 1500; - switch_core_codec_encode(other_codec, NULL, raw_buff, decoded_data_len, 0, rtp_buff+12, &encoded_data_len, &encoded_rate, &flag); + switch_core_codec_encode_video(other_codec, img, rtp_buff+12, &encoded_data_len, &flag); while(encoded_data_len) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%u\n", other_codec->implementation->iananame, encoded_data_len, flag, *ts); + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%u\n", other_codec->implementation->iananame, encoded_data_len, flag, *ts); write_frame.datalen = encoded_data_len; write_frame.packetlen = write_frame.datalen + 12; @@ -4448,7 +4442,7 @@ static switch_status_t video_bridge_callback(switch_core_session_t *session, swi switch_core_session_write_video_frame(session_b, &write_frame, SWITCH_IO_FLAG_NONE, 0); encoded_data_len = 1500; - switch_core_codec_encode(other_codec, NULL, NULL, 0, 0, rtp_buff+12, &encoded_data_len, &encoded_rate, &flag); + switch_core_codec_encode_video(other_codec, NULL, rtp_buff+12, &encoded_data_len, &flag); } }