Merge pull request #1192 in FS/freeswitch from bugfix/FS-10050-chromakey to master

* commit 'a0a7b416f4e8aecfa746a86cc06ae2bbd808ff2c':
  FS-10050 add mod_video_filter
  FS-10050 add chromakey

build/modules.conf.in

@@ -1,5 +1,6 @@
 #applications/mod_abstraction
 #applications/mod_av
+#applications/mod_video_filter
 #applications/mod_avmd
 #applications/mod_bert
 #applications/mod_blacklist

configure.ac

@@ -1936,6 +1936,7 @@ AC_CONFIG_FILES([Makefile
 		src/mod/xml_int/mod_xml_rpc/Makefile
 		src/mod/xml_int/mod_xml_scgi/Makefile
 		src/mod/applications/mod_av/Makefile
+		src/mod/applications/mod_video_filter/Makefile
 		src/include/switch_am_config.h
 		build/getsounds.sh
 		build/getlib.sh

src/include/switch_core_video.h

@@ -387,7 +387,11 @@ SWITCH_DECLARE(switch_status_t) switch_I420_copy(const uint8_t* src_y, int src_s
 SWITCH_DECLARE(switch_status_t) switch_I420_copy2(uint8_t *src_planes[], int src_stride[],
 												  uint8_t *dst_planes[], int dst_stride[],
 												  int width, int height);
-/** @} */
+
+/*!\brief chromakey an img, img must be RGBA and return modified img */
+
+SWITCH_DECLARE(void) switch_img_chromakey(switch_image_t *img, switch_rgb_color_t *mask, int threshold);
+
 
 SWITCH_END_EXTERN_C
 #endif

src/mod/applications/mod_video_filter/Makefile.am

@@ -0,0 +1,8 @@
+include $(top_srcdir)/build/modmake.rulesam
+MODNAME=mod_video_filter
+
+mod_LTLIBRARIES = mod_video_filter.la
+mod_av_la_SOURCES  = mod_video_filter.c
+mod_av_la_CFLAGS   = $(AM_CFLAGS)
+mod_av_la_LIBADD   = $(switch_builddir)/libfreeswitch.la
+mod_av_la_LDFLAGS  = -avoid-version -module -no-undefined -shared -lm -lz

src/mod/applications/mod_video_filter/mod_video_filter.c

@@ -0,0 +1,334 @@
+/*
+ * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ * Copyright (C) 2005-2017, Seven Du <dujinfang@gmail.com>
+ *
+ * Version: MPL 1.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ *
+ * The Initial Developer of the Original Code is
+ * Seven Du <dujinfang@gmail.com>
+ * Portions created by the Initial Developer are Copyright (C)
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Seven Du <dujinfang@gmail.com>
+ * Anthony Minessale <anthm@freeswitch.org>
+ *
+ * mod_video_filter -- FS Video Codec / File Format using libav.org
+ *
+ */
+
+#include <switch.h>
+
+switch_loadable_module_interface_t *MODULE_INTERFACE;
+
+SWITCH_MODULE_LOAD_FUNCTION(mod_video_filter_load);
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_video_filter_shutdown);
+SWITCH_MODULE_DEFINITION(mod_video_filter, mod_video_filter_load, mod_video_filter_shutdown, NULL);
+
+typedef struct chromakey_context_s {
+	int threshold;
+	switch_image_t *bgimg;
+	switch_rgb_color_t bgcolor;
+	switch_rgb_color_t mask;
+	switch_core_session_t *session;
+} chromakey_context_t;
+
+static void init_context(chromakey_context_t *context)
+{
+	switch_color_set_rgb(&context->bgcolor, "#000000");
+	switch_color_set_rgb(&context->mask, "#FFFFFF");
+	context->threshold = 300;
+}
+
+static void uninit_context(chromakey_context_t *context)
+{
+	switch_img_free(&context->bgimg);
+}
+
+static void parse_params(chromakey_context_t *context, int start, int argc, char **argv)
+{
+	int n = argc - start;
+	int i = start;
+
+	if (n > 0 && argv[i]) { // color
+		switch_color_set_rgb(&context->mask, argv[i]);
+	}
+
+	i++;
+
+	if (n > 1 && argv[i]) { // thresh
+		int thresh = atoi(argv[i]);
+
+		if (thresh > 0) context->threshold = thresh;
+	}
+
+	i++;
+
+	if (n > 2 && argv[i]) {
+		if (argv[i][0] == '#') { // bgcolor
+			switch_color_set_rgb(&context->bgcolor, argv[i]);
+		} else {
+			if (!context->bgimg) {
+				context->bgimg = switch_img_read_png(argv[i], SWITCH_IMG_FMT_ARGB);
+			}
+		}
+	}
+}
+
+static switch_status_t video_thread_callback(switch_core_session_t *session, switch_frame_t *frame, void *user_data)
+{
+	chromakey_context_t *context = (chromakey_context_t *)user_data;
+	switch_channel_t *channel = switch_core_session_get_channel(session);
+	switch_image_t *img = NULL;
+	void *data = NULL;
+
+	if (!switch_channel_ready(channel)) {
+		return SWITCH_STATUS_FALSE;
+	}
+
+	if (!frame->img) {
+		return SWITCH_STATUS_SUCCESS;
+	}
+
+	data = malloc(frame->img->d_w * frame->img->d_h * 4);
+	switch_assert(data);
+
+	switch_img_to_raw(frame->img, data, frame->img->d_w * 4, SWITCH_IMG_FMT_ARGB);
+	img = switch_img_wrap(NULL, SWITCH_IMG_FMT_ARGB, frame->img->d_w, frame->img->d_h, 1, data);
+	switch_assert(img);
+	switch_img_chromakey(img, &context->mask, context->threshold);
+
+	if (context->bgimg) {
+		switch_img_patch(frame->img, context->bgimg, 0, 0);
+	} else {
+		switch_img_fill(frame->img, 0, 0, img->d_w, img->d_h, &context->bgcolor);
+	}
+
+	switch_img_patch(frame->img, img, 0, 0);
+	switch_img_free(&img);
+	free(data);
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+static switch_bool_t chromakey_bug_callback(switch_media_bug_t *bug, void *user_data, switch_abc_type_t type)
+{
+	chromakey_context_t *context = (chromakey_context_t *)user_data;
+
+	switch_channel_t *channel = switch_core_session_get_channel(context->session);
+
+	switch (type) {
+	case SWITCH_ABC_TYPE_INIT:
+		{
+			switch_channel_set_flag_recursive(channel, CF_VIDEO_DECODED_READ);
+		}
+		break;
+	case SWITCH_ABC_TYPE_CLOSE:
+		{
+			switch_thread_rwlock_unlock(MODULE_INTERFACE->rwlock);
+			switch_channel_clear_flag_recursive(channel, CF_VIDEO_DECODED_READ);
+			uninit_context(context);
+		}
+		break;
+	case SWITCH_ABC_TYPE_READ_VIDEO_PING:
+	case SWITCH_ABC_TYPE_VIDEO_PATCH:
+		{
+			switch_frame_t *frame = switch_core_media_bug_get_video_ping_frame(bug);
+			video_thread_callback(context->session, frame, context);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return SWITCH_TRUE;
+}
+
+#define CHROMAKEY_APP_SYNTAX "<#mask_color> [threshold] [#bg_color|path/to/image.png]"
+SWITCH_STANDARD_APP(chromakey_start_function)
+{
+	switch_media_bug_t *bug;
+	switch_status_t status;
+	switch_channel_t *channel = switch_core_session_get_channel(session);
+	char *argv[4] = { 0 };
+	int argc;
+	char *lbuf;
+	switch_media_bug_flag_t flags = SMBF_READ_VIDEO_PING | SMBF_READ_VIDEO_PATCH;
+	const char *function = "chromakey";
+	chromakey_context_t *context;
+
+	if ((bug = (switch_media_bug_t *) switch_channel_get_private(channel, "_chromakey_bug_"))) {
+		if (!zstr(data) && !strcasecmp(data, "stop")) {
+			switch_channel_set_private(channel, "_chromakey_bug_", NULL);
+			switch_core_media_bug_remove(session, &bug);
+		} else {
+			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_WARNING, "Cannot run 2 chromakey at once on the same channel!\n");
+		}
+		return;
+	}
+
+	switch_channel_wait_for_flag(channel, CF_VIDEO_READY, SWITCH_TRUE, 10000, NULL);
+
+	context = (chromakey_context_t *) switch_core_session_alloc(session, sizeof(*context));
+	switch_assert(context != NULL);
+	memset(context, 0, sizeof(*context));
+	init_context(context);
+	context->session = session;
+
+    if (data && (lbuf = switch_core_session_strdup(session, data))
+        && (argc = switch_separate_string(lbuf, ' ', argv, (sizeof(argv) / sizeof(argv[0]))))) {
+        parse_params(context, 1, argc, argv);
+    }
+
+	switch_thread_rwlock_rdlock(MODULE_INTERFACE->rwlock);
+
+	if ((status = switch_core_media_bug_add(session, function, NULL, chromakey_bug_callback, context, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
+		switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Failure!\n");
+		switch_thread_rwlock_unlock(MODULE_INTERFACE->rwlock);
+		return;
+	}
+
+	switch_channel_set_private(channel, "_chromakey_bug_", bug);
+}
+
+/* API Interface Function */
+#define CHROMAKEY_API_SYNTAX "<uuid> [start|stop] " CHROMAKEY_APP_SYNTAX
+SWITCH_STANDARD_API(chromakey_api_function)
+{
+	switch_core_session_t *rsession = NULL;
+	switch_channel_t *channel = NULL;
+	switch_media_bug_t *bug;
+	switch_status_t status;
+	chromakey_context_t *context;
+	char *mycmd = NULL;
+	int argc = 0;
+	char *argv[25] = { 0 };
+	char *uuid = NULL;
+	char *action = NULL;
+	switch_media_bug_flag_t flags = SMBF_READ_VIDEO_PING | SMBF_READ_VIDEO_PATCH;
+	const char *function = "chromakey";
+
+	if (zstr(cmd)) {
+		goto usage;
+	}
+
+	if (!(mycmd = strdup(cmd))) {
+		goto usage;
+	}
+
+	if ((argc = switch_separate_string(mycmd, ' ', argv, (sizeof(argv) / sizeof(argv[0])))) < 2) {
+		goto usage;
+	}
+
+	uuid = argv[0];
+	action = argv[1];
+
+	if (!(rsession = switch_core_session_locate(uuid))) {
+		stream->write_function(stream, "-ERR Cannot locate session!\n");
+		goto done;
+	}
+
+	channel = switch_core_session_get_channel(rsession);
+
+	if ((bug = (switch_media_bug_t *) switch_channel_get_private(channel, "_chromakey_bug_"))) {
+		if (!zstr(action)) {
+			if (!strcasecmp(action, "stop")) {
+				switch_channel_set_private(channel, "_chromakey_bug_", NULL);
+				switch_core_media_bug_remove(rsession, &bug);
+				stream->write_function(stream, "+OK Success\n");
+			} else if (!strcasecmp(action, "start")) {
+				context = (chromakey_context_t *) switch_core_media_bug_get_user_data(bug);
+				switch_assert(context);
+				parse_params(context, 2, argc, argv);
+				stream->write_function(stream, "+OK Success\n");
+			}
+		} else {
+			stream->write_function(stream, "-ERR Invalid action\n");
+		}
+		goto done;
+	}
+
+	if (!zstr(action) && strcasecmp(action, "start")) {
+		goto usage;
+	}
+
+	context = (chromakey_context_t *) switch_core_session_alloc(rsession, sizeof(*context));
+	switch_assert(context != NULL);
+	context->session = rsession;
+
+	init_context(context);
+	parse_params(context, 2, argc, argv);
+
+	switch_thread_rwlock_rdlock(MODULE_INTERFACE->rwlock);
+
+	if ((status = switch_core_media_bug_add(rsession, function, NULL,
+											chromakey_bug_callback, context, 0, flags, &bug)) != SWITCH_STATUS_SUCCESS) {
+		stream->write_function(stream, "-ERR Failure!\n");
+		switch_thread_rwlock_unlock(MODULE_INTERFACE->rwlock);
+		goto done;
+	} else {
+		switch_channel_set_private(channel, "_chromakey_bug_", bug);
+		stream->write_function(stream, "+OK Success\n");
+		goto done;
+	}
+
+ usage:
+	stream->write_function(stream, "-USAGE: %s\n", CHROMAKEY_API_SYNTAX);
+
+ done:
+	if (rsession) {
+		switch_core_session_rwunlock(rsession);
+	}
+
+	switch_safe_free(mycmd);
+	return SWITCH_STATUS_SUCCESS;
+}
+
+
+SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_video_filter_shutdown)
+{
+	return SWITCH_STATUS_SUCCESS;
+}
+
+SWITCH_MODULE_LOAD_FUNCTION(mod_video_filter_load)
+{
+	switch_application_interface_t *app_interface;
+    switch_api_interface_t *api_interface;
+
+	/* connect my internal structure to the blank pointer passed to me */
+	*module_interface = switch_loadable_module_create_module_interface(pool, modname);
+	MODULE_INTERFACE = *module_interface;
+
+	SWITCH_ADD_APP(app_interface, "chromakey", "chromakey", "chromakey bug",
+				   chromakey_start_function, CHROMAKEY_APP_SYNTAX, SAF_NONE);
+
+    SWITCH_ADD_API(api_interface, "chromakey", "chromakey", chromakey_api_function, CHROMAKEY_API_SYNTAX);
+
+    switch_console_set_complete("add chromakey ::console::list_uuid ::[start:stop");
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4:
+ */

src/switch_core_video.c

@@ -73,6 +73,13 @@ static inline void switch_color_rgb2yuv(switch_rgb_color_t *rgb, switch_yuv_colo
 static inline void switch_color_yuv2rgb(switch_yuv_color_t *yuv, switch_rgb_color_t *rgb);
 #endif
 
+/*!\brief compute distance between two colors
+*
+* \param[in]    c1        RGB color1
+* \param[in]    c2        RGB color2
+*/
+static inline int switch_color_distance(switch_rgb_color_t *c1, switch_rgb_color_t *c2);
+
 /*!\brief Draw a pixel on an image
 *
 * \param[in]    img       Image descriptor
@@ -527,6 +534,27 @@ SWITCH_DECLARE(switch_image_t *) switch_img_copy_rect(switch_image_t *img, uint3
 #endif
 }
 
+SWITCH_DECLARE(void) switch_img_chromakey(switch_image_t *img, switch_rgb_color_t *mask, int threshold)
+{
+	uint8_t *pixel;
+	switch_assert(img);
+
+	if (img->fmt != SWITCH_IMG_FMT_ARGB) return;
+
+	pixel = img->planes[SWITCH_PLANE_PACKED];
+
+	for (; pixel < (img->planes[SWITCH_PLANE_PACKED] + img->d_w * img->d_h * 4); pixel += 4) {
+		switch_rgb_color_t *color = (switch_rgb_color_t *)pixel;
+		int distance = switch_color_distance(color, mask);
+
+		if (distance <= threshold) {
+			*pixel = 0;
+		}
+	}
+
+	return;
+}
+
 static inline void switch_img_draw_pixel(switch_image_t *img, int x, int y, switch_rgb_color_t *color)
 {
 #ifdef SWITCH_HAVE_YUV
@@ -767,6 +795,16 @@ static inline void switch_color_rgb2yuv(switch_rgb_color_t *rgb, switch_yuv_colo
 }
 #endif
 
+static inline int switch_color_distance(switch_rgb_color_t *c1, switch_rgb_color_t *c2)
+{
+    int rmean = ( c1->r + c2->r ) / 2;
+	int r = c1->r - c2->r;
+    int g = c1->g - c2->g;
+    int b = c1->b - c2->b;
+
+    return sqrt((((512+rmean)*r*r)>>8) + 4*g*g + (((767-rmean)*b*b)>>8));
+}
+
 #define CLAMP(val) MAX(0, MIN(val, 255))
 
 #ifdef SWITCH_HAVE_YUV
@@ -2008,7 +2046,7 @@ static inline uint32_t switch_img_fmt2fourcc(switch_img_fmt_t fmt)
 		case SWITCH_IMG_FMT_YVYU:      fourcc = (uint32_t)FOURCC_ANY ; break;
 		case SWITCH_IMG_FMT_BGR24:     fourcc = (uint32_t)FOURCC_RAW ; break;
 		case SWITCH_IMG_FMT_RGB32_LE:  fourcc = (uint32_t)FOURCC_ANY ; break;
-		case SWITCH_IMG_FMT_ARGB:      fourcc = (uint32_t)FOURCC_ANY ; break;
+		case SWITCH_IMG_FMT_ARGB:      fourcc = (uint32_t)FOURCC_BGRA; break;
 		case SWITCH_IMG_FMT_ARGB_LE:   fourcc = (uint32_t)FOURCC_ANY ; break;
 		case SWITCH_IMG_FMT_RGB565_LE: fourcc = (uint32_t)FOURCC_ANY ; break;
 		case SWITCH_IMG_FMT_RGB555_LE: fourcc = (uint32_t)FOURCC_ANY ; break;