FS-11014 [core] add vad to core
This commit is contained in:
Seven Du
Muteesa Fred
parent
bea634679c
commit
99d2e5e243
10
Makefile.am
10
Makefile.am
|
|
@ -179,6 +179,10 @@ if HAVE_GUMBO
|
|||
CORE_CFLAGS += -DSWITCH_HAVE_GUMBO $(LIBGUMBO_CFLAGS)
|
||||
endif
|
||||
|
||||
if HAVE_FVAD
|
||||
CORE_CFLAGS += -DSWITCH_HAVE_FVAD $(LIBFVAD_CFLAGS)
|
||||
endif
|
||||
|
||||
##
|
||||
## libfreeswitch
|
||||
##
|
||||
|
|
@ -241,9 +245,9 @@ CORE_LIBS+=libfreeswitch_libyuv.la
|
|||
endif
|
||||
|
||||
lib_LTLIBRARIES = libfreeswitch.la
|
||||
libfreeswitch_la_CFLAGS = $(CORE_CFLAGS) $(SQLITE_CFLAGS) $(GUMBO_CFLAGS) $(FREETYPE_CFLAGS) $(CURL_CFLAGS) $(PCRE_CFLAGS) $(SPEEX_CFLAGS) $(LIBEDIT_CFLAGS) $(openssl_CFLAGS) $(AM_CFLAGS)
|
||||
libfreeswitch_la_CFLAGS = $(CORE_CFLAGS) $(SQLITE_CFLAGS) $(GUMBO_CFLAGS) $(FVAD_CFLAGS) $(FREETYPE_CFLAGS) $(CURL_CFLAGS) $(PCRE_CFLAGS) $(SPEEX_CFLAGS) $(LIBEDIT_CFLAGS) $(openssl_CFLAGS) $(AM_CFLAGS)
|
||||
libfreeswitch_la_LDFLAGS = -version-info 1:0:0 $(AM_LDFLAGS) $(PLATFORM_CORE_LDFLAGS) -no-undefined
|
||||
libfreeswitch_la_LIBADD = $(CORE_LIBS) $(APR_LIBS) $(SQLITE_LIBS) $(GUMBO_LIBS) $(FREETYPE_LIBS) $(CURL_LIBS) $(PCRE_LIBS) $(SPEEX_LIBS) $(LIBEDIT_LIBS) $(openssl_LIBS) $(PLATFORM_CORE_LIBS)
|
||||
libfreeswitch_la_LIBADD = $(CORE_LIBS) $(APR_LIBS) $(SQLITE_LIBS) $(GUMBO_LIBS) $(FVAD_LIBS) $(FREETYPE_LIBS) $(CURL_LIBS) $(PCRE_LIBS) $(SPEEX_LIBS) $(LIBEDIT_LIBS) $(openssl_LIBS) $(PLATFORM_CORE_LIBS)
|
||||
libfreeswitch_la_DEPENDENCIES = $(BUILT_SOURCES)
|
||||
|
||||
if HAVE_PNG
|
||||
|
|
@ -312,6 +316,7 @@ library_include_HEADERS = \
|
|||
src/include/switch_utf8.h \
|
||||
src/include/switch_msrp.h \
|
||||
src/include/switch_vpx.h \
|
||||
src/include/switch_vad.h \
|
||||
libs/libteletone/src/libteletone_detect.h \
|
||||
libs/libteletone/src/libteletone_generate.h \
|
||||
libs/libteletone/src/libteletone.h \
|
||||
|
|
@ -395,6 +400,7 @@ libfreeswitch_la_SOURCES = \
|
|||
src/switch_hashtable.c\
|
||||
src/switch_utf8.c \
|
||||
src/switch_msrp.c \
|
||||
src/switch_vad.c \
|
||||
src/switch_vpx.c \
|
||||
libs/libtpl-1.5/src/tpl.c \
|
||||
libs/libteletone/src/libteletone_detect.c \
|
||||
|
|
|
|||
|
|
@ -1310,6 +1310,10 @@ PKG_CHECK_MODULES([GUMBO], [gumbo >= 0.10.1],[
|
|||
AM_CONDITIONAL([HAVE_GUMBO],[true])],[
|
||||
AC_MSG_RESULT([no]); AM_CONDITIONAL([HAVE_GUMBO],[false])])
|
||||
|
||||
PKG_CHECK_MODULES([FVAD], [libfvad >= 1.0],[
|
||||
AM_CONDITIONAL([HAVE_FVAD],[true])],[
|
||||
AC_MSG_RESULT([no]); AM_CONDITIONAL([HAVE_FVAD],[false])])
|
||||
|
||||
PKG_CHECK_MODULES([SQLITE], [sqlite3 >= 3.6.20])
|
||||
PKG_CHECK_MODULES([CURL], [libcurl >= 7.19])
|
||||
PKG_CHECK_MODULES([PCRE], [libpcre >= 7.8])
|
||||
|
|
|
|||
|
|
@ -145,6 +145,7 @@
|
|||
#include "switch_core_video.h"
|
||||
#include "switch_jitterbuffer.h"
|
||||
#include "switch_estimators.h"
|
||||
#include "switch_vad.h"
|
||||
#include <libteletone.h>
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -622,6 +622,14 @@ typedef enum {
|
|||
} switch_vad_flag_enum_t;
|
||||
typedef uint32_t switch_vad_flag_t;
|
||||
|
||||
typedef enum {
|
||||
SWITCH_VAD_STATE_NONE,
|
||||
SWITCH_VAD_STATE_START_TALKING,
|
||||
SWITCH_VAD_STATE_TALKING,
|
||||
SWITCH_VAD_STATE_STOP_TALKING,
|
||||
SWITCH_VAD_STATE_ERROR
|
||||
} switch_vad_state_t;
|
||||
typedef struct switch_vad_s switch_vad_t;
|
||||
|
||||
typedef struct error_period {
|
||||
int64_t start;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
/*
|
||||
* FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
|
||||
* Copyright (C) 2018, Anthony Minessale II <anthm@freeswitch.org>
|
||||
*
|
||||
* Version: MPL 1.1
|
||||
*
|
||||
* The contents of this file are subject to the Mozilla Public License Version
|
||||
* 1.1 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* http://www.mozilla.org/MPL/
|
||||
*
|
||||
* Software distributed under the License is distributed on an "AS IS" basis,
|
||||
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
|
||||
* for the specific language governing rights and limitations under the
|
||||
* License.
|
||||
*
|
||||
* The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
|
||||
*
|
||||
* The Initial Developer of the Original Code is
|
||||
* Anthony Minessale II <anthm@freeswitch.org>
|
||||
* Portions created by the Initial Developer are Copyright (C)
|
||||
* the Initial Developer. All Rights Reserved.
|
||||
*
|
||||
* Contributor(s):
|
||||
*
|
||||
* Seven Du <dujinfang@gmail.com>
|
||||
*
|
||||
*
|
||||
* switch_vad.h VAD code with optional libfvad
|
||||
*
|
||||
*/
|
||||
/*!
|
||||
\defgroup vad1 VAD code with optional libfvad
|
||||
\ingroup core1
|
||||
\{
|
||||
*/
|
||||
#ifndef FREESWITCH_VAD_H
|
||||
#define FREESWITCH_VAD_H
|
||||
|
||||
SWITCH_BEGIN_EXTERN_C
|
||||
|
||||
SWITCH_DECLARE(switch_vad_t *) switch_vad_init(int sample_rate, int channels);
|
||||
|
||||
/*
|
||||
* Valid modes are -1 ("disable fvad, using native"), 0 ("quality"), 1 ("low bitrate"), 2 ("aggressive"), and 3 * ("very aggressive").
|
||||
* The default mode is -1.
|
||||
*/
|
||||
|
||||
SWITCH_DECLARE(int) switch_vad_set_mode(switch_vad_t *vad, int mode);
|
||||
SWITCH_DECLARE(void) switch_vad_set_param(switch_vad_t *vad, const char *key, int val);
|
||||
SWITCH_DECLARE(switch_vad_state_t) switch_vad_process(switch_vad_t *vad, int16_t *data, unsigned int samples);
|
||||
SWITCH_DECLARE(void) switch_vad_reset(switch_vad_t *vad);
|
||||
SWITCH_DECLARE(void) switch_vad_destroy(switch_vad_t **vad);
|
||||
|
||||
SWITCH_END_EXTERN_C
|
||||
#endif
|
||||
/* For Emacs:
|
||||
* Local Variables:
|
||||
* mode:c
|
||||
* indent-tabs-mode:t
|
||||
* tab-width:4
|
||||
* c-basic-offset:4
|
||||
* End:
|
||||
* For VIM:
|
||||
* vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
|
||||
*/
|
||||
|
|
@ -6128,6 +6128,73 @@ SWITCH_STANDARD_APP(deduplicate_dtmf_app_function)
|
|||
}
|
||||
}
|
||||
|
||||
SWITCH_STANDARD_APP(vad_test_function)
|
||||
{
|
||||
switch_channel_t *channel = switch_core_session_get_channel(session);
|
||||
switch_codec_implementation_t imp = { 0 };
|
||||
switch_vad_t *vad;
|
||||
switch_frame_t *frame = { 0 };
|
||||
switch_vad_state_t vad_state;
|
||||
int mode = -1;
|
||||
const char *var = NULL;
|
||||
int tmp;
|
||||
|
||||
if (!zstr(data)) {
|
||||
mode = atoi(data);
|
||||
|
||||
if (mode > 3) mode = 3;
|
||||
}
|
||||
|
||||
switch_core_session_raw_read(session);
|
||||
switch_core_session_get_read_impl(session, &imp);
|
||||
|
||||
vad = switch_vad_init(imp.samples_per_second, imp.number_of_channels);
|
||||
switch_assert(vad);
|
||||
switch_vad_set_mode(vad, mode);
|
||||
|
||||
if ((var = switch_channel_get_variable(channel, "vad_hangover_len"))) {
|
||||
tmp = atoi(var);
|
||||
|
||||
if (tmp > 0) switch_vad_set_param(vad, "hangover_len", tmp);
|
||||
}
|
||||
|
||||
if ((var = switch_channel_get_variable(channel, "vad_thresh"))) {
|
||||
tmp = atoi(var);
|
||||
|
||||
if (tmp > 0) switch_vad_set_param(vad, "thresh", tmp);
|
||||
}
|
||||
|
||||
if ((var = switch_channel_get_variable(channel, "vad_timeout_len"))) {
|
||||
tmp = atoi(var);
|
||||
|
||||
if (tmp > 0) switch_vad_set_param(vad, "timeout_len", tmp);
|
||||
}
|
||||
|
||||
while(switch_channel_ready(channel)) {
|
||||
switch_core_session_read_frame(session, &frame, SWITCH_IO_FLAG_NONE, 0);
|
||||
|
||||
if (switch_test_flag(frame, SFF_CNG)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
vad_state = switch_vad_process(vad, frame->data, frame->datalen / 2);
|
||||
|
||||
if (vad_state == SWITCH_VAD_STATE_START_TALKING) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "START TALKING\n");
|
||||
switch_core_session_write_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
|
||||
} else if (vad_state == SWITCH_VAD_STATE_STOP_TALKING) {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "STOP TALKING\n");
|
||||
} else if (vad_state == SWITCH_VAD_STATE_TALKING) {
|
||||
switch_core_session_write_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
|
||||
} else {
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "vad_state: %d\n", vad_state);
|
||||
}
|
||||
}
|
||||
|
||||
switch_vad_destroy(&vad);
|
||||
switch_core_session_reset(session, SWITCH_TRUE, SWITCH_TRUE);
|
||||
}
|
||||
|
||||
#define SPEAK_DESC "Speak text to a channel via the tts interface"
|
||||
#define DISPLACE_DESC "Displace audio from a file to the channels input"
|
||||
#define SESS_REC_DESC "Starts a background recording of the entire session"
|
||||
|
|
@ -6462,6 +6529,7 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_dptools_load)
|
|||
SWITCH_ADD_APP(app_interface, "pickup", "Pickup", "Pickup a call", pickup_function, PICKUP_SYNTAX, SAF_SUPPORT_NOMEDIA);
|
||||
SWITCH_ADD_APP(app_interface, "deduplicate_dtmf", "Prevent duplicate inband + 2833 dtmf", "", deduplicate_dtmf_app_function, "[only_rtp]", SAF_SUPPORT_NOMEDIA);
|
||||
|
||||
SWITCH_ADD_APP(app_interface, "vad_test", "VAD test", "VAD test, mode = -1(default), 0, 1, 2, 3", vad_test_function, "[mode]", SAF_NONE);
|
||||
|
||||
SWITCH_ADD_DIALPLAN(dp_interface, "inline", inline_dialplan_hunt);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,219 @@
|
|||
/*
|
||||
* FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
|
||||
* Copyright (C) 2018, Anthony Minessale II <anthm@freeswitch.org>
|
||||
*
|
||||
* Version: MPL 1.1
|
||||
*
|
||||
* The contents of this file are subject to the Mozilla Public License Version
|
||||
* 1.1 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* http://www.mozilla.org/MPL/
|
||||
*
|
||||
* Software distributed under the License is distributed on an "AS IS" basis,
|
||||
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
|
||||
* for the specific language governing rights and limitations under the
|
||||
* License.
|
||||
*
|
||||
* The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
|
||||
*
|
||||
* The Initial Developer of the Original Code is
|
||||
* Anthony Minessale II <anthm@freeswitch.org>
|
||||
* Portions created by the Initial Developer are Copyright (C)
|
||||
* the Initial Developer. All Rights Reserved.
|
||||
*
|
||||
* Contributor(s):
|
||||
*
|
||||
* Seven Du <dujinfang@gmail.com>
|
||||
*
|
||||
*
|
||||
* switch_vad.c VAD code with optional libfvad
|
||||
*
|
||||
*/
|
||||
|
||||
#include <switch.h>
|
||||
|
||||
#ifdef SWITCH_HAVE_FVAD
|
||||
#include <fvad.h>
|
||||
#endif
|
||||
|
||||
struct switch_vad_s {
|
||||
int talking;
|
||||
int talked;
|
||||
int talk_hits;
|
||||
int hangover;
|
||||
int hangover_len;
|
||||
int divisor;
|
||||
int thresh;
|
||||
int timeout_len;
|
||||
int timeout;
|
||||
int channels;
|
||||
int sample_rate;
|
||||
|
||||
int _hangover_len;
|
||||
int _thresh;
|
||||
int _timeout_len;
|
||||
#ifdef SWITCH_HAVE_FVAD
|
||||
Fvad *fvad;
|
||||
#endif
|
||||
};
|
||||
|
||||
SWITCH_DECLARE(switch_vad_t *) switch_vad_init(int sample_rate, int channels)
|
||||
{
|
||||
switch_vad_t *vad = malloc(sizeof(switch_vad_t));
|
||||
|
||||
if (!vad) return NULL;
|
||||
|
||||
memset(vad, 0, sizeof(*vad));
|
||||
vad->sample_rate = sample_rate ? sample_rate : 8000;
|
||||
vad->channels = channels;
|
||||
vad->_hangover_len = 25;
|
||||
vad->_thresh = 100;
|
||||
vad->_timeout_len = 25;
|
||||
|
||||
switch_vad_reset(vad);
|
||||
|
||||
return vad;
|
||||
}
|
||||
|
||||
//Valid modes are 0 ("quality"), 1 ("low bitrate"), 2 ("aggressive"), and 3 * ("very aggressive"). The default mode is 0.
|
||||
SWITCH_DECLARE(int) switch_vad_set_mode(switch_vad_t *vad, int mode)
|
||||
{
|
||||
#ifdef SWITCH_HAVE_FVAD
|
||||
int ret;
|
||||
|
||||
if (mode < 0) return 0;
|
||||
|
||||
vad->fvad = fvad_new();
|
||||
ret = fvad_set_mode(vad->fvad, mode);
|
||||
fvad_set_sample_rate(vad->fvad, vad->sample_rate);
|
||||
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "libfvad started, mode = %d\n", mode);
|
||||
return ret;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
SWITCH_DECLARE(void) switch_vad_set_param(switch_vad_t *vad, const char *key, int val)
|
||||
{
|
||||
if (!key) return;
|
||||
|
||||
if (!strcmp(key, "hangover_len")) {
|
||||
vad->hangover_len = vad->_hangover_len = val;
|
||||
} else if (!strcmp(key, "thresh")) {
|
||||
vad->thresh = vad->_thresh = val;
|
||||
} else if (!strcmp(key, "timeout_len")) {
|
||||
vad->timeout = vad->timeout_len = vad->_timeout_len = val;
|
||||
}
|
||||
}
|
||||
|
||||
SWITCH_DECLARE(void) switch_vad_reset(switch_vad_t *vad)
|
||||
{
|
||||
#ifdef SWITCH_HAVE_FVAD
|
||||
if (vad->fvad) {
|
||||
fvad_reset(vad->fvad);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
vad->talking = 0;
|
||||
vad->talked = 0;
|
||||
vad->talk_hits = 0;
|
||||
vad->hangover = 0;
|
||||
vad->hangover_len = vad->_hangover_len;
|
||||
vad->divisor = vad->sample_rate / 8000;
|
||||
vad->thresh = vad->_thresh;
|
||||
vad->timeout_len = vad->_timeout_len;
|
||||
vad->timeout = vad->timeout_len;
|
||||
}
|
||||
|
||||
SWITCH_DECLARE(switch_vad_state_t) switch_vad_process(switch_vad_t *vad, int16_t *data, unsigned int samples)
|
||||
{
|
||||
int energy = 0, j = 0, count = 0;
|
||||
int score = 0;
|
||||
switch_vad_state_t vad_state = SWITCH_VAD_STATE_NONE;
|
||||
|
||||
#ifdef SWITCH_HAVE_FVAD
|
||||
if (vad->fvad) {
|
||||
int ret = fvad_process(vad->fvad, data, samples);
|
||||
|
||||
// printf("%d ", ret); fflush(stdout);
|
||||
|
||||
score = vad->thresh + ret - 1;
|
||||
} else {
|
||||
#endif
|
||||
|
||||
for (energy = 0, j = 0, count = 0; count < samples; count++) {
|
||||
energy += abs(data[j]);
|
||||
j += vad->channels;
|
||||
}
|
||||
|
||||
score = (uint32_t) (energy / (samples / vad->divisor));
|
||||
|
||||
#ifdef SWITCH_HAVE_FVAD
|
||||
}
|
||||
#endif
|
||||
|
||||
// printf("%d ", score); fflush(stdout);
|
||||
// printf("yay %d %d %d\n", score, vad->hangover, vad->talking);
|
||||
|
||||
if (vad->talking && score < vad->thresh) {
|
||||
if (vad->hangover > 0) {
|
||||
vad->hangover--;
|
||||
} else {// if (hangover <= 0) {
|
||||
vad->talking = 0;
|
||||
vad->talk_hits = 0;
|
||||
vad->hangover = 0;
|
||||
}
|
||||
} else {
|
||||
if (score >= vad->thresh) {
|
||||
vad_state = vad->talking ? SWITCH_VAD_STATE_TALKING : SWITCH_VAD_STATE_START_TALKING;
|
||||
vad->talking = 1;
|
||||
vad->hangover = vad->hangover_len;
|
||||
}
|
||||
}
|
||||
|
||||
// printf("WTF %d %d %d\n", score, vad->talked, vad->talking);
|
||||
|
||||
if (vad->talking) {
|
||||
vad->talk_hits++;
|
||||
// printf("WTF %d %d %d\n", vad->talking, vad->talk_hits, vad->talked);
|
||||
if (vad->talk_hits > 10) {
|
||||
vad->talked = 1;
|
||||
vad_state = SWITCH_VAD_STATE_TALKING;
|
||||
}
|
||||
} else {
|
||||
vad->talk_hits = 0;
|
||||
}
|
||||
|
||||
if (vad->timeout > 0 && !vad->talking) {
|
||||
vad->timeout--;
|
||||
}
|
||||
|
||||
if ((vad->talked && !vad->talking)) {
|
||||
// printf("NOT TALKING ANYMORE\n");
|
||||
vad->talked = 0;
|
||||
vad->timeout = vad->timeout_len;
|
||||
vad_state = SWITCH_VAD_STATE_STOP_TALKING;
|
||||
} else {
|
||||
// if (vad->skip > 0) {
|
||||
// vad->skip--;
|
||||
// }
|
||||
}
|
||||
|
||||
if (vad_state) return vad_state;
|
||||
|
||||
return SWITCH_VAD_STATE_NONE;
|
||||
}
|
||||
|
||||
SWITCH_DECLARE(void) switch_vad_destroy(switch_vad_t **vad)
|
||||
{
|
||||
if (*vad) {
|
||||
|
||||
#ifdef SWITCH_HAVE_FVAD
|
||||
if ((*vad)->fvad) free ((*vad)->fvad);
|
||||
#endif
|
||||
|
||||
free(*vad);
|
||||
*vad = NULL;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue