From 20295c53efcda037cb67703c36fc3fc62656cc2d Mon Sep 17 00:00:00 2001 From: Andrey Volk Date: Sun, 15 Sep 2019 23:56:04 +0400 Subject: [PATCH] FS-11827: [Core] Add feature allowing FreeSWITCH to store CORE DB (SQLite) fully in memory (core-db-dsn). --- src/include/switch_core.h | 13 ++++++-- src/include/switch_core_db.h | 11 +++++++ src/include/switch_types.h | 1 + src/switch_core_db.c | 59 +++++++++++++++++++++++++++++---- src/switch_core_sqldb.c | 64 +++++++++++++++++++++++++----------- 5 files changed, 121 insertions(+), 27 deletions(-) diff --git a/src/include/switch_core.h b/src/include/switch_core.h index f5df53ff11..dbff45d46b 100644 --- a/src/include/switch_core.h +++ b/src/include/switch_core.h @@ -1854,6 +1854,13 @@ SWITCH_DECLARE(switch_codec_t *) switch_core_session_get_video_write_codec(_In_ */ SWITCH_DECLARE(switch_core_db_t *) switch_core_db_open_file(const char *filename); +/*! + \brief Open a core db (SQLite) in-memory + \param uri to the db to open + \return the db handle +*/ +SWITCH_DECLARE(switch_core_db_t *) switch_core_db_open_in_memory(const char *uri); + /*! \brief Execute a sql stmt until it is accepted \param db the db handle @@ -2485,7 +2492,8 @@ typedef int (*switch_core_db_event_callback_func_t) (void *pArg, switch_event_t #define CACHE_DB_LEN 256 typedef enum { CDF_INUSE = (1 << 0), - CDF_PRUNE = (1 << 1) + CDF_PRUNE = (1 << 1), + CDF_NONEXPIRING = (1 << 2) } cache_db_flag_t; typedef enum { @@ -2495,13 +2503,14 @@ typedef enum { } switch_cache_db_handle_type_t; typedef union { - switch_core_db_t *core_db_dbh; + switch_core_db_handle_t *core_db_dbh; switch_odbc_handle_t *odbc_dbh; switch_database_interface_handle_t *database_interface_dbh; } switch_cache_db_native_handle_t; typedef struct { char *db_path; + switch_bool_t in_memory; } switch_cache_db_core_db_options_t; typedef struct { diff --git a/src/include/switch_core_db.h b/src/include/switch_core_db.h index 4a6b3eb4bf..ab8351b782 100644 --- a/src/include/switch_core_db.h +++ b/src/include/switch_core_db.h @@ -51,6 +51,12 @@ SWITCH_BEGIN_EXTERN_C * following opaque structure. */ typedef struct sqlite3 switch_core_db_t; + +struct switch_core_db_handle { + switch_bool_t in_memory; + switch_core_db_t *handle; +}; + typedef struct sqlite3_stmt switch_core_db_stmt_t; typedef int (*switch_core_db_callback_func_t) (void *pArg, int argc, char **argv, char **columnNames); @@ -102,6 +108,11 @@ SWITCH_DECLARE(int) switch_core_db_close(switch_core_db_t *db); */ SWITCH_DECLARE(int) switch_core_db_open(const char *filename, switch_core_db_t **ppDb); +/** + Same as switch_core_db_open() but additionally allows SQLITE_OPEN_URI + */ +SWITCH_DECLARE(int) switch_core_db_open_v2(const char *filename, switch_core_db_t **ppDb); + /** * ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), * even empty strings, are always zero-terminated. ^The return diff --git a/src/include/switch_types.h b/src/include/switch_types.h index 9d8445a1e7..456b3c4c66 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -2318,6 +2318,7 @@ typedef struct switch_codec_implementation switch_codec_implementation_t; typedef struct switch_buffer switch_buffer_t; typedef union switch_codec_settings switch_codec_settings_t; typedef struct switch_codec_fmtp switch_codec_fmtp_t; +typedef struct switch_core_db_handle switch_core_db_handle_t; typedef struct switch_odbc_handle switch_odbc_handle_t; typedef struct switch_database_interface_handle switch_database_interface_handle_t; diff --git a/src/switch_core_db.c b/src/switch_core_db.c index ae118b7694..d3701ce91f 100644 --- a/src/switch_core_db.c +++ b/src/switch_core_db.c @@ -50,6 +50,11 @@ SWITCH_DECLARE(int) switch_core_db_open(const char *filename, switch_core_db_t * return sqlite3_open(filename, ppDb); } +SWITCH_DECLARE(int) switch_core_db_open_v2(const char *filename, switch_core_db_t **ppDb) +{ + return sqlite3_open_v2(filename, ppDb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI, NULL); +} + SWITCH_DECLARE(int) switch_core_db_close(switch_core_db_t *db) { return sqlite3_close(db); @@ -84,6 +89,7 @@ SWITCH_DECLARE(int) switch_core_db_exec(switch_core_db_t *db, const char *sql, s while (--sane > 0) { ret = sqlite3_exec(db, sql, callback, data, &err); if (ret == SQLITE_BUSY || ret == SQLITE_LOCKED) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "SQLite is %s, sane=%d [%s]\n", (ret == SQLITE_BUSY ? "BUSY" : "LOCKED"), sane, sql); if (sane > 1) { switch_core_db_free(err); switch_yield(100000); @@ -187,6 +193,35 @@ SWITCH_DECLARE(int) switch_core_db_load_extension(switch_core_db_t *db, const ch return ret; } +static int switch_core_db_connection_setup(switch_core_db_t *db, switch_bool_t in_memory) { + int db_ret; + + if ((db_ret = switch_core_db_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, NULL)) != SQLITE_OK) { + goto end; + } + if ((db_ret = switch_core_db_exec(db, "PRAGMA count_changes=OFF;", NULL, NULL, NULL)) != SQLITE_OK) { + goto end; + } + if ((db_ret = switch_core_db_exec(db, "PRAGMA temp_store=MEMORY;", NULL, NULL, NULL)) != SQLITE_OK) { + goto end; + } + if (!in_memory) { + if ((db_ret = switch_core_db_exec(db, "PRAGMA cache_size=8000;", NULL, NULL, NULL)) != SQLITE_OK) { + goto end; + } + } else { + if ((db_ret = switch_core_db_exec(db, "PRAGMA cache_size=-8192;", NULL, NULL, NULL)) != SQLITE_OK) { + goto end; + } + if ((db_ret = switch_core_db_exec(db, "PRAGMA journal_mode=OFF;", NULL, NULL, NULL)) != SQLITE_OK) { + goto end; + } + } + +end: + return db_ret; +} + SWITCH_DECLARE(switch_core_db_t *) switch_core_db_open_file(const char *filename) { switch_core_db_t *db; @@ -197,16 +232,28 @@ SWITCH_DECLARE(switch_core_db_t *) switch_core_db_open_file(const char *filename if ((db_ret = switch_core_db_open(path, &db)) != SQLITE_OK) { goto end; } - if ((db_ret = switch_core_db_exec(db, "PRAGMA synchronous=OFF;", NULL, NULL, NULL)) != SQLITE_OK) { + if ((db_ret = switch_core_db_connection_setup(db, SWITCH_FALSE)) != SQLITE_OK) { goto end; } - if ((db_ret = switch_core_db_exec(db, "PRAGMA count_changes=OFF;", NULL, NULL, NULL)) != SQLITE_OK) { + +end: + if (db_ret != SQLITE_OK) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "SQL ERR [%s]\n", switch_core_db_errmsg(db)); + switch_core_db_close(db); + db = NULL; + } + return db; +} + +SWITCH_DECLARE(switch_core_db_t *) switch_core_db_open_in_memory(const char *uri) +{ + switch_core_db_t *db; + int db_ret; + + if ((db_ret = switch_core_db_open_v2(uri, &db)) != SQLITE_OK) { goto end; } - if ((db_ret = switch_core_db_exec(db, "PRAGMA cache_size=8000;", NULL, NULL, NULL)) != SQLITE_OK) { - goto end; - } - if ((db_ret = switch_core_db_exec(db, "PRAGMA temp_store=MEMORY;", NULL, NULL, NULL)) != SQLITE_OK) { + if ((db_ret = switch_core_db_connection_setup(db, SWITCH_TRUE)) != SQLITE_OK) { goto end; } diff --git a/src/switch_core_sqldb.c b/src/switch_core_sqldb.c index ffc82aa2e5..b2ba795115 100644 --- a/src/switch_core_sqldb.c +++ b/src/switch_core_sqldb.c @@ -95,6 +95,14 @@ static switch_cache_db_handle_t *create_handle(switch_cache_db_handle_type_t typ return new_dbh; } +static void destroy_handle(switch_cache_db_handle_t **dbh) +{ + if (dbh && *dbh && (*dbh)->pool) { + switch_core_destroy_memory_pool(&(*dbh)->pool); + *dbh = NULL; + } +} + static void add_handle(switch_cache_db_handle_t *dbh, const char *db_str, const char *db_callsite_str, const char *thread_str) { switch_ssize_t hlen = -1; @@ -161,7 +169,7 @@ SWITCH_DECLARE(void) switch_cache_db_database_interface_flush_handles(switch_dat del_handle(dbh_ptr); switch_mutex_unlock(dbh_ptr->mutex); - switch_core_destroy_memory_pool(&dbh_ptr->pool); + destroy_handle(&dbh_ptr); } } @@ -266,7 +274,7 @@ static void sql_close(time_t prune) diff = (time_t) prune - dbh->last_used; } - if (prune > 0 && (dbh->use_count || (diff < SQL_CACHE_TIMEOUT && !switch_test_flag(dbh, CDF_PRUNE)))) { + if (prune > 0 && (dbh->use_count || switch_test_flag(dbh, CDF_NONEXPIRING) || (diff < SQL_CACHE_TIMEOUT && !switch_test_flag(dbh, CDF_PRUNE)))) { continue; } @@ -287,8 +295,8 @@ static void sql_close(time_t prune) break; case SCDB_TYPE_CORE_DB: { - switch_core_db_close(dbh->native_handle.core_db_dbh); - dbh->native_handle.core_db_dbh = NULL; + switch_core_db_close(dbh->native_handle.core_db_dbh->handle); + dbh->native_handle.core_db_dbh->handle = NULL; } break; } @@ -453,6 +461,10 @@ SWITCH_DECLARE(switch_status_t) _switch_cache_db_get_db_handle_dsn_ex(switch_cac if (!strncasecmp(dsn, "sqlite://", 9)) { type = SCDB_TYPE_CORE_DB; connection_options.core_db_options.db_path = (char *)(dsn + 9); + if (!strncasecmp(connection_options.core_db_options.db_path, "memory://", 9)) { + connection_options.core_db_options.in_memory = SWITCH_TRUE; + connection_options.core_db_options.db_path = (char *)(connection_options.core_db_options.db_path + 9); + } } else if ((!(i = strncasecmp(dsn, "odbc://", 7))) || (strchr(dsn + 2, ':') && !colon_slashes)) { type = SCDB_TYPE_ODBC; @@ -615,7 +627,11 @@ SWITCH_DECLARE(switch_status_t) _switch_cache_db_get_db_handle(switch_cache_db_h break; case SCDB_TYPE_CORE_DB: { - db = switch_core_db_open_file(connection_options->core_db_options.db_path); + if (!connection_options->core_db_options.in_memory) { + db = switch_core_db_open_file(connection_options->core_db_options.db_path); + } else { + db = switch_core_db_open_in_memory(connection_options->core_db_options.db_path); + } } break; @@ -636,7 +652,13 @@ SWITCH_DECLARE(switch_status_t) _switch_cache_db_get_db_handle(switch_cache_db_h if (database_interface_dbh) { new_dbh->native_handle.database_interface_dbh = database_interface_dbh; } else if (db) { - new_dbh->native_handle.core_db_dbh = db; + if (!(new_dbh->native_handle.core_db_dbh = switch_core_alloc(new_dbh->pool, sizeof(*new_dbh->native_handle.core_db_dbh)))) { + destroy_handle(&new_dbh); + switch_core_db_close(db); + goto end; + } + new_dbh->native_handle.core_db_dbh->handle = db; + new_dbh->native_handle.core_db_dbh->in_memory = connection_options->core_db_options.in_memory; } else if (odbc_dbh) { new_dbh->native_handle.odbc_dbh = odbc_dbh; } @@ -686,7 +708,7 @@ static switch_status_t switch_cache_db_execute_sql_real(switch_cache_db_handle_t break; case SCDB_TYPE_CORE_DB: { - int ret = switch_core_db_exec(dbh->native_handle.core_db_dbh, sql, NULL, NULL, &errmsg); + int ret = switch_core_db_exec(dbh->native_handle.core_db_dbh->handle, sql, NULL, NULL, &errmsg); type = "NATIVE"; if (ret == SWITCH_CORE_DB_OK) { @@ -810,7 +832,7 @@ SWITCH_DECLARE(int) switch_cache_db_affected_rows(switch_cache_db_handle_t *dbh) switch (dbh->type) { case SCDB_TYPE_CORE_DB: { - return switch_core_db_changes(dbh->native_handle.core_db_dbh); + return switch_core_db_changes(dbh->native_handle.core_db_dbh->handle); } break; case SCDB_TYPE_ODBC: @@ -836,7 +858,7 @@ SWITCH_DECLARE(int) switch_cache_db_load_extension(switch_cache_db_handle_t *dbh case SCDB_TYPE_CORE_DB: { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "try to load extension [%s]!\n", extension); - return switch_core_db_load_extension(dbh->native_handle.core_db_dbh, extension); + return switch_core_db_load_extension(dbh->native_handle.core_db_dbh->handle, extension); } break; case SCDB_TYPE_ODBC: @@ -868,7 +890,7 @@ SWITCH_DECLARE(char *) switch_cache_db_execute_sql2str(switch_cache_db_handle_t { switch_core_db_stmt_t *stmt; - if (switch_core_db_prepare(dbh->native_handle.core_db_dbh, sql, -1, &stmt, 0)) { + if (switch_core_db_prepare(dbh->native_handle.core_db_dbh->handle, sql, -1, &stmt, 0)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Statement Error [%s]!\n", sql); goto end; } else { @@ -1206,7 +1228,7 @@ SWITCH_DECLARE(switch_status_t) switch_cache_db_execute_sql_event_callback(switc break; case SCDB_TYPE_CORE_DB: { - int ret = switch_core_db_exec(dbh->native_handle.core_db_dbh, sql, helper_callback, &h, &errmsg); + int ret = switch_core_db_exec(dbh->native_handle.core_db_dbh->handle, sql, helper_callback, &h, &errmsg); if (ret == SWITCH_CORE_DB_OK || ret == SWITCH_CORE_DB_ABORT) { status = SWITCH_STATUS_SUCCESS; @@ -1273,7 +1295,7 @@ SWITCH_DECLARE(switch_status_t) switch_cache_db_execute_sql_event_callback_err(s break; case SCDB_TYPE_CORE_DB: { - int ret = switch_core_db_exec(dbh->native_handle.core_db_dbh, sql, helper_callback, &h, &errmsg); + int ret = switch_core_db_exec(dbh->native_handle.core_db_dbh->handle, sql, helper_callback, &h, &errmsg); if (ret == SWITCH_CORE_DB_OK || ret == SWITCH_CORE_DB_ABORT) { status = SWITCH_STATUS_SUCCESS; @@ -1332,7 +1354,7 @@ SWITCH_DECLARE(switch_status_t) switch_cache_db_execute_sql_callback(switch_cach break; case SCDB_TYPE_CORE_DB: { - int ret = switch_core_db_exec(dbh->native_handle.core_db_dbh, sql, callback, pdata, &errmsg); + int ret = switch_core_db_exec(dbh->native_handle.core_db_dbh->handle, sql, callback, pdata, &errmsg); if (ret == SWITCH_CORE_DB_OK || ret == SWITCH_CORE_DB_ABORT) { status = SWITCH_STATUS_SUCCESS; @@ -1394,7 +1416,7 @@ SWITCH_DECLARE(switch_status_t) switch_cache_db_execute_sql_callback_err(switch_ break; case SCDB_TYPE_CORE_DB: { - int ret = switch_core_db_exec(dbh->native_handle.core_db_dbh, sql, callback, pdata, &errmsg); + int ret = switch_core_db_exec(dbh->native_handle.core_db_dbh->handle, sql, callback, pdata, &errmsg); if (ret == SWITCH_CORE_DB_OK || ret == SWITCH_CORE_DB_ABORT) { status = SWITCH_STATUS_SUCCESS; @@ -1521,21 +1543,21 @@ SWITCH_DECLARE(switch_bool_t) switch_cache_db_test_reactive_ex(switch_cache_db_h case SCDB_TYPE_CORE_DB: { char *errmsg = NULL; - switch_core_db_exec(dbh->native_handle.core_db_dbh, test_sql, NULL, NULL, &errmsg); + switch_core_db_exec(dbh->native_handle.core_db_dbh->handle, test_sql, NULL, NULL, &errmsg); if (errmsg) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "SQL ERR [%s]\n[%s]\nAuto Generating Table!\n", errmsg, test_sql); switch_core_db_free(errmsg); errmsg = NULL; if (drop_sql) { - switch_core_db_exec(dbh->native_handle.core_db_dbh, drop_sql, NULL, NULL, &errmsg); + switch_core_db_exec(dbh->native_handle.core_db_dbh->handle, drop_sql, NULL, NULL, &errmsg); } if (errmsg) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Ignoring SQL ERR [%s]\n[%s]\n", errmsg, drop_sql); switch_core_db_free(errmsg); errmsg = NULL; } - switch_core_db_exec(dbh->native_handle.core_db_dbh, reactive_sql, NULL, NULL, &errmsg); + switch_core_db_exec(dbh->native_handle.core_db_dbh->handle, reactive_sql, NULL, NULL, &errmsg); if (errmsg) { r = SWITCH_FALSE; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "SQL ERR [%s]\n[%s]\n", errmsg, reactive_sql); @@ -3779,6 +3801,10 @@ switch_status_t switch_core_sqldb_start(switch_memory_pool_t *pool, switch_bool_ switch_cache_db_execute_sql(sql_manager.dbh, create_tasks_sql, NULL); switch_cache_db_execute_sql(sql_manager.dbh, detailed_calls_sql, NULL); switch_cache_db_execute_sql(sql_manager.dbh, basic_calls_sql, NULL); + + if (sql_manager.dbh->native_handle.core_db_dbh->in_memory == SWITCH_TRUE) { + switch_set_flag(sql_manager.dbh, CDF_NONEXPIRING); + } } break; } @@ -4009,14 +4035,14 @@ SWITCH_DECLARE(void) switch_cache_db_status(switch_stream_handle_t *stream) used++; } - stream->write_function(stream, "%s\n\tType: %s\n\tLast used: %d\n\tTotal used: %ld\n\tFlags: %s, %s(%d)\n" + stream->write_function(stream, "%s\n\tType: %s\n\tLast used: %d\n\tTotal used: %ld\n\tFlags: %s, %s(%d)%s\n" "\tCreator: %s\n\tLast User: %s\n", cleankey_str, switch_cache_db_type_name(dbh->type), diff, dbh->total_used_count, locked ? "Locked" : "Unlocked", - dbh->use_count ? "Attached" : "Detached", dbh->use_count, dbh->creator, dbh->last_user); + dbh->use_count ? "Attached" : "Detached", dbh->use_count, switch_test_flag(dbh, CDF_NONEXPIRING) ? ", Non-expiring" : "", dbh->creator, dbh->last_user); } stream->write_function(stream, "%d total. %d in use.\n", count, used);