Replace Berkeley DB with SQLite 3

There were some bugs in the very ancient version of Berkeley DB that Asterisk
used. Instead of spending the time tracking down the bugs in the Berkeley code
we move to the much better documented SQLite 3.

Conversion of the old astdb happens at runtime by running the included
astdb2sqlite3 utility. The ast_db API with SQLite 3 backend should behave
identically to the old Berkeley backend, but in the future we could offer a
much more robust interface.

We do not include the SQLite 3 library in the source tree, but instead rely
upon the distribution-provided libraries. SQLite is so ubiquitous that this
should not place undue burden on administrators.


git-svn-id: http://svn.digium.com/svn/asterisk/trunk@326589 f38db490-d61c-443f-a65b-d21fe96a405b

CHANGES

@@ -188,6 +188,14 @@ Applications
  * Added ability to include '@parkinglot' to ParkedCall extension in order to specify
    a specific parkinglot on which to search the extension.
 
+Asterisk Database
+-----------------
+ * The internal Asterisk database has been switched from Berkeley DB 1.86 to
+   SQLite 3. An existing Berkeley astdb file can be converted with the astdb2sqlite3
+   utility in the UTILS section of menuselect. If an existing astdb is found and no
+   astdb.sqlite3 exists, astdb2sqlite3 will be compiled automatically. Asterisk will
+   convert an existing astdb to the SQLite3 version automatically at runtime.
+
 ------------------------------------------------------------------------------
 --- Functionality changes from Asterisk 1.6.2 to Asterisk 1.8 ----------------
 ------------------------------------------------------------------------------

Makefile

@@ -838,6 +838,7 @@ sounds:
 
 cleantest:
 	@cmp -s .cleancount .lastclean || $(MAKE) clean
+	@[ -f "$(DESTDIR)$(ASTDBDIR)/astdb.sqlite3" ] || [ ! -f "$(DESTDIR)$(ASTDBDIR)/astdb" ] || [ ! -f menuselect.makeopts ] || grep -q MENUSELECT_UTILS=.*astdb2sqlite3 menuselect.makeopts || (sed -i.orig -e's/MENUSELECT_UTILS=\(.*\)/MENUSELECT_UTILS=\1 astdb2sqlite3/' menuselect.makeopts && echo "Updating menuselect.makeopts to include astdb2sqlite3" && echo "Original version backed up to menuselect.makeopts.orig")
 
 $(SUBDIRS_UNINSTALL):
 	+@$(SUBMAKE) -C $(@:-uninstall=) uninstall

UPGRADE.txt

@@ -54,5 +54,12 @@ Queue:
  - Mark QUEUE_MEMBER_PENALTY Deprecated it never worked for realtime members
  - QUEUE_MEMBER is now R/W supporting setting paused, ignorebusy and penalty.
 
+ Asterisk Database:
+ - The internal Asterisk database has been switched from Berkeley DB 1.86 to
+   SQLite 3. An existing Berkeley astdb file can be converted with the astdb2sqlite3
+   utility in the UTILS section of menuselect. If an existing astdb is found and no
+   astdb.sqlite3 exists, astdb2sqlite3 will be compiled automatically. Asterisk will
+   convert an existing astdb to the SQLite3 version automatically at runtime.
+
 ===========================================================
 ===========================================================

configure.ac

@@ -1970,6 +1970,12 @@ AST_EXT_LIB_CHECK([SQLITE], [sqlite], [sqlite_exec], [sqlite.h])
 
 AST_EXT_LIB_CHECK([SQLITE3], [sqlite3], [sqlite3_open], [sqlite3.h], [${PTHREAD_LIBS}], [${PTHREAD_CFLAGS}])
 
+if test "${PBX_SQLITE3}" != 1; then
+	AC_MSG_WARN(*** Asterisk now uses SQLite3 for the internal Asterisk database.)
+	AC_MSG_WARN(*** Please install the SQLite3 development package.)
+	exit 1
+fi
+
 AST_EXT_LIB_CHECK([CRYPTO], [crypto], [AES_encrypt], [openssl/aes.h])
 
 if test "$PBX_CRYPTO" = "1";

main/Makefile

@@ -33,6 +33,7 @@ endif
 AST_LIBS += $(OPENSSL_LIB)
 AST_LIBS += $(BKTR_LIB)
 AST_LIBS += $(LIBXML2_LIB) 
+AST_LIBS += $(SQLITE3_LIB)
 
 ifneq ($(findstring $(OSARCH), linux-gnu uclinux linux-uclibc linux-gnueabi kfreebsd-gnu),)
   ifneq ($(findstring LOADABLE_MODULES,$(MENUSELECT_CFLAGS)),)
@@ -111,9 +112,6 @@ editline/libedit.a: CHECK_SUBDIR
 	cd editline && test -f config.h || CFLAGS="$(PTHREAD_CFLAGS) $(subst $(ASTTOPDIR),../../,$(_ASTCFLAGS:-Werror=) $(ASTCFLAGS))" LDFLAGS="$(_ASTLDFLAGS) $(ASTLDFLAGS)" ./configure --build=$(BUILD_PLATFORM) --host=$(HOST_PLATFORM) --with-ncurses=$(NCURSES_DIR) --with-curses=$(CURSES_DIR) --with-termcap=$(TERMCAP_DIR) --with-tinfo=$(TINFO_DIR)
 	$(MAKE) -C editline libedit.a
 
-db1-ast/libdb1.a: CHECK_SUBDIR
-	_ASTCFLAGS="$(_ASTCFLAGS) -Wno-strict-aliasing" ASTCFLAGS="$(ASTCFLAGS)" $(MAKE) -C db1-ast libdb1.a
-
 ifneq ($(findstring REBUILD_PARSERS,$(MENUSELECT_CFLAGS)),)
 ast_expr2.c ast_expr2.h: ast_expr2.y
 else
@@ -143,6 +141,8 @@ testexpr2: ast_expr2f.c ast_expr2.c ast_expr2.h
 	$(CC) -g -o testexpr2 ast_expr2f.o ast_expr2.o -lm
 	rm ast_expr2.o ast_expr2f.o 
 
+db.o: _ASTCFLAGS+=$(SQLITE3_INCLUDE)
+
 ifneq ($(findstring ENABLE_UPLOADS,$(MENUSELECT_CFLAGS)),)
 http.o: _ASTCFLAGS+=$(GMIME_INCLUDE)
 endif
@@ -177,13 +177,13 @@ endif
 
 $(OBJS): _ASTCFLAGS+=-DAST_MODULE=\"core\"
 
-$(MAIN_TGT): $(OBJS) editline/libedit.a db1-ast/libdb1.a $(AST_EMBED_LDSCRIPTS)
+$(MAIN_TGT): $(OBJS) editline/libedit.a $(AST_EMBED_LDSCRIPTS)
 	@$(CC) -c -o buildinfo.o $(_ASTCFLAGS) buildinfo.c $(ASTCFLAGS)
-	$(ECHO_PREFIX) echo "   [LD] $(OBJS) editline/libedit.a db1-ast/libdb1.a $(AST_EMBED_LDSCRIPTS) -> $@"
+	$(ECHO_PREFIX) echo "   [LD] $(OBJS) editline/libedit.a $(AST_EMBED_LDSCRIPTS) -> $@"
 ifneq ($(findstring chan_h323,$(MENUSELECT_CHANNELS)),)
-	$(CMD_PREFIX) $(CC) $(STATIC_BUILD) -o $@ $(ASTLINK) $(AST_EMBED_LDFLAGS) $(_ASTLDFLAGS) $(ASTLDFLAGS) $(OBJS) editline/libedit.a db1-ast/libdb1.a $(AST_EMBED_LDSCRIPTS) buildinfo.o $(AST_LIBS) $(AST_EMBED_LIBS) $(GMIMELDFLAGS)
+	$(CMD_PREFIX) $(CC) $(STATIC_BUILD) -o $@ $(ASTLINK) $(AST_EMBED_LDFLAGS) $(_ASTLDFLAGS) $(ASTLDFLAGS) $(OBJS) editline/libedit.a $(AST_EMBED_LDSCRIPTS) buildinfo.o $(AST_LIBS) $(AST_EMBED_LIBS) $(GMIMELDFLAGS)
 else
-	$(CMD_PREFIX) $(CXX) $(STATIC_BUILD) -o $@ $(ASTLINK) $(AST_EMBED_LDFLAGS) $(_ASTLDFLAGS) $(ASTLDFLAGS) $(H323LDFLAGS) $(OBJS) editline/libedit.a db1-ast/libdb1.a $(AST_EMBED_LDSCRIPTS) buildinfo.o $(AST_LIBS) $(AST_EMBED_LIBS) $(H323LDLIBS) $(GMIMELDFLAGS)
+	$(CMD_PREFIX) $(CXX) $(STATIC_BUILD) -o $@ $(ASTLINK) $(AST_EMBED_LDFLAGS) $(_ASTLDFLAGS) $(ASTLDFLAGS) $(H323LDFLAGS) $(OBJS) editline/libedit.a $(AST_EMBED_LDSCRIPTS) buildinfo.o $(AST_LIBS) $(AST_EMBED_LIBS) $(H323LDLIBS) $(GMIMELDFLAGS)
 endif
 
 ifeq ($(GNU_LD),1)
@@ -194,9 +194,7 @@ endif
 
 clean::
 	rm -f asterisk
-	rm -f db1-ast/.*.d
 	rm -f asterisk.exports
 	@if [ -f editline/Makefile ]; then $(MAKE) -C editline distclean ; fi
-	@$(MAKE) -C db1-ast clean
 	@$(MAKE) -C stdtime clean
 	rm -f libresample/src/*.o

main/asterisk.c

@@ -3759,6 +3759,11 @@ int main(int argc, char *argv[])
 	ast_xmldoc_load_documentation();
 #endif
 
+	if (astdb_init()) {
+		printf("%s", term_quit());
+		exit(1);
+	}
+
 	if (ast_msg_init()) {
 		printf("%s", term_quit());
 		exit(1);
@@ -3834,11 +3839,6 @@ int main(int argc, char *argv[])
 		exit(1);
 	}
 
-	if (astdb_init()) {
-		printf("%s", term_quit());
-		exit(1);
-	}
-
 	if (ast_enum_init()) {
 		printf("%s", term_quit());
 		exit(1);

main/db.c

@@ -20,11 +20,11 @@
  *
  * \brief ASTdb Management
  *
- * \author Mark Spencer <markster@digium.com> 
+ * \author Mark Spencer <markster@digium.com>
  *
  * \note DB3 is licensed under Sleepycat Public License and is thus incompatible
- * with GPL.  To avoid having to make another exception (and complicate 
- * licensing even further) we elect to use DB1 which is BSD licensed 
+ * with GPL.  To avoid having to make another exception (and complicate
+ * licensing even further) we elect to use DB1 which is BSD licensed
  */
 
 #include "asterisk.h"
@@ -34,8 +34,12 @@ ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
 #include "asterisk/_private.h"
 #include "asterisk/paths.h"	/* use ast_config_AST_DB */
 #include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
 #include <signal.h>
 #include <dirent.h>
+#include <sqlite3.h>
 
 #include "asterisk/channel.h"
 #include "asterisk/file.h"
@@ -44,9 +48,7 @@ ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
 #include "asterisk/astdb.h"
 #include "asterisk/cli.h"
 #include "asterisk/utils.h"
-#include "asterisk/lock.h"
 #include "asterisk/manager.h"
-#include "db1-ast/include/db.h"
 
 /*** DOCUMENTATION
 	<manager name="DBGet" language="en_US">
@@ -101,242 +103,362 @@ ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
  ***/
 
 #define MAX_DB_FIELD 256
-
-static DB *astdb;
 AST_MUTEX_DEFINE_STATIC(dblock);
 static ast_cond_t dbcond;
-typedef int (*process_keys_cb)(DBT *key, DBT *value, const char *filter, void *data);
+static sqlite3 *astdb;
+static pthread_t syncthread;
+static int doexit;
 
 static void db_sync(void);
 
-static int dbinit(void) 
+#define DEFINE_SQL_STATEMENT(stmt,sql) static sqlite3_stmt *stmt; \
+	const char stmt##_sql[] = sql;
+
+DEFINE_SQL_STATEMENT(put_stmt, "INSERT OR REPLACE INTO astdb (key, value) VALUES (?, ?)")
+DEFINE_SQL_STATEMENT(get_stmt, "SELECT value FROM astdb WHERE key=?")
+DEFINE_SQL_STATEMENT(del_stmt, "DELETE FROM astdb WHERE key=?")
+DEFINE_SQL_STATEMENT(deltree_stmt, "DELETE FROM astdb WHERE key LIKE ? || '/' || '%'")
+DEFINE_SQL_STATEMENT(deltree_all_stmt, "DELETE FROM astdb")
+DEFINE_SQL_STATEMENT(gettree_stmt, "SELECT key, value FROM astdb WHERE key LIKE ? || '/' || '%'")
+DEFINE_SQL_STATEMENT(gettree_all_stmt, "SELECT key, value FROM astdb")
+DEFINE_SQL_STATEMENT(showkey_stmt, "SELECT key, value FROM astdb WHERE key LIKE '%' || '/' || ?")
+DEFINE_SQL_STATEMENT(create_astdb_stmt, "CREATE TABLE IF NOT EXISTS astdb(key VARCHAR(256), value VARCHAR(256), PRIMARY KEY(key))")
+
+static int init_stmt(sqlite3_stmt **stmt, const char *sql, size_t len)
 {
-	if (!astdb && !(astdb = dbopen(ast_config_AST_DB, O_CREAT | O_RDWR, AST_FILE_MODE, DB_BTREE, NULL))) {
-		ast_log(LOG_WARNING, "Unable to open Asterisk database '%s': %s\n", ast_config_AST_DB, strerror(errno));
-		return -1;
-	}
-	return 0;
-}
-
-
-static inline int keymatch(const char *key, const char *prefix)
-{
-	int preflen = strlen(prefix);
-	if (!preflen)
-		return 1;
-	if (!strcasecmp(key, prefix))
-		return 1;
-	if ((strlen(key) > preflen) && !strncasecmp(key, prefix, preflen)) {
-		if (key[preflen] == '/')
-			return 1;
-	}
-	return 0;
-}
-
-static inline int subkeymatch(const char *key, const char *suffix)
-{
-	int suffixlen = strlen(suffix);
-	if (suffixlen) {
-		const char *subkey = key + strlen(key) - suffixlen;
-		if (subkey < key)
-			return 0;
-		if (!strcasecmp(subkey, suffix))
-			return 1;
-	}
-	return 0;
-}
-
-static const char *dbt_data2str(DBT *dbt)
-{
-	char *data = "";
-
-	if (dbt->size) {
-		data = dbt->data;
-		data[dbt->size - 1] = '\0';
-	}
-
-	return data;
-}
-
-static inline const char *dbt_data2str_full(DBT *dbt, const char *def)
-{
-	return S_OR(dbt_data2str(dbt), def);
-}
-
-static int process_db_keys(process_keys_cb cb, void *data, const char *filter, int sync)
-{
-	DBT key = { 0, }, value = { 0, }, last_key = { 0, };
-	int counter = 0;
-	int res, last = 0;
-	char last_key_s[MAX_DB_FIELD];
-
 	ast_mutex_lock(&dblock);
-	if (dbinit()) {
+	if (sqlite3_prepare_v2(astdb, sql, len, stmt, NULL) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Couldn't prepare statement '%s': %s\n", sql, sqlite3_errmsg(astdb));
 		ast_mutex_unlock(&dblock);
 		return -1;
 	}
-
-	/* Somehow, the database can become corrupted such that astdb->seq will continue looping through
-	 * the database indefinitely. The pointer to last_key.data ends up getting re-used by the BDB lib
-	 * so this specifically queries for the last entry, makes a copy of the key, and then uses it as
-	 * a sentinel to avoid repeatedly looping over the list. */
-
-	if (astdb->seq(astdb, &last_key, &value, R_LAST)) {
-		/* Empty database */
-		ast_mutex_unlock(&dblock);
-		return 0;
-	}
-
-	memcpy(last_key_s, last_key.data, MIN(last_key.size - 1, sizeof(last_key_s)));
-	last_key_s[last_key.size - 1] = '\0';
-	for (res = astdb->seq(astdb, &key, &value, R_FIRST);
-			!res;
-			res = astdb->seq(astdb, &key, &value, R_NEXT)) {
-		/* The callback might delete the key, so we have to check it before calling */
-		last = !strcmp(dbt_data2str_full(&key, "<bad key>"), last_key_s);
-		counter += cb(&key, &value, filter, data);
-		if (last) {
-			break;
-		}
-	}
-
-	if (sync) {
-		db_sync();
-	}
-
 	ast_mutex_unlock(&dblock);
 
-	return counter;
+	return 0;
 }
 
-static int db_deltree_cb(DBT *key, DBT *value, const char *filter, void *data)
+static int init_statements(void)
+{
+	/* Don't initialize create_astdb_statment here as the astdb table needs to exist
+	 * brefore these statments can be initialized */
+	return init_stmt(&get_stmt, get_stmt_sql, sizeof(get_stmt_sql))
+	|| init_stmt(&del_stmt, del_stmt_sql, sizeof(del_stmt_sql))
+	|| init_stmt(&deltree_stmt, deltree_stmt_sql, sizeof(deltree_stmt_sql))
+	|| init_stmt(&deltree_all_stmt, deltree_all_stmt_sql, sizeof(deltree_all_stmt_sql))
+	|| init_stmt(&gettree_stmt, gettree_stmt_sql, sizeof(gettree_stmt_sql))
+	|| init_stmt(&gettree_all_stmt, gettree_all_stmt_sql, sizeof(gettree_all_stmt_sql))
+	|| init_stmt(&showkey_stmt, showkey_stmt_sql, sizeof(showkey_stmt_sql))
+	|| init_stmt(&put_stmt, put_stmt_sql, sizeof(put_stmt_sql));
+}
+
+static int convert_bdb_to_sqlite3(void)
+{
+	char *cmd;
+	int res;
+
+	ast_asprintf(&cmd, "astdb2sqlite3 '%s'\n", ast_config_AST_DB);
+	res = ast_safe_system(cmd);
+	ast_free(cmd);
+
+	return res;
+}
+
+static int db_create_astdb(void)
 {
 	int res = 0;
 
-	if (keymatch(dbt_data2str_full(key, "<bad key>"), filter)) {
-		astdb->del(astdb, key, 0);
-		res = 1;
+	if (!create_astdb_stmt) {
+		init_stmt(&create_astdb_stmt, create_astdb_stmt_sql, sizeof(create_astdb_stmt_sql));
 	}
+
+	ast_mutex_lock(&dblock);
+	if (sqlite3_step(create_astdb_stmt) != SQLITE_DONE) {
+		ast_log(LOG_WARNING, "Couldn't create astdb table: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	}
+	sqlite3_reset(create_astdb_stmt);
+	db_sync();
+	ast_mutex_unlock(&dblock);
+
 	return res;
 }
 
-int ast_db_deltree(const char *family, const char *keytree)
+static int db_open(void)
 {
-	char prefix[MAX_DB_FIELD];
+	char *dbname;
+	struct stat dont_care;
 
-	if (family) {
-		if (keytree) {
-			snprintf(prefix, sizeof(prefix), "/%s/%s", family, keytree);
+	if (!(dbname = alloca(strlen(ast_config_AST_DB) + sizeof(".sqlite3")))) {
+		return -1;
+	}
+	strcpy(dbname, ast_config_AST_DB);
+	strcat(dbname, ".sqlite3");
+
+	if (stat(dbname, &dont_care) && !stat(ast_config_AST_DB, &dont_care)) {
+		if (convert_bdb_to_sqlite3()) {
+			ast_log(LOG_ERROR, "*** Database conversion failed!\n");
+			ast_log(LOG_ERROR, "*** Asterisk now uses SQLite3 for its internal\n");
+			ast_log(LOG_ERROR, "*** database. Conversion from the old astdb\n");
+			ast_log(LOG_ERROR, "*** failed. Most likely the astdb2sqlite3 utility\n");
+			ast_log(LOG_ERROR, "*** was not selected for build. To convert the\n");
+			ast_log(LOG_ERROR, "*** old astdb, please delete '%s'\n", dbname);
+			ast_log(LOG_ERROR, "*** and re-run 'make menuselect' and select astdb2sqlite3\n");
+			ast_log(LOG_ERROR, "*** in the Utilities section, then 'make && make install'.\n");
+			sleep(5);
 		} else {
+			ast_log(LOG_NOTICE, "Database conversion succeeded!\n");
+		}
+	}
+
+	ast_mutex_lock(&dblock);
+	if (sqlite3_open_v2(dbname, &astdb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Unable to open Asterisk database '%s': %s\n", dbname, sqlite3_errmsg(astdb));
+		sqlite3_close(astdb);
+		ast_mutex_unlock(&dblock);
+		return -1;
+	}
+	ast_mutex_unlock(&dblock);
+
+	return 0;
+}
+
+static int db_init(void)
+{
+	if (astdb) {
+		return 0;
+	}
+
+	if (db_open() || db_create_astdb() || init_statements()) {
+		return -1;
+	}
+
+	return 0;
+}
+
+/* We purposely don't lock around the sqlite3 call because the transaction
+ * calls will be called with the database lock held. For any other use, make
+ * sure to take the dblock yourself. */
+static int db_execute_sql(const char *sql, int (*callback)(void *, int, char **, char **), void *arg)
+{
+	char *errmsg = NULL;
+	int res =0;
+
+	sqlite3_exec(astdb, sql, callback, arg, &errmsg);
+	if (errmsg) {
+		ast_log(LOG_WARNING, "Error executing SQL: %s\n", errmsg);
+		sqlite3_free(errmsg);
+		res = -1;
+	}
+
+	return res;
+}
+
+static int ast_db_begin_transaction(void)
+{
+	return db_execute_sql("BEGIN TRANSACTION", NULL, NULL);
+}
+
+static int ast_db_commit_transaction(void)
+{
+	return db_execute_sql("COMMIT", NULL, NULL);
+}
+
+static int ast_db_rollback_transaction(void)
+{
+	return db_execute_sql("ROLLBACK", NULL, NULL);
+}
+
+int ast_db_put(const char *family, const char *key, const char *value)
+{
+	char fullkey[MAX_DB_FIELD];
+	size_t fullkey_len;
+	int res = 0;
+
+	if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
+		ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
+		return -1;
+	}
+
+	fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
+
+	ast_mutex_lock(&dblock);
+	if (sqlite3_bind_text(put_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_bind_text(put_stmt, 2, value, -1, SQLITE_STATIC) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Couldn't bind value to stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_step(put_stmt) != SQLITE_DONE) {
+		ast_log(LOG_WARNING, "Couldn't execute statment: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	}
+
+	sqlite3_reset(put_stmt);
+	db_sync();
+	ast_mutex_unlock(&dblock);
+
+	return res;
+}
+
+int ast_db_get(const char *family, const char *key, char *value, int valuelen)
+{
+	const unsigned char *result;
+	char fullkey[MAX_DB_FIELD];
+	size_t fullkey_len;
+	int res = 0;
+
+	if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
+		ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
+		return -1;
+	}
+
+	fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
+
+	ast_mutex_lock(&dblock);
+	if (sqlite3_bind_text(get_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_step(get_stmt) != SQLITE_ROW) {
+		ast_debug(1, "Unable to find key '%s' in family '%s'\n", key, family);
+		res = -1;
+	} else if (!(result = sqlite3_column_text(get_stmt, 0))) {
+		ast_log(LOG_WARNING, "Couldn't get value\n");
+		res = -1;
+	} else {
+		strncpy(value, (const char *) result, valuelen);
+	}
+	sqlite3_reset(get_stmt);
+	ast_mutex_unlock(&dblock);
+
+	return res;
+}
+
+int ast_db_del(const char *family, const char *key)
+{
+	char fullkey[MAX_DB_FIELD];
+	size_t fullkey_len;
+	int res = 0;
+
+	if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
+		ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
+		return -1;
+	}
+
+	fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
+
+	ast_mutex_lock(&dblock);
+	if (sqlite3_bind_text(del_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
+		ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_step(del_stmt) != SQLITE_DONE) {
+		ast_debug(1, "Unable to find key '%s' in family '%s'\n", key, family);
+		res = -1;
+	}
+	sqlite3_reset(del_stmt);
+	db_sync();
+	ast_mutex_unlock(&dblock);
+
+	return res;
+}
+
+int ast_db_deltree(const char *family, const char *subfamily)
+{
+	sqlite3_stmt *stmt = deltree_stmt;
+	char prefix[MAX_DB_FIELD];
+	int res = 0;
+
+	if (!ast_strlen_zero(family)) {
+		if (!ast_strlen_zero(subfamily)) {
+			/* Family and key tree */
+			snprintf(prefix, sizeof(prefix), "/%s/%s", family, subfamily);
+		} else {
+			/* Family only */
 			snprintf(prefix, sizeof(prefix), "/%s", family);
 		}
-	} else if (keytree) {
-		return -1;
 	} else {
 		prefix[0] = '\0';
+		stmt = deltree_all_stmt;
 	}
 
-	return process_db_keys(db_deltree_cb, NULL, prefix, 1);
-}
-
-int ast_db_put(const char *family, const char *keys, const char *value)
-{
-	char fullkey[MAX_DB_FIELD];
-	DBT key, data;
-	int res, fullkeylen;
-
 	ast_mutex_lock(&dblock);
-	if (dbinit()) {
-		ast_mutex_unlock(&dblock);
-		return -1;
+	if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
+		ast_log(LOG_WARNING, "Could bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_step(stmt) != SQLITE_DONE) {
+		ast_log(LOG_WARNING, "Couldn't execute stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
 	}
-
-	fullkeylen = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, keys);
-	memset(&key, 0, sizeof(key));
-	memset(&data, 0, sizeof(data));
-	key.data = fullkey;
-	key.size = fullkeylen + 1;
-	data.data = (char *) value;
-	data.size = strlen(value) + 1;
-	res = astdb->put(astdb, &key, &data, 0);
+	res = sqlite3_changes(astdb);
+	sqlite3_reset(stmt);
 	db_sync();
 	ast_mutex_unlock(&dblock);
-	if (res)
-		ast_log(LOG_WARNING, "Unable to put value '%s' for key '%s' in family '%s'\n", value, keys, family);
 
 	return res;
 }
 
-int ast_db_get(const char *family, const char *keys, char *value, int valuelen)
+struct ast_db_entry *ast_db_gettree(const char *family, const char *subfamily)
 {
-	char fullkey[MAX_DB_FIELD] = "";
-	DBT key, data;
-	int res, fullkeylen;
+	char prefix[MAX_DB_FIELD];
+	sqlite3_stmt *stmt = gettree_stmt;
+	struct ast_db_entry *cur, *last = NULL, *ret = NULL;
 
-	ast_mutex_lock(&dblock);
-	if (dbinit()) {
-		ast_mutex_unlock(&dblock);
-		return -1;
-	}
-
-	fullkeylen = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, keys);
-	memset(&key, 0, sizeof(key));
-	memset(&data, 0, sizeof(data));
-	memset(value, 0, valuelen);
-	key.data = fullkey;
-	key.size = fullkeylen + 1;
-
-	res = astdb->get(astdb, &key, &data, 0);
-
-	/* Be sure to NULL terminate our data either way */
-	if (res) {
-		ast_debug(1, "Unable to find key '%s' in family '%s'\n", keys, family);
-	} else {
-#if 0
-		printf("Got value of size %d\n", data.size);
-#endif
-		if (data.size) {
-			((char *)data.data)[data.size - 1] = '\0';
-			/* Make sure that we don't write too much to the dst pointer or we don't read too much from the source pointer */
-			ast_copy_string(value, data.data, (valuelen > data.size) ? data.size : valuelen);
+	if (!ast_strlen_zero(family)) {
+		if (!ast_strlen_zero(subfamily)) {
+			/* Family and key tree */
+			snprintf(prefix, sizeof(prefix), "/%s/%s", family, subfamily);
 		} else {
-			ast_log(LOG_NOTICE, "Strange, empty value for /%s/%s\n", family, keys);
+			/* Family only */
+			snprintf(prefix, sizeof(prefix), "/%s", family);
 		}
+	} else {
+		prefix[0] = '\0';
+		stmt = gettree_all_stmt;
 	}
 
-	/* Data is not fully isolated for concurrency, so the lock must be extended
-	 * to after the copy to the output buffer. */
-	ast_mutex_unlock(&dblock);
-
-	return res;
-}
-
-int ast_db_del(const char *family, const char *keys)
-{
-	char fullkey[MAX_DB_FIELD];
-	DBT key;
-	int res, fullkeylen;
-
 	ast_mutex_lock(&dblock);
-	if (dbinit()) {
+	if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
+		ast_log(LOG_WARNING, "Could bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
+		sqlite3_reset(stmt);
 		ast_mutex_unlock(&dblock);
-		return -1;
+		return NULL;
 	}
-	
-	fullkeylen = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, keys);
-	memset(&key, 0, sizeof(key));
-	key.data = fullkey;
-	key.size = fullkeylen + 1;
-	
-	res = astdb->del(astdb, &key, 0);
-	db_sync();
-	
+
+	while (sqlite3_step(stmt) == SQLITE_ROW) {
+		const char *key_s, *value_s;
+		if (!(key_s = (const char *) sqlite3_column_text(stmt, 0))) {
+			break;
+		}
+		if (!(value_s = (const char *) sqlite3_column_text(stmt, 1))) {
+			break;
+		}
+		if (!(cur = ast_malloc(sizeof(*cur) + strlen(key_s) + strlen(value_s) + 2))) {
+			break;
+		}
+		cur->next = NULL;
+		cur->key = cur->data + strlen(value_s) + 1;
+		strcpy(cur->data, value_s);
+		strcpy(cur->key, key_s);
+		if (last) {
+			last->next = cur;
+		} else {
+			ret = cur;
+		}
+		last = cur;
+	}
+	sqlite3_reset(stmt);
 	ast_mutex_unlock(&dblock);
 
-	if (res) {
-		ast_debug(1, "Unable to find key '%s' in family '%s'\n", keys, family);
+	return ret;
+}
+
+void ast_db_freetree(struct ast_db_entry *dbe)
+{
+	struct ast_db_entry *last;
+	while (dbe) {
+		last = dbe;
+		dbe = dbe->next;
+		ast_free(last);
 	}
-	return res;
 }
 
 static char *handle_cli_database_put(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
@@ -429,8 +551,8 @@ static char *handle_cli_database_deltree(struct ast_cli_entry *e, int cmd, struc
 	case CLI_INIT:
 		e->command = "database deltree";
 		e->usage =
-			"Usage: database deltree <family> [keytree]\n"
-			"       Deletes a family or specific keytree within a family\n"
+			"Usage: database deltree <family> [subfamily]\n"
+			"       Deletes a family or specific subfamily within a family\n"
 			"       in the Asterisk database.\n";
 		return NULL;
 	case CLI_GENERATE:
@@ -452,32 +574,19 @@ static char *handle_cli_database_deltree(struct ast_cli_entry *e, int cmd, struc
 	return CLI_SUCCESS;
 }
 
-static int db_show_cb(DBT *key, DBT *value, const char *filter, void *data)
-{
-	struct ast_cli_args *a = data;
-	const char *key_s = dbt_data2str_full(key, "<bad key>");
-	const char *value_s = dbt_data2str_full(value, "<bad value>");
-
-	if (keymatch(key_s, filter)) {
-		ast_cli(a->fd, "%-50s: %-25s\n", key_s, value_s);
-		return 1;
-	}
-
-	return 0;
-}
-
 static char *handle_cli_database_show(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
 	char prefix[MAX_DB_FIELD];
 	int counter = 0;
+	sqlite3_stmt *stmt = gettree_stmt;
 
 	switch (cmd) {
 	case CLI_INIT:
 		e->command = "database show";
 		e->usage =
-			"Usage: database show [family [keytree]]\n"
+			"Usage: database show [family [subfamily]]\n"
 			"       Shows Asterisk database contents, optionally restricted\n"
-			"       to a given family, or family and keytree.\n";
+			"       to a given family, or family and subfamily.\n";
 		return NULL;
 	case CLI_GENERATE:
 		return NULL;
@@ -492,118 +601,123 @@ static char *handle_cli_database_show(struct ast_cli_entry *e, int cmd, struct a
 	} else if (a->argc == 2) {
 		/* Neither */
 		prefix[0] = '\0';
+		stmt = gettree_all_stmt;
+
 	} else {
 		return CLI_SHOWUSAGE;
 	}
 
-	if((counter = process_db_keys(db_show_cb, a, prefix, 0)) < 0) {
-		ast_cli(a->fd, "Database unavailable\n");
-		return CLI_SUCCESS;
+	ast_mutex_lock(&dblock);
+	if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
+		ast_log(LOG_WARNING, "Could bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
+		sqlite3_reset(stmt);
+		ast_mutex_unlock(&dblock);
+		return NULL;
 	}
 
+	while (sqlite3_step(stmt) == SQLITE_ROW) {
+		const char *key_s, *value_s;
+		if (!(key_s = (const char *) sqlite3_column_text(stmt, 0))) {
+			ast_log(LOG_WARNING, "Skipping invalid key!\n");
+			continue;
+		}
+		if (!(value_s = (const char *) sqlite3_column_text(stmt, 1))) {
+			ast_log(LOG_WARNING, "Skipping invalid value!\n");
+			continue;
+		}
+		++counter;
+		ast_cli(a->fd, "%-50s: %-25s\n", key_s, value_s);
+	}
+
+	sqlite3_reset(stmt);
+	ast_mutex_unlock(&dblock);
+
 	ast_cli(a->fd, "%d results found.\n", counter);
 	return CLI_SUCCESS;
 }
 
-static int db_showkey_cb(DBT *key, DBT *value, const char *filter, void *data)
-{
-	struct ast_cli_args *a = data;
-	const char *key_s = dbt_data2str_full(key, "<bad key>");
-	const char *value_s = dbt_data2str_full(value, "<bad value>");
-
-	if (subkeymatch(key_s, filter)) {
-		ast_cli(a->fd, "%-50s: %-25s\n", key_s, value_s);
-		return 1;
-	}
-
-	return 0;
-}
-
 static char *handle_cli_database_showkey(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
-	char suffix[MAX_DB_FIELD];
 	int counter = 0;
 
 	switch (cmd) {
 	case CLI_INIT:
 		e->command = "database showkey";
 		e->usage =
-			"Usage: database showkey <keytree>\n"
+			"Usage: database showkey <subfamily>\n"
 			"       Shows Asterisk database contents, restricted to a given key.\n";
 		return NULL;
 	case CLI_GENERATE:
 		return NULL;
 	}
 
-	if (a->argc == 3) {
-		/* Key only */
-		snprintf(suffix, sizeof(suffix), "/%s", a->argv[2]);
-	} else {
+	if (a->argc != 3) {
 		return CLI_SHOWUSAGE;
 	}
 
-	if ((counter = process_db_keys(db_showkey_cb, a, suffix, 0)) < 0) {
-		ast_cli(a->fd, "Database unavailable\n");
-		return CLI_SUCCESS;
+	ast_mutex_lock(&dblock);
+	if (!ast_strlen_zero(a->argv[2]) && (sqlite3_bind_text(showkey_stmt, 1, a->argv[2], -1, SQLITE_STATIC) != SQLITE_OK)) {
+		ast_log(LOG_WARNING, "Could bind %s to stmt: %s\n", a->argv[2], sqlite3_errmsg(astdb));
+		sqlite3_reset(showkey_stmt);
+		ast_mutex_unlock(&dblock);
+		return NULL;
 	}
 
+	while (sqlite3_step(showkey_stmt) == SQLITE_ROW) {
+		const char *key_s, *value_s;
+		if (!(key_s = (const char *) sqlite3_column_text(showkey_stmt, 0))) {
+			break;
+		}
+		if (!(value_s = (const char *) sqlite3_column_text(showkey_stmt, 1))) {
+			break;
+		}
+		++counter;
+		ast_cli(a->fd, "%-50s: %-25s\n", key_s, value_s);
+	}
+	sqlite3_reset(showkey_stmt);
+	ast_mutex_unlock(&dblock);
+
 	ast_cli(a->fd, "%d results found.\n", counter);
 	return CLI_SUCCESS;
 }
 
-static int db_gettree_cb(DBT *key, DBT *value, const char *filter, void *data)
+static int display_results(void *arg, int columns, char **values, char **colnames)
 {
-	struct ast_db_entry **ret = data;
-	struct ast_db_entry *cur;
-	const char *key_s = dbt_data2str_full(key, "<bad key>");
-	const char *value_s = dbt_data2str_full(value, "<bad value>");
-	size_t key_slen = strlen(key_s) + 1, value_slen = strlen(value_s) + 1;
+	struct ast_cli_args *a = arg;
+	size_t x;
 
-	if (keymatch(key_s, filter) && (cur = ast_malloc(sizeof(*cur) + key_slen + value_slen))) {
-		cur->next = *ret;
-		cur->key = cur->data + value_slen;
-		strcpy(cur->data, value_s);
-		strcpy(cur->key, key_s);
-		*ret = cur;
-		return 1;
+	for (x = 0; x < columns; x++) {
+		ast_cli(a->fd, "%-5s: %-50s\n", colnames[x], values[x]);
 	}
+	ast_cli(a->fd, "\n");
 
 	return 0;
 }
 
-struct ast_db_entry *ast_db_gettree(const char *family, const char *keytree)
+static char *handle_cli_database_query(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
-	char prefix[MAX_DB_FIELD];
-	struct ast_db_entry *ret = NULL;
 
-	if (!ast_strlen_zero(family)) {
-		if (!ast_strlen_zero(keytree)) {
-			/* Family and key tree */
-			snprintf(prefix, sizeof(prefix), "/%s/%s", family, keytree);
-		} else {
-			/* Family only */
-			snprintf(prefix, sizeof(prefix), "/%s", family);
-		}
-	} else {
-		prefix[0] = '\0';
-	}
-
-	if (process_db_keys(db_gettree_cb, &ret, prefix, 0) < 0) {
-		ast_log(LOG_WARNING, "Database unavailable\n");
+	switch (cmd) {
+	case CLI_INIT:
+		e->command = "database query";
+		e->usage =
+			"Usage: database query \"<SQL Statement>\"\n"
+			"       Run a user-specified SQL query on the database. Be careful.\n";
+		return NULL;
+	case CLI_GENERATE:
 		return NULL;
 	}
 
-	return ret;
-}
-
-void ast_db_freetree(struct ast_db_entry *dbe)
-{
-	struct ast_db_entry *last;
-	while (dbe) {
-		last = dbe;
-		dbe = dbe->next;
-		ast_free(last);
+	if (a->argc != 3) {
+		return CLI_SHOWUSAGE;
 	}
+
+	ast_mutex_lock(&dblock);
+	db_execute_sql(a->argv[2], display_results, a);
+	db_sync(); /* Go ahead and sync the db in case they write */
+	ast_mutex_unlock(&dblock);
+
+	return CLI_SUCCESS;
 }
 
 static struct ast_cli_entry cli_database[] = {
@@ -612,7 +726,8 @@ static struct ast_cli_entry cli_database[] = {
 	AST_CLI_DEFINE(handle_cli_database_get,     "Gets database value"),
 	AST_CLI_DEFINE(handle_cli_database_put,     "Adds/updates database value"),
 	AST_CLI_DEFINE(handle_cli_database_del,     "Removes database key/value"),
-	AST_CLI_DEFINE(handle_cli_database_deltree, "Removes database keytree/values")
+	AST_CLI_DEFINE(handle_cli_database_deltree, "Removes database subfamily/values"),
+	AST_CLI_DEFINE(handle_cli_database_query,   "Run a user-specified query on the astdb"),
 };
 
 static int manager_dbput(struct mansession *s, const struct message *m)
@@ -726,7 +841,7 @@ static int manager_dbdeltree(struct mansession *s, const struct message *m)
 		astman_send_error(s, m, "Database entry not found");
 	else
 		astman_send_ack(s, m, "Key tree deleted successfully");
-	
+
 	return 0;
 }
 
@@ -754,27 +869,48 @@ static void db_sync(void)
 static void *db_sync_thread(void *data)
 {
 	ast_mutex_lock(&dblock);
+	ast_db_begin_transaction();
 	for (;;) {
+		/* We're ok with spurious wakeups, so we don't worry about a predicate */
 		ast_cond_wait(&dbcond, &dblock);
+		if (ast_db_commit_transaction()) {
+			ast_db_rollback_transaction();
+		}
+		if (doexit) {
+			ast_mutex_unlock(&dblock);
+			break;
+		}
+		ast_db_begin_transaction();
 		ast_mutex_unlock(&dblock);
 		sleep(1);
 		ast_mutex_lock(&dblock);
-		astdb->sync(astdb, 0);
 	}
 
 	return NULL;
 }
 
+static void astdb_atexit(void)
+{
+	doexit = 1;
+	db_sync();
+	pthread_join(syncthread, NULL);
+	ast_mutex_lock(&dblock);
+	sqlite3_close(astdb);
+	ast_mutex_unlock(&dblock);
+}
+
 int astdb_init(void)
 {
-	pthread_t dont_care;
-
-	ast_cond_init(&dbcond, NULL);
-	if (ast_pthread_create_background(&dont_care, NULL, db_sync_thread, NULL)) {
+	if (db_init()) {
 		return -1;
 	}
 
-	dbinit();
+	ast_cond_init(&dbcond, NULL);
+	if (ast_pthread_create_background(&syncthread, NULL, db_sync_thread, NULL)) {
+		return -1;
+	}
+
+	ast_register_atexit(astdb_atexit);
 	ast_cli_register_multiple(cli_database, ARRAY_LEN(cli_database));
 	ast_manager_register_xml("DBGet", EVENT_FLAG_SYSTEM | EVENT_FLAG_REPORTING, manager_dbget);
 	ast_manager_register_xml("DBPut", EVENT_FLAG_SYSTEM, manager_dbput);

tests/test_db.c

@@ -205,10 +205,37 @@ AST_TEST_DEFINE(gettree_deltree)
 	return res;
 }
 
+AST_TEST_DEFINE(perftest)
+{
+	int res = AST_TEST_PASS;
+	size_t x;
+	char buf[10];
+
+	switch (cmd) {
+	case TEST_INIT:
+		info->name = "perftest";
+		info->category = "/main/astdb/";
+		info->summary = "astdb performance unit test";
+		info->description =
+			"Measure astdb performance";
+		return AST_TEST_NOT_RUN;
+	case TEST_EXECUTE:
+		break;
+	}
+
+	for (x = 0; x < 100000; x++) {
+		sprintf(buf, "%zu", x);
+		ast_db_put("astdbtest", buf, buf);
+	}
+	ast_db_deltree("astdbtest", NULL);
+
+	return res;
+}
 static int unload_module(void)
 {
 	AST_TEST_UNREGISTER(put_get_del);
 	AST_TEST_UNREGISTER(gettree_deltree);
+	AST_TEST_UNREGISTER(perftest);
 	return 0;
 }
 
@@ -216,6 +243,7 @@ static int load_module(void)
 {
 	AST_TEST_REGISTER(put_get_del);
 	AST_TEST_REGISTER(gettree_deltree);
+	AST_TEST_REGISTER(perftest);
 	return AST_MODULE_LOAD_SUCCESS;
 }
 

utils/Makefile

@@ -89,6 +89,8 @@ clean:
 	rm -f md5.c strcompat.c ast_expr2.c ast_expr2.h ast_expr2f.c pbx_ael.c pval.c hashtab.c lock.c
 	rm -f aelparse.c aelbison.c conf2ael
 	rm -f utils.c strings.c threadstorage.c sha1.c astobj2.c hashtest2 hashtest refcounter
+	rm -f db1-ast/.*.d
+	@$(MAKE) -C db1-ast clean
 
 md5.c: $(ASTTOPDIR)/main/md5.c
 	$(ECHO_PREFIX) echo "   [CP] $(subst $(ASTTOPDIR)/,,$<) -> $@"
@@ -206,6 +208,14 @@ muted: muted.o
 muted: LIBS+=$(AUDIO_LIBS)
 muted: _ASTCFLAGS:=$(filter-out -Werror,$(_ASTCFLAGS))
 
+CHECK_SUBDIR:	# do nothing, just make sure that we recurse in the subdir/
+db1-ast/libdb1.a: CHECK_SUBDIR
+	_ASTCFLAGS="$(_ASTCFLAGS) -Wno-strict-aliasing" ASTCFLAGS="$(ASTCFLAGS)" $(MAKE) -C db1-ast libdb1.a
+
+astdb2sqlite3: LIBS+=$(SQLITE3_LIB)
+astdb2sqlite3: _ASTCFLAGS+=$(SQLITE3_INCLUDE)
+astdb2sqlite3: db1-ast/libdb1.a 
+
 ifneq ($(wildcard .*.d),)
    include .*.d
 endif

utils/astdb2sqlite3.c

@@ -0,0 +1,231 @@
+/*
+ * Asterisk -- An open source telephony toolkit.
+ *
+ * Copyright (C) 1999 - 2005, Digium, Inc.
+ *
+ * Mark Spencer <markster@digium.com>
+ *
+ * See http://www.asterisk.org for more information about
+ * the Asterisk project. Please do not directly contact
+ * any of the maintainers of this project for assistance;
+ * the project provides a web site, mailing lists and IRC
+ * channels for your use.
+ *
+ * This program is free software, distributed under the terms of
+ * the GNU General Public License Version 2. See the LICENSE file
+ * at the top of the source tree.
+ */
+
+/*! \file
+ *
+ * \brief Berekeley DB to SQLite3 converter
+ *
+ * \author Terry Wilson <twilson@digium.com>
+ */
+
+#include "asterisk.h"
+
+//ASTERISK_FILE_VERSION(__FILE__, "$Revision$")
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sqlite3.h>
+#include <libgen.h> /* OS X doesn't have the basename function in strings.h */
+
+#include "db1-ast/include/db.h"
+
+#define MAX_DB_FIELD 256
+#define MIN(a,b) \
+({ typeof (a) _a = (a); \
+	typeof (b) _b = (b); \
+	a < _b ? _a : _b; })
+
+static sqlite3 *astdb;
+
+#define DEFINE_SQL_STATEMENT(stmt,sql) static sqlite3_stmt *stmt; \
+	const char stmt##_sql[] = sql;
+
+DEFINE_SQL_STATEMENT(put_stmt, "INSERT OR REPLACE INTO astdb (key, value) VALUES (?, ?)")
+DEFINE_SQL_STATEMENT(create_astdb_stmt, "CREATE TABLE IF NOT EXISTS astdb(key VARCHAR(256), value VARCHAR(256), PRIMARY KEY(key))")
+
+static int db_execute_transaction_sql(const char *sql)
+{
+	char *errmsg = NULL;
+	int res =0;
+
+	sqlite3_exec(astdb, sql, NULL, NULL, &errmsg);
+	if (errmsg) {
+		fprintf(stderr, "Error executing SQL: %s\n", errmsg);
+		sqlite3_free(errmsg);
+		res = -1;
+	}
+
+	return res;
+}
+
+static int ast_db_begin_transaction(void)
+{
+	return db_execute_transaction_sql("BEGIN TRANSACTION");
+}
+
+static int ast_db_commit_transaction(void)
+{
+	return db_execute_transaction_sql("COMMIT");
+}
+
+static int ast_db_rollback_transaction(void)
+{
+	return db_execute_transaction_sql("ROLLBACK");
+}
+
+static int db_put_raw(const char *key, size_t keylen, const char *value, size_t valuelen)
+{
+	int res = 0;
+
+	if (sqlite3_bind_text(put_stmt, 1, key, keylen, SQLITE_STATIC) != SQLITE_OK) {
+		fprintf(stderr, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_bind_text(put_stmt, 2, value, valuelen, SQLITE_STATIC) != SQLITE_OK) {
+		fprintf(stderr, "Couldn't bind value to stmt: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	} else if (sqlite3_step(put_stmt) != SQLITE_DONE) {
+		fprintf(stderr, "Couldn't execute statment: %s\n", sqlite3_errmsg(astdb));
+		res = -1;
+	}
+	sqlite3_reset(put_stmt);
+
+	return res;
+}
+
+static int convert_bdb_to_sqlite3(const char *bdb_dbname)
+{
+	DB *bdb;
+	DBT key = { 0, }, value = { 0, }, last_key = { 0, };
+	int res, last = 0;
+	char last_key_s[MAX_DB_FIELD];
+
+	if (!(bdb = dbopen(bdb_dbname, O_RDONLY, AST_FILE_MODE, DB_BTREE, NULL))) {
+		fprintf(stderr, "Unable to open Asterisk database '%s'\n", bdb_dbname);
+		return -1;
+	}
+
+	if (bdb->seq(bdb, &last_key, &value, R_LAST)) {
+		/* Empty database */
+		return 0;
+	}
+
+	memcpy(last_key_s, last_key.data, MIN(last_key.size - 1, sizeof(last_key_s)));
+	last_key_s[last_key.size - 1] = '\0';
+	for (res = bdb->seq(bdb, &key, &value, R_FIRST);
+			!res; res = bdb->seq(bdb, &key, &value, R_NEXT)) {
+		last = !strcmp(key.data, last_key_s);
+		db_put_raw((const char *) key.data, key.size - 1, (const char *) value.data, value.size - 1);
+		if (last) {
+			break;
+		}
+	}
+
+	bdb->close(bdb);
+
+	return 0;
+}
+
+static int init_stmt(sqlite3_stmt **stmt, const char *sql, size_t len)
+{
+	if (sqlite3_prepare_v2(astdb, sql, len, stmt, NULL) != SQLITE_OK) {
+		fprintf(stderr, "Couldn't prepare statement '%s': %s\n", sql, sqlite3_errmsg(astdb));
+		return -1;
+	}
+
+	return 0;
+}
+
+static int db_create_astdb(void)
+{
+	if (init_stmt(&create_astdb_stmt, create_astdb_stmt_sql, sizeof(create_astdb_stmt_sql))) {
+		return -1;
+	}
+
+	ast_db_begin_transaction();
+	if (sqlite3_step(create_astdb_stmt) != SQLITE_DONE) {
+		fprintf(stderr, "Couldn't create astdb table: %s\n", sqlite3_errmsg(astdb));
+		ast_db_rollback_transaction();
+		sqlite3_reset(create_astdb_stmt);
+		return -1;
+	}
+
+	ast_db_commit_transaction();
+	sqlite3_reset(create_astdb_stmt);
+
+	return 0;
+}
+
+static int init_statements(void)
+{
+	/* Don't initialize create_astdb_statment here as the astdb table needs to exist
+	 * brefore these statments can be initialized */
+	return init_stmt(&put_stmt, put_stmt_sql, sizeof(put_stmt_sql));
+}
+
+static int db_open(const char *dbname)
+{
+	if (sqlite3_open_v2(dbname, &astdb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, NULL) != SQLITE_OK) {
+		fprintf(stderr, "Unable to open Asterisk database '%s': %s\n", dbname, sqlite3_errmsg(astdb));
+		sqlite3_close(astdb);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int sql_db_init(const char *dbname)
+{
+	if (db_open(dbname) || db_create_astdb() || init_statements()) {
+		return -1;
+	}
+
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	char *dbname;
+	struct stat dont_care;
+
+	if (argc != 2) {
+		fprintf(stderr, "%s takes the path of astdb as its only argument\n", basename(argv[0]));
+		exit(-1);
+	}
+
+	if (stat(argv[1], &dont_care)) {
+		fprintf(stderr, "Unable to open %s: %s\n", argv[1], strerror(errno));
+		exit(-1);
+	}
+
+	if (!(dbname = alloca(strlen(argv[1]) + sizeof(".sqlite3")))) {
+		exit(-1);
+	}
+
+	strcpy(dbname, argv[1]);
+	strcat(dbname, ".sqlite3");
+
+	if (!stat(dbname, &dont_care)) {
+		fprintf(stderr, "%s already exists!\n", dbname);
+		exit(-1);
+	}
+
+	if (sql_db_init(dbname)) {
+		exit(-1);
+	}
+
+	if (convert_bdb_to_sqlite3(argv[1])) {
+		fprintf(stderr, "Database conversion failed!\n");
+		exit(-1);
+		sqlite3_close(astdb);
+	}
+
+	sqlite3_close(astdb);
+	return 0;
+}

utils/db1-ast/mpool/mpool.c

@@ -31,8 +31,6 @@
  * SUCH DAMAGE.
  */
 
-#include "asterisk.h"
-
 #if defined(LIBC_SCCS) && !defined(lint)
 static char sccsid[] = "@(#)mpool.c	8.5 (Berkeley) 7/26/94";
 #endif /* LIBC_SCCS and not lint */
@@ -49,8 +47,6 @@ static char sccsid[] = "@(#)mpool.c	8.5 (Berkeley) 7/26/94";
 
 #include "../include/db.h"
 
-#include "asterisk/utils.h"
-
 #define	__MPOOLINTERFACE_PRIVATE
 #include <mpool.h>
 
@@ -384,7 +380,7 @@ mpool_bkt(mp)
 			return (bp);
 		}
 
-new:	if ((bp = (BKT *) ast_calloc(1, sizeof(BKT) + mp->pagesize)) == NULL)
+new:	if ((bp = (BKT *) calloc(1, sizeof(BKT) + mp->pagesize)) == NULL)
 		return (NULL);
 #ifdef STATISTICS
 	++mp->pagealloc;

utils/utils.xml

@@ -5,6 +5,9 @@
   <member name="astcanary">
 	<defaultenabled>yes</defaultenabled>
   </member>
+  <member name="astdb2sqlite3">
+	<defaultenabled>yes</defaultenabled>
+  </member>
   <member name="astman">
 	<defaultenabled>no</defaultenabled>
 	<depend>newt</depend>