From 722d6b06f606bc045ecef6fe5c8f2c700a3dad65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Valverde?= Date: Fri, 8 Jun 2018 18:09:11 +0100 Subject: [PATCH] Rewrite make-{dissectors,taps} in Python Having these build tools in machine code poses problems when cross-compiling. The most significant being that we need to find the host and build GLiB dependencies at compile-time. There is no noticeable speed difference between the Python and C implementation. Ping-Bug: 14622 Change-Id: Id13f823c7f4abf51edfa291e703028873748989f Reviewed-on: https://code.wireshark.org/review/28130 Petri-Dish: Peter Wu Tested-by: Petri Dish Buildbot Reviewed-by: Peter Wu --- .gitignore | 2 - CMakeLists.txt | 2 +- cmake/modules/UseMakeTaps.cmake | 4 +- epan/dissectors/CMakeLists.txt | 4 +- tools/CMakeLists.txt | 32 ------- tools/make-dissectors.c | 161 -------------------------------- tools/make-lib.c | 89 ------------------ tools/make-lib.h | 38 -------- tools/make-regs.py | 127 +++++++++++++++++++++++++ tools/make-taps.c | 122 ------------------------ tools/pre-commit-ignore.conf | 1 - 11 files changed, 132 insertions(+), 450 deletions(-) delete mode 100644 tools/CMakeLists.txt delete mode 100644 tools/make-dissectors.c delete mode 100644 tools/make-lib.c delete mode 100644 tools/make-lib.h create mode 100755 tools/make-regs.py delete mode 100644 tools/make-taps.c diff --git a/.gitignore b/.gitignore index 44de56f6ba..834539099a 100644 --- a/.gitignore +++ b/.gitignore @@ -48,8 +48,6 @@ plugins/codecs/*/plugin.c plugins/epan/mate/mate_parser.c version.h tools/lemon/lemon -tools/make-dissectors -tools/make-taps tshark-tap-register.c wiretap/ascend.c wiretap/ascend.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 994a748fd3..2e4791ba20 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1267,7 +1267,7 @@ add_subdirectory( docbook EXCLUDE_FROM_ALL ) add_subdirectory( epan ) add_subdirectory( extcap ) add_subdirectory( randpkt_core ) -add_subdirectory( tools ) +add_subdirectory( tools/lemon ) add_subdirectory( ui ) add_subdirectory( wiretap ) add_subdirectory( writecap ) diff --git a/cmake/modules/UseMakeTaps.cmake b/cmake/modules/UseMakeTaps.cmake index 18efdfc9e1..aed93183ce 100644 --- a/cmake/modules/UseMakeTaps.cmake +++ b/cmake/modules/UseMakeTaps.cmake @@ -5,9 +5,9 @@ MACRO(REGISTER_TAP_FILES _outputfile) OUTPUT ${_outputfile} COMMAND - make-taps ${_outputfile} ${_sources} + ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/make-regs.py taps ${_outputfile} ${_sources} DEPENDS - make-taps + ${CMAKE_SOURCE_DIR}/tools/make-regs.py ${_sources} COMMENT "Making ${_outputfile}" diff --git a/epan/dissectors/CMakeLists.txt b/epan/dissectors/CMakeLists.txt index 5d4c906fdf..00d8c3b4c9 100644 --- a/epan/dissectors/CMakeLists.txt +++ b/epan/dissectors/CMakeLists.txt @@ -1908,8 +1908,8 @@ file(GENERATE ) add_custom_command( OUTPUT dissectors.c - COMMAND make-dissectors dissectors.c @dissectors.in.txt - DEPENDS make-dissectors ${ALL_DISSECTOR_SRC} + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/make-regs.py dissectors dissectors.c @dissectors.in.txt + DEPENDS ${CMAKE_SOURCE_DIR}/tools/make-regs.py ${ALL_DISSECTOR_SRC} "${CMAKE_CURRENT_BINARY_DIR}/dissectors.in.txt" COMMENT "Making dissectors.c" ) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt deleted file mode 100644 index efcba14808..0000000000 --- a/tools/CMakeLists.txt +++ /dev/null @@ -1,32 +0,0 @@ -# CMakeLists.txt -# -# Wireshark - Network traffic analyzer -# By Gerald Combs -# Copyright 1998 Gerald Combs -# -# SPDX-License-Identifier: GPL-2.0-or-later -# - -add_subdirectory(lemon) - -add_executable(make-dissectors make-dissectors.c make-lib.c) -target_link_libraries(make-dissectors ${GLIB2_LIBRARIES}) -if(WIN32) - add_dependencies(make-dissectors copy_cli_dlls) -endif() - -add_executable(make-taps make-taps.c make-lib.c) -target_link_libraries(make-taps ${GLIB2_LIBRARIES}) - -# -# Editor modelines - http://www.wireshark.org/tools/modelines.html -# -# Local variables: -# c-basic-offset: 8 -# tab-width: 8 -# indent-tabs-mode: t -# End: -# -# vi: set shiftwidth=8 tabstop=8 noexpandtab: -# :indentSize=8:tabSize=8:noTabs=false: -# diff --git a/tools/make-dissectors.c b/tools/make-dissectors.c deleted file mode 100644 index e98a81d7d3..0000000000 --- a/tools/make-dissectors.c +++ /dev/null @@ -1,161 +0,0 @@ -/* make-dissectors.c - * Tool to build the dissector registration arrays. - * - * Wireshark - Network traffic analyzer - * By Gerald Combs - * Copyright 1998 Gerald Combs - * - * SPDX-License-Identifier: GPL-2.0-or-later - */ - -#include -#include -#include -#include "make-lib.h" - -#define ARRAY_RESERVED_SIZE 2048 -#define STRING_RESERVED_SIZE (300 * 1024) - - -int main(int argc, char **argv) -{ - GRegex *protos_regex, *handoffs_regex; - GPtrArray *protos = NULL, *handoffs = NULL; - struct symbol_item items[2]; - GError *err = NULL; - guint i; - GString *s; - const char *outfile; - guint count_protos, count_handoffs; - - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - exit(1); - } - - protos = g_ptr_array_new_full(ARRAY_RESERVED_SIZE, g_free); - handoffs = g_ptr_array_new_full(ARRAY_RESERVED_SIZE, g_free); - - protos_regex = g_regex_new("void\\s+(proto_register_[[:alnum:]_]+)\\s*\\(\\s*void\\s*\\)\\s*{", - G_REGEX_OPTIMIZE, G_REGEX_MATCH_NOTEMPTY, &err); - if (err) { - fprintf(stderr, "GRegex: %s\n", err->message); - exit(1); - } - handoffs_regex = g_regex_new("void\\s+(proto_reg_handoff_[[:alnum:]_]+)\\s*\\(\\s*void\\s*\\)\\s*{", - G_REGEX_OPTIMIZE, G_REGEX_MATCH_NOTEMPTY, &err); - if (err) { - fprintf(stderr, "GRegex: %s\n", err->message); - exit(1); - } - - items[0].regex = protos_regex; - items[0].ptr_array = protos; - items[1].regex = handoffs_regex; - items[1].ptr_array = handoffs; - - outfile = argv[1]; - for (int arg = 2; arg < argc; arg++) { - if (argv[arg][0] == '@') { - scan_list(&argv[arg][1], items, G_N_ELEMENTS(items)); - } - else { - scan_file(argv[arg], items, G_N_ELEMENTS(items)); - } - } - - if (protos->len == 0) { - fprintf(stderr, "No protocol registrations found.\n"); - exit(1); - } - - g_ptr_array_sort(protos, compare_symbols); - g_ptr_array_sort(handoffs, compare_symbols); - - s = g_string_sized_new(STRING_RESERVED_SIZE); - - g_string_append(s, - "/*\n" - " * Do not modify this file. Changes will be overwritten.\n" - " *\n" - " * Generated automatically using \"make-dissectors\".\n" - " */\n" - "\n" - "#include \n" - "\n"); - - g_string_append_printf(s, - "const gulong dissector_reg_proto_count = %d;\n" - "const gulong dissector_reg_handoff_count = %d;\n" - "\n", - protos->len, handoffs->len); - - for (i = 0; i < protos->len; i++) { - g_string_append_printf(s, - "void %s(void);\n", - (char *)protos->pdata[i]); - } - g_string_append(s, - "\n" - "dissector_reg_t dissector_reg_proto[] = {\n"); - for (i = 0; i < protos->len; i++) { - g_string_append_printf(s, - " { \"%s\", %s },\n", - (char *)protos->pdata[i], (char *)protos->pdata[i]); - } - g_string_append(s, - " { NULL, NULL }\n" - "};\n" - "\n"); - - for (i = 0; i < handoffs->len; i++) { - g_string_append_printf(s, - "void %s(void);\n", - (char *)handoffs->pdata[i]); - } - g_string_append(s, - "\n" - "dissector_reg_t dissector_reg_handoff[] = {\n"); - for (i = 0; i < handoffs->len; i++) { - g_string_append_printf(s, - " { \"%s\", %s },\n", - (char *)handoffs->pdata[i], (char *)handoffs->pdata[i]); - } - g_string_append(s, - " { NULL, NULL }\n" - "};\n"); - - if (!g_file_set_contents(outfile, s->str, s->len, &err)) { - fprintf(stderr, "%s: %s\n", outfile, err->message); - exit(1); - } - - count_protos = protos->len; - count_handoffs = handoffs->len; - - g_string_free(s, TRUE); - - g_regex_unref(protos_regex); - g_regex_unref(handoffs_regex); - - g_ptr_array_free(protos, TRUE); - g_ptr_array_free(handoffs, TRUE); - - printf("Found %u registrations and %u handoffs.\n", - count_protos, count_handoffs); - - return 0; -} - -/* - * Editor modelines - * - * Local Variables: - * c-basic-offset: 4 - * tab-width: 8 - * indent-tabs-mode: nil - * End: - * - * ex: set shiftwidth=4 tabstop=8 expandtab: - * :indentSize=4:tabSize=8:noTabs=true: - */ diff --git a/tools/make-lib.c b/tools/make-lib.c deleted file mode 100644 index 4bd93264e7..0000000000 --- a/tools/make-lib.c +++ /dev/null @@ -1,89 +0,0 @@ -/* make-lib.c - * - * Wireshark - Network traffic analyzer - * By Gerald Combs - * Copyright 1998 Gerald Combs - * - * SPDX-License-Identifier: GPL-2.0-or-later - */ - -#include -#include -#include -#include "make-lib.h" - - -#ifdef _WIN32 - #define SEP "\r\n" -#else - #define SEP "\n" -#endif - - -int -compare_symbols(gconstpointer a, gconstpointer b) -{ - return g_strcmp0(*(const char **)a, *(const char **)b); -} - -static void -scan_matches(GRegex *regex, const char *string, GPtrArray *dst) -{ - GMatchInfo *match_info; - char *match; - - g_regex_match(regex, string, G_REGEX_MATCH_NOTEMPTY, &match_info); - while (g_match_info_matches(match_info)) { - match = g_match_info_fetch(match_info, 1); - g_ptr_array_add(dst, match); - g_match_info_next(match_info, NULL); - } - g_match_info_free(match_info); -} - -void -scan_file(const char *file, struct symbol_item *items, size_t items_len) -{ - char *contents; - GError *err = NULL; - size_t i; - - if (!g_file_get_contents(file, &contents, NULL, &err)) { - fprintf(stderr, "%s: %s\n", file, err->message); - exit(1); - } - for (i = 0; i < items_len; i++) { - scan_matches(items[i].regex, contents, items[i].ptr_array); - } - g_free(contents); -} - -void -scan_list(const char *file, struct symbol_item *items, size_t items_len) -{ - char *contents, *arg; - GError *err = NULL; - - if (!g_file_get_contents(file, &contents, NULL, &err)) { - fprintf(stderr, "%s: %s\n", file, err->message); - exit(1); - } - for (arg = strtok(contents, SEP); arg != NULL; arg = strtok(NULL, SEP)) { - scan_file(arg, items, items_len); - } - g_free(contents); -} - - -/* - * Editor modelines - * - * Local Variables: - * c-basic-offset: 4 - * tab-width: 8 - * indent-tabs-mode: nil - * End: - * - * ex: set shiftwidth=4 tabstop=8 expandtab: - * :indentSize=4:tabSize=8:noTabs=true: - */ diff --git a/tools/make-lib.h b/tools/make-lib.h deleted file mode 100644 index 9d51ff7077..0000000000 --- a/tools/make-lib.h +++ /dev/null @@ -1,38 +0,0 @@ -/* make-lib.h - * - * Wireshark - Network traffic analyzer - * By Gerald Combs - * Copyright 1998 Gerald Combs - * - * SPDX-License-Identifier: GPL-2.0-or-later - */ - -#include - -struct symbol_item { - GRegex *regex; - GPtrArray *ptr_array; -}; - -/* Compares symbols using strcmp() */ -int compare_symbols(gconstpointer a, gconstpointer b); - -/* Scan a C source file for symbols */ -void scan_file(const char *file, struct symbol_item *items, size_t items_len); - -/* Takes a text file containing a list of C source files on which to call scan_file() */ -void scan_list(const char *file, struct symbol_item *items, size_t items_len); - - -/* - * Editor modelines - * - * Local Variables: - * c-basic-offset: 4 - * tab-width: 8 - * indent-tabs-mode: nil - * End: - * - * ex: set shiftwidth=4 tabstop=8 expandtab: - * :indentSize=4:tabSize=8:noTabs=true: - */ diff --git a/tools/make-regs.py b/tools/make-regs.py new file mode 100755 index 0000000000..7cb8a2285c --- /dev/null +++ b/tools/make-regs.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# +# Looks for registration routines in the source files +# and assembles C code to call all the routines. +# +# SPDX-License-Identifier: GPL-2.0-or-later +# + +import sys +import re + +preamble = """\ +/* + * Do not modify this file. Changes will be overwritten. + * + * Generated automatically using \"make-regs.py\". + */ + +""" + +def gen_prototypes(funcs): + output = "" + for f in funcs: + output += "void {}(void);\n".format(f) + return output + +def gen_array(funcs, name): + output = "{}[] = {{\n".format(name) + for f in funcs: + output += " {{ \"{0}\", {0} }},\n".format(f) + output += " { NULL, NULL }\n};\n" + return output + +def scan_files(infiles, regs): + for path in infiles: + with open(path, 'r') as f: + source = f.read() + for array, regex in regs: + matches = re.findall(regex, source) + array.extend(matches) + +def make_dissectors(outfile, infiles): + protos = [] + protos_regex = r"void\s+(proto_register_[\w]+)\s*\(\s*void\s*\)\s*{" + handoffs = [] + handoffs_regex = r"void\s+(proto_reg_handoff_[\w]+)\s*\(\s*void\s*\)\s*{" + + scan_files(infiles, [(protos, protos_regex), (handoffs, handoffs_regex)]) + + if len(protos) < 1: + sys.exit("No protocol registrations found.") + + protos.sort() + handoffs.sort() + + output = preamble + output += """\ +#include "dissectors.h" + +const gulong dissector_reg_proto_count = {0}; +const gulong dissector_reg_handoff_count = {1}; + +""".format(len(protos), len(handoffs)) + + output += gen_prototypes(protos) + output += "\n" + output += gen_array(protos, "dissector_reg_t dissector_reg_proto") + output += "\n" + output += gen_prototypes(handoffs) + output += "\n" + output += gen_array(handoffs, "dissector_reg_t dissector_reg_handoff") + + with open(outfile, "w") as f: + f.write(output) + + print("Found {0} registrations and {1} handoffs.".format(len(protos), len(handoffs))) + +def make_taps(outfile, infiles): + taps = [] + taps_regex = r"void\s+(register_tap_listener_[\w]+)\s*\(\s*void\s*\)\s*{" + + scan_files(infiles, [(taps, taps_regex)]) + + if len(taps) < 1: + sys.exit("No tap registrations found.") + + taps.sort() + + output = preamble + output += """\ +#include "ui/taps.h" + +const gulong tap_reg_listener_count = {0}; + +""".format(len(taps)) + + output += gen_prototypes(taps) + output += "\n" + output += gen_array(taps, "tap_reg_t tap_reg_listener") + + with open(outfile, "w") as f: + f.write(output) + + print("Found {0} registrations.".format(len(taps))) + + +def print_usage(): + sys.exit("Usage: {0} \n".format(sys.argv[0])) + +if __name__ == "__main__": + if len(sys.argv) < 4: + print_usage() + + mode = sys.argv[1] + outfile = sys.argv[2] + if sys.argv[3].startswith("@"): + with open(sys.argv[3][1:]) as f: + infiles = [l.strip() for l in f.readlines()] + else: + infiles = sys.argv[3:] + + if mode == "dissectors": + make_dissectors(outfile, infiles) + elif mode == "taps": + make_taps(outfile, infiles) + else: + print_usage() diff --git a/tools/make-taps.c b/tools/make-taps.c deleted file mode 100644 index d02cc52073..0000000000 --- a/tools/make-taps.c +++ /dev/null @@ -1,122 +0,0 @@ -/* make-taps.c - * Tool to build the tap registration arrays. - * - * Wireshark - Network traffic analyzer - * By Gerald Combs - * Copyright 1998 Gerald Combs - * - * SPDX-License-Identifier: GPL-2.0-or-later - */ - -#include -#include -#include -#include "make-lib.h" - -#define ARRAY_RESERVED_SIZE 128 -#define STRING_RESERVED_SIZE (8 * 1024) - - -int main(int argc, char **argv) -{ - GRegex *taps_regex; - GPtrArray *taps = NULL; - struct symbol_item items[1]; - GError *err = NULL; - guint i; - GString *s; - const char *outfile; - guint count_taps; - - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - exit(1); - } - - taps = g_ptr_array_new_full(ARRAY_RESERVED_SIZE, g_free); - - taps_regex = g_regex_new("void\\s+(register_tap_listener_[[:alnum:]_]+)\\s*\\(\\s*void\\s*\\)\\s*{", - G_REGEX_OPTIMIZE, G_REGEX_MATCH_NOTEMPTY, &err); - if (err) { - fprintf(stderr, "GRegex: %s\n", err->message); - exit(1); - } - - items[0].regex = taps_regex; - items[0].ptr_array = taps; - - outfile = argv[1]; - for (int arg = 2; arg < argc; arg++) { - scan_file(argv[arg], items, G_N_ELEMENTS(items)); - } - - if (taps->len == 0) { - fprintf(stderr, "No tap registrations found.\n"); - exit(1); - } - - g_ptr_array_sort(taps, compare_symbols); - - s = g_string_sized_new(STRING_RESERVED_SIZE); - - g_string_append(s, - "/*\n" - " * Do not modify this file. Changes will be overwritten.\n" - " *\n" - " * Generated automatically using \"make-taps\".\n" - " */\n" - "\n" - "#include \"ui/taps.h\"\n" - "\n"); - - g_string_append_printf(s, - "const gulong tap_reg_listener_count = %d;\n" - "\n", - taps->len); - - for (i = 0; i < taps->len; i++) { - g_string_append_printf(s, - "void %s(void);\n", - (char *)taps->pdata[i]); - } - g_string_append(s, - "\n" - "tap_reg_t tap_reg_listener[] = {\n"); - for (i = 0; i < taps->len; i++) { - g_string_append_printf(s, - " { \"%s\", %s },\n", - (char *)taps->pdata[i], (char *)taps->pdata[i]); - } - g_string_append(s, - " { NULL, NULL }\n" - "};\n" - "\n"); - - if (!g_file_set_contents(outfile, s->str, s->len, &err)) { - fprintf(stderr, "%s: %s\n", outfile, err->message); - exit(1); - } - - count_taps = taps->len; - - g_string_free(s, TRUE); - - g_regex_unref(taps_regex); - - g_ptr_array_free(taps, TRUE); - - printf("Found %u registrations.\n", count_taps); -} - -/* - * Editor modelines - * - * Local Variables: - * c-basic-offset: 4 - * tab-width: 8 - * indent-tabs-mode: nil - * End: - * - * ex: set shiftwidth=4 tabstop=8 expandtab: - * :indentSize=4:tabSize=8:noTabs=true: - */ diff --git a/tools/pre-commit-ignore.conf b/tools/pre-commit-ignore.conf index ae9b29e06f..d3f23ca057 100644 --- a/tools/pre-commit-ignore.conf +++ b/tools/pre-commit-ignore.conf @@ -19,7 +19,6 @@ extcap/* image/stock_icons/* mmdbresolve.c tools/lemon/* -tools/make-dissectors.c wsutil/file_util.h wsutil/strptime.c wsutil/getopt_long.c