Rewrite make-{dissectors,taps} in Python

Having these build tools in machine code poses problems when cross-compiling.
The most significant being that we need to find the host and build GLiB
dependencies at compile-time.

There is no noticeable speed difference between the Python and C implementation.

Ping-Bug: 14622
Change-Id: Id13f823c7f4abf51edfa291e703028873748989f
Reviewed-on: https://code.wireshark.org/review/28130
Petri-Dish: Peter Wu <peter@lekensteyn.nl>
Tested-by: Petri Dish Buildbot
Reviewed-by: Peter Wu <peter@lekensteyn.nl>
This commit is contained in:
João Valverde 2018-06-08 18:09:11 +01:00 committed by Peter Wu
parent bfb7028ba9
commit 722d6b06f6
11 changed files with 132 additions and 450 deletions

2
.gitignore vendored
View File

@ -48,8 +48,6 @@ plugins/codecs/*/plugin.c
plugins/epan/mate/mate_parser.c
version.h
tools/lemon/lemon
tools/make-dissectors
tools/make-taps
tshark-tap-register.c
wiretap/ascend.c
wiretap/ascend.h

View File

@ -1267,7 +1267,7 @@ add_subdirectory( docbook EXCLUDE_FROM_ALL )
add_subdirectory( epan )
add_subdirectory( extcap )
add_subdirectory( randpkt_core )
add_subdirectory( tools )
add_subdirectory( tools/lemon )
add_subdirectory( ui )
add_subdirectory( wiretap )
add_subdirectory( writecap )

View File

@ -5,9 +5,9 @@ MACRO(REGISTER_TAP_FILES _outputfile)
OUTPUT
${_outputfile}
COMMAND
make-taps ${_outputfile} ${_sources}
${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/make-regs.py taps ${_outputfile} ${_sources}
DEPENDS
make-taps
${CMAKE_SOURCE_DIR}/tools/make-regs.py
${_sources}
COMMENT
"Making ${_outputfile}"

View File

@ -1908,8 +1908,8 @@ file(GENERATE
)
add_custom_command(
OUTPUT dissectors.c
COMMAND make-dissectors dissectors.c @dissectors.in.txt
DEPENDS make-dissectors ${ALL_DISSECTOR_SRC}
COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/make-regs.py dissectors dissectors.c @dissectors.in.txt
DEPENDS ${CMAKE_SOURCE_DIR}/tools/make-regs.py ${ALL_DISSECTOR_SRC}
"${CMAKE_CURRENT_BINARY_DIR}/dissectors.in.txt"
COMMENT "Making dissectors.c"
)

View File

@ -1,32 +0,0 @@
# CMakeLists.txt
#
# Wireshark - Network traffic analyzer
# By Gerald Combs <gerald@wireshark.org>
# Copyright 1998 Gerald Combs
#
# SPDX-License-Identifier: GPL-2.0-or-later
#
add_subdirectory(lemon)
add_executable(make-dissectors make-dissectors.c make-lib.c)
target_link_libraries(make-dissectors ${GLIB2_LIBRARIES})
if(WIN32)
add_dependencies(make-dissectors copy_cli_dlls)
endif()
add_executable(make-taps make-taps.c make-lib.c)
target_link_libraries(make-taps ${GLIB2_LIBRARIES})
#
# Editor modelines - http://www.wireshark.org/tools/modelines.html
#
# Local variables:
# c-basic-offset: 8
# tab-width: 8
# indent-tabs-mode: t
# End:
#
# vi: set shiftwidth=8 tabstop=8 noexpandtab:
# :indentSize=8:tabSize=8:noTabs=false:
#

View File

@ -1,161 +0,0 @@
/* make-dissectors.c
* Tool to build the dissector registration arrays.
*
* Wireshark - Network traffic analyzer
* By Gerald Combs <gerald@wireshark.org>
* Copyright 1998 Gerald Combs
*
* SPDX-License-Identifier: GPL-2.0-or-later
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "make-lib.h"
#define ARRAY_RESERVED_SIZE 2048
#define STRING_RESERVED_SIZE (300 * 1024)
int main(int argc, char **argv)
{
GRegex *protos_regex, *handoffs_regex;
GPtrArray *protos = NULL, *handoffs = NULL;
struct symbol_item items[2];
GError *err = NULL;
guint i;
GString *s;
const char *outfile;
guint count_protos, count_handoffs;
if (argc < 3) {
fprintf(stderr, "Usage: %s <outfile> <infiles...>\n", argv[0]);
exit(1);
}
protos = g_ptr_array_new_full(ARRAY_RESERVED_SIZE, g_free);
handoffs = g_ptr_array_new_full(ARRAY_RESERVED_SIZE, g_free);
protos_regex = g_regex_new("void\\s+(proto_register_[[:alnum:]_]+)\\s*\\(\\s*void\\s*\\)\\s*{",
G_REGEX_OPTIMIZE, G_REGEX_MATCH_NOTEMPTY, &err);
if (err) {
fprintf(stderr, "GRegex: %s\n", err->message);
exit(1);
}
handoffs_regex = g_regex_new("void\\s+(proto_reg_handoff_[[:alnum:]_]+)\\s*\\(\\s*void\\s*\\)\\s*{",
G_REGEX_OPTIMIZE, G_REGEX_MATCH_NOTEMPTY, &err);
if (err) {
fprintf(stderr, "GRegex: %s\n", err->message);
exit(1);
}
items[0].regex = protos_regex;
items[0].ptr_array = protos;
items[1].regex = handoffs_regex;
items[1].ptr_array = handoffs;
outfile = argv[1];
for (int arg = 2; arg < argc; arg++) {
if (argv[arg][0] == '@') {
scan_list(&argv[arg][1], items, G_N_ELEMENTS(items));
}
else {
scan_file(argv[arg], items, G_N_ELEMENTS(items));
}
}
if (protos->len == 0) {
fprintf(stderr, "No protocol registrations found.\n");
exit(1);
}
g_ptr_array_sort(protos, compare_symbols);
g_ptr_array_sort(handoffs, compare_symbols);
s = g_string_sized_new(STRING_RESERVED_SIZE);
g_string_append(s,
"/*\n"
" * Do not modify this file. Changes will be overwritten.\n"
" *\n"
" * Generated automatically using \"make-dissectors\".\n"
" */\n"
"\n"
"#include <dissectors.h>\n"
"\n");
g_string_append_printf(s,
"const gulong dissector_reg_proto_count = %d;\n"
"const gulong dissector_reg_handoff_count = %d;\n"
"\n",
protos->len, handoffs->len);
for (i = 0; i < protos->len; i++) {
g_string_append_printf(s,
"void %s(void);\n",
(char *)protos->pdata[i]);
}
g_string_append(s,
"\n"
"dissector_reg_t dissector_reg_proto[] = {\n");
for (i = 0; i < protos->len; i++) {
g_string_append_printf(s,
" { \"%s\", %s },\n",
(char *)protos->pdata[i], (char *)protos->pdata[i]);
}
g_string_append(s,
" { NULL, NULL }\n"
"};\n"
"\n");
for (i = 0; i < handoffs->len; i++) {
g_string_append_printf(s,
"void %s(void);\n",
(char *)handoffs->pdata[i]);
}
g_string_append(s,
"\n"
"dissector_reg_t dissector_reg_handoff[] = {\n");
for (i = 0; i < handoffs->len; i++) {
g_string_append_printf(s,
" { \"%s\", %s },\n",
(char *)handoffs->pdata[i], (char *)handoffs->pdata[i]);
}
g_string_append(s,
" { NULL, NULL }\n"
"};\n");
if (!g_file_set_contents(outfile, s->str, s->len, &err)) {
fprintf(stderr, "%s: %s\n", outfile, err->message);
exit(1);
}
count_protos = protos->len;
count_handoffs = handoffs->len;
g_string_free(s, TRUE);
g_regex_unref(protos_regex);
g_regex_unref(handoffs_regex);
g_ptr_array_free(protos, TRUE);
g_ptr_array_free(handoffs, TRUE);
printf("Found %u registrations and %u handoffs.\n",
count_protos, count_handoffs);
return 0;
}
/*
* Editor modelines
*
* Local Variables:
* c-basic-offset: 4
* tab-width: 8
* indent-tabs-mode: nil
* End:
*
* ex: set shiftwidth=4 tabstop=8 expandtab:
* :indentSize=4:tabSize=8:noTabs=true:
*/

View File

@ -1,89 +0,0 @@
/* make-lib.c
*
* Wireshark - Network traffic analyzer
* By Gerald Combs <gerald@wireshark.org>
* Copyright 1998 Gerald Combs
*
* SPDX-License-Identifier: GPL-2.0-or-later
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "make-lib.h"
#ifdef _WIN32
#define SEP "\r\n"
#else
#define SEP "\n"
#endif
int
compare_symbols(gconstpointer a, gconstpointer b)
{
return g_strcmp0(*(const char **)a, *(const char **)b);
}
static void
scan_matches(GRegex *regex, const char *string, GPtrArray *dst)
{
GMatchInfo *match_info;
char *match;
g_regex_match(regex, string, G_REGEX_MATCH_NOTEMPTY, &match_info);
while (g_match_info_matches(match_info)) {
match = g_match_info_fetch(match_info, 1);
g_ptr_array_add(dst, match);
g_match_info_next(match_info, NULL);
}
g_match_info_free(match_info);
}
void
scan_file(const char *file, struct symbol_item *items, size_t items_len)
{
char *contents;
GError *err = NULL;
size_t i;
if (!g_file_get_contents(file, &contents, NULL, &err)) {
fprintf(stderr, "%s: %s\n", file, err->message);
exit(1);
}
for (i = 0; i < items_len; i++) {
scan_matches(items[i].regex, contents, items[i].ptr_array);
}
g_free(contents);
}
void
scan_list(const char *file, struct symbol_item *items, size_t items_len)
{
char *contents, *arg;
GError *err = NULL;
if (!g_file_get_contents(file, &contents, NULL, &err)) {
fprintf(stderr, "%s: %s\n", file, err->message);
exit(1);
}
for (arg = strtok(contents, SEP); arg != NULL; arg = strtok(NULL, SEP)) {
scan_file(arg, items, items_len);
}
g_free(contents);
}
/*
* Editor modelines
*
* Local Variables:
* c-basic-offset: 4
* tab-width: 8
* indent-tabs-mode: nil
* End:
*
* ex: set shiftwidth=4 tabstop=8 expandtab:
* :indentSize=4:tabSize=8:noTabs=true:
*/

View File

@ -1,38 +0,0 @@
/* make-lib.h
*
* Wireshark - Network traffic analyzer
* By Gerald Combs <gerald@wireshark.org>
* Copyright 1998 Gerald Combs
*
* SPDX-License-Identifier: GPL-2.0-or-later
*/
#include <glib.h>
struct symbol_item {
GRegex *regex;
GPtrArray *ptr_array;
};
/* Compares symbols using strcmp() */
int compare_symbols(gconstpointer a, gconstpointer b);
/* Scan a C source file for symbols */
void scan_file(const char *file, struct symbol_item *items, size_t items_len);
/* Takes a text file containing a list of C source files on which to call scan_file() */
void scan_list(const char *file, struct symbol_item *items, size_t items_len);
/*
* Editor modelines
*
* Local Variables:
* c-basic-offset: 4
* tab-width: 8
* indent-tabs-mode: nil
* End:
*
* ex: set shiftwidth=4 tabstop=8 expandtab:
* :indentSize=4:tabSize=8:noTabs=true:
*/

127
tools/make-regs.py Executable file
View File

@ -0,0 +1,127 @@
#!/usr/bin/env python
#
# Looks for registration routines in the source files
# and assembles C code to call all the routines.
#
# SPDX-License-Identifier: GPL-2.0-or-later
#
import sys
import re
preamble = """\
/*
* Do not modify this file. Changes will be overwritten.
*
* Generated automatically using \"make-regs.py\".
*/
"""
def gen_prototypes(funcs):
output = ""
for f in funcs:
output += "void {}(void);\n".format(f)
return output
def gen_array(funcs, name):
output = "{}[] = {{\n".format(name)
for f in funcs:
output += " {{ \"{0}\", {0} }},\n".format(f)
output += " { NULL, NULL }\n};\n"
return output
def scan_files(infiles, regs):
for path in infiles:
with open(path, 'r') as f:
source = f.read()
for array, regex in regs:
matches = re.findall(regex, source)
array.extend(matches)
def make_dissectors(outfile, infiles):
protos = []
protos_regex = r"void\s+(proto_register_[\w]+)\s*\(\s*void\s*\)\s*{"
handoffs = []
handoffs_regex = r"void\s+(proto_reg_handoff_[\w]+)\s*\(\s*void\s*\)\s*{"
scan_files(infiles, [(protos, protos_regex), (handoffs, handoffs_regex)])
if len(protos) < 1:
sys.exit("No protocol registrations found.")
protos.sort()
handoffs.sort()
output = preamble
output += """\
#include "dissectors.h"
const gulong dissector_reg_proto_count = {0};
const gulong dissector_reg_handoff_count = {1};
""".format(len(protos), len(handoffs))
output += gen_prototypes(protos)
output += "\n"
output += gen_array(protos, "dissector_reg_t dissector_reg_proto")
output += "\n"
output += gen_prototypes(handoffs)
output += "\n"
output += gen_array(handoffs, "dissector_reg_t dissector_reg_handoff")
with open(outfile, "w") as f:
f.write(output)
print("Found {0} registrations and {1} handoffs.".format(len(protos), len(handoffs)))
def make_taps(outfile, infiles):
taps = []
taps_regex = r"void\s+(register_tap_listener_[\w]+)\s*\(\s*void\s*\)\s*{"
scan_files(infiles, [(taps, taps_regex)])
if len(taps) < 1:
sys.exit("No tap registrations found.")
taps.sort()
output = preamble
output += """\
#include "ui/taps.h"
const gulong tap_reg_listener_count = {0};
""".format(len(taps))
output += gen_prototypes(taps)
output += "\n"
output += gen_array(taps, "tap_reg_t tap_reg_listener")
with open(outfile, "w") as f:
f.write(output)
print("Found {0} registrations.".format(len(taps)))
def print_usage():
sys.exit("Usage: {0} <dissectors|taps> <outfile> <infiles...|@filelist>\n".format(sys.argv[0]))
if __name__ == "__main__":
if len(sys.argv) < 4:
print_usage()
mode = sys.argv[1]
outfile = sys.argv[2]
if sys.argv[3].startswith("@"):
with open(sys.argv[3][1:]) as f:
infiles = [l.strip() for l in f.readlines()]
else:
infiles = sys.argv[3:]
if mode == "dissectors":
make_dissectors(outfile, infiles)
elif mode == "taps":
make_taps(outfile, infiles)
else:
print_usage()

View File

@ -1,122 +0,0 @@
/* make-taps.c
* Tool to build the tap registration arrays.
*
* Wireshark - Network traffic analyzer
* By Gerald Combs <gerald@wireshark.org>
* Copyright 1998 Gerald Combs
*
* SPDX-License-Identifier: GPL-2.0-or-later
*/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "make-lib.h"
#define ARRAY_RESERVED_SIZE 128
#define STRING_RESERVED_SIZE (8 * 1024)
int main(int argc, char **argv)
{
GRegex *taps_regex;
GPtrArray *taps = NULL;
struct symbol_item items[1];
GError *err = NULL;
guint i;
GString *s;
const char *outfile;
guint count_taps;
if (argc < 3) {
fprintf(stderr, "Usage: %s <outfile> <infiles...>\n", argv[0]);
exit(1);
}
taps = g_ptr_array_new_full(ARRAY_RESERVED_SIZE, g_free);
taps_regex = g_regex_new("void\\s+(register_tap_listener_[[:alnum:]_]+)\\s*\\(\\s*void\\s*\\)\\s*{",
G_REGEX_OPTIMIZE, G_REGEX_MATCH_NOTEMPTY, &err);
if (err) {
fprintf(stderr, "GRegex: %s\n", err->message);
exit(1);
}
items[0].regex = taps_regex;
items[0].ptr_array = taps;
outfile = argv[1];
for (int arg = 2; arg < argc; arg++) {
scan_file(argv[arg], items, G_N_ELEMENTS(items));
}
if (taps->len == 0) {
fprintf(stderr, "No tap registrations found.\n");
exit(1);
}
g_ptr_array_sort(taps, compare_symbols);
s = g_string_sized_new(STRING_RESERVED_SIZE);
g_string_append(s,
"/*\n"
" * Do not modify this file. Changes will be overwritten.\n"
" *\n"
" * Generated automatically using \"make-taps\".\n"
" */\n"
"\n"
"#include \"ui/taps.h\"\n"
"\n");
g_string_append_printf(s,
"const gulong tap_reg_listener_count = %d;\n"
"\n",
taps->len);
for (i = 0; i < taps->len; i++) {
g_string_append_printf(s,
"void %s(void);\n",
(char *)taps->pdata[i]);
}
g_string_append(s,
"\n"
"tap_reg_t tap_reg_listener[] = {\n");
for (i = 0; i < taps->len; i++) {
g_string_append_printf(s,
" { \"%s\", %s },\n",
(char *)taps->pdata[i], (char *)taps->pdata[i]);
}
g_string_append(s,
" { NULL, NULL }\n"
"};\n"
"\n");
if (!g_file_set_contents(outfile, s->str, s->len, &err)) {
fprintf(stderr, "%s: %s\n", outfile, err->message);
exit(1);
}
count_taps = taps->len;
g_string_free(s, TRUE);
g_regex_unref(taps_regex);
g_ptr_array_free(taps, TRUE);
printf("Found %u registrations.\n", count_taps);
}
/*
* Editor modelines
*
* Local Variables:
* c-basic-offset: 4
* tab-width: 8
* indent-tabs-mode: nil
* End:
*
* ex: set shiftwidth=4 tabstop=8 expandtab:
* :indentSize=4:tabSize=8:noTabs=true:
*/

View File

@ -19,7 +19,6 @@ extcap/*
image/stock_icons/*
mmdbresolve.c
tools/lemon/*
tools/make-dissectors.c
wsutil/file_util.h
wsutil/strptime.c
wsutil/getopt_long.c