osmith
/
wireshark
forked from osmocom/wireshark
1
0
Fork 0
Code Pull Requests Releases Wiki Activity

tshark: improve -G elastic-mapping command by adding filters. 

The generated elastic mapping file is huge and it can hassle softwares
like Kibana. This change adds the ability to append desired filters
that will appear in the mapping file.

This change adds the option --elastic-mapping-filter <protocols> to tshark.

Example: tshark -G elastic-mapping --elastic-mapping-filter ip,udp,dns

make only those 3 protocols to appear in the mapping file.

Change-Id: Ie2dcd6e44be2d084e8e50cd6554bd90178da4e38
Reviewed-on: https://code.wireshark.org/review/27001
Reviewed-by: Peter Wu <peter@lekensteyn.nl>
Petri-Dish: Peter Wu <peter@lekensteyn.nl>
Tested-by: Petri Dish Buildbot
Reviewed-by: Dario Lombardo <lomato@gmail.com>
This commit is contained in:
Dario Lombardo 2018-04-19 13:58:43 +02:00
parent ca45f88e3b
commit fc6b8ab698
4 changed files with 71 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
doc/tshark.pod
View File

@ -66,7 +66,7 @@ S<[ B<--disable-heuristic> E<lt>short_nameE<gt> ]>
S<[ E<lt>capture filterE<gt> ]>
B<tshark>
B<-G> [ E<lt>report typeE<gt> ]
B<-G> [ E<lt>report typeE<gt> ] [ --elastic-mapping-filter E<lt>protocolsE<gt> ]
=head1 DESCRIPTION
@ -839,7 +839,12 @@ Example of usage to import data into Elasticsearch:
Elastic requires a mapping file to be loaded as template for packets-*
index in order to convert wireshark types to elastic types. This file
can be auto-generated with the command "tshark -G elastic-mapping".
can be auto-generated with the command "tshark -G elastic-mapping". Since
the mapping file can be huge, protocols can be selected by using the option
--elastic-mapping-filter:
tshark -G elastic-mapping --elastic-mapping-filter ip,udp,dns
B<fields> The values of fields specified with the B<-e> option, in a
form specified by the B<-E> option. For example,
@ -1743,6 +1748,14 @@ If a key appears multiple times in an object, only write it a single time with
as value a json array containing all the separate values. (Only works with
-T json)
=item --elastic-mapping-filter E<lt>protocolE<gt>,E<lt>protocolE<gt>,...
When generating the ElasticSearch mapping file, only put the specified protocols
in it, to avoid a huge mapping file that can choke some software (such as Kibana).
The option takes a list of wanted protocol abbreviations, separated by comma.
Example: ip,udp,dns puts only those three protocols in the mapping file.
=item --export-objects E<lt>protocolE<gt>,E<lt>destdirE<gt>
Export all objects within a protocol into directory B<destdir>. The available

40
epan/proto.c
View File

@ -10052,7 +10052,7 @@ dot_to_underscore(gchar* str)
/* Dumps a mapping file for ElasticSearch
*/
void
proto_registrar_dump_elastic(void)
proto_registrar_dump_elastic(const gchar* filter)
{
header_field_info *hfinfo;
header_field_info *parent_hfinfo;
@ -10065,11 +10065,20 @@ proto_registrar_dump_elastic(void)
const char* prev_proto = NULL;
gchar* data;
gchar* str;
gchar** protos = NULL;
gchar* proto;
gboolean found;
guint j;
/* We have filtering protocols. Extract them. */
if (filter) {
protos = g_strsplit(filter, ",", -1);
}
/*
To help traking down the json tree, objects have been appended with a comment:
n.label -> where n is the indentation level and label the name of the object
*/
* To help tracking down the json tree, objects have been appended with a comment:
* n.label -> where n is the indentation level and label the name of the object
*/
builder = json_builder_new();
json_builder_begin_object(builder); // 1.root
@ -10104,7 +10113,7 @@ proto_registrar_dump_elastic(void)
/*
* Skip the pseudo-field for "proto_tree_add_text()" since
* we don't want it in the list of filterable fields.
* we don't want it in the list of filterable protocols.
*/
if (hfinfo->id == hf_text_only)
continue;
@ -10112,6 +10121,26 @@ proto_registrar_dump_elastic(void)
if (!proto_registrar_is_protocol(i)) {
PROTO_REGISTRAR_GET_NTH(hfinfo->parent, parent_hfinfo);
/*
* Skip the field if filter protocols have been set and this one's
* parent is not listed.
*/
if (protos) {
found = FALSE;
j = 0;
proto = protos[0];
while(proto) {
if (!g_strcmp0(proto, parent_hfinfo->abbrev)) {
found = TRUE;
break;
}
j++;
proto = protos[j];
}
if (!found)
continue;
}
if (prev_proto && g_strcmp0(parent_hfinfo->abbrev, prev_proto)) {
json_builder_end_object(builder); // 8.properties
json_builder_end_object(builder); // 7.parent_hfinfo->abbrev
@ -10159,6 +10188,7 @@ proto_registrar_dump_elastic(void)
g_object_unref(generator);
ws_debug_printf("%s\n", data);
g_free(data);
g_strfreev(protos);
}
#endif

2
epan/proto.h
View File

@ -2455,7 +2455,7 @@ WS_DLL_PUBLIC void proto_registrar_dump_values(void);
#ifdef HAVE_JSONGLIB
/** Dumps a mapping file for loading tshark output into ElasticSearch */
WS_DLL_PUBLIC void proto_registrar_dump_elastic(void);
WS_DLL_PUBLIC void proto_registrar_dump_elastic(const gchar* filter);
#endif
/** Dumps the number of protocol and field registrations to STDOUT.

22
tshark.c
View File

@ -142,6 +142,9 @@
*/
#define LONGOPT_COLOR (65536+1000)
#define LONGOPT_NO_DUPLICATE_KEYS (65536+1001)
#ifdef HAVE_JSONGLIB
#define LONGOPT_ELASTIC_MAPPING_FILTER (65536+1002)
#endif
#if 0
#define tshark_debug(...) g_warning(__VA_ARGS__)
@ -439,7 +442,11 @@ print_usage(FILE *output)
fprintf(output, " (Note that attributes are nonstandard)\n");
fprintf(output, " --no-duplicate-keys If -T json is specified, merge duplicate keys in an object\n");
fprintf(output, " into a single key with as value a json array containing all\n");
fprintf(output, " values");
fprintf(output, " values\n");
#ifdef HAVE_JSONGLIB
fprintf(output, " --elastic-mapping-filter <protocols> If -G elastic-mapping is specified, put only the\n");
fprintf(output, " specified protocols within the mapping file\n");
#endif
fprintf(output, "\n");
fprintf(output, "Miscellaneous:\n");
@ -680,6 +687,9 @@ main(int argc, char *argv[])
{"export-objects", required_argument, NULL, LONGOPT_EXPORT_OBJECTS},
{"color", no_argument, NULL, LONGOPT_COLOR},
{"no-duplicate-keys", no_argument, NULL, LONGOPT_NO_DUPLICATE_KEYS},
#ifdef HAVE_JSONGLIB
{"elastic-mapping-filter", required_argument, NULL, LONGOPT_ELASTIC_MAPPING_FILTER},
#endif
{0, 0, 0, 0 }
};
gboolean arg_error = FALSE;
@ -723,6 +733,9 @@ main(int argc, char *argv[])
gchar *volatile pdu_export_arg = NULL;
const char *volatile exp_pdu_filename = NULL;
exp_pdu_t exp_pdu_tap_data;
#ifdef HAVE_JSONGLIB
const gchar* elastic_mapping_filter = NULL;
#endif
/*
* The leading + ensures that getopt_long() does not permute the argv[]
@ -864,6 +877,11 @@ main(int argc, char *argv[])
case 'X':
ex_opt_add(optarg);
break;
#ifdef HAVE_JSONGLIB
case LONGOPT_ELASTIC_MAPPING_FILTER:
elastic_mapping_filter = optarg;
break;
#endif
default:
break;
}
@ -969,7 +987,7 @@ main(int argc, char *argv[])
dissector_dump_dissector_tables();
#ifdef HAVE_JSONGLIB
else if (strcmp(argv[2], "elastic-mapping") == 0)
proto_registrar_dump_elastic();
proto_registrar_dump_elastic(elastic_mapping_filter);
#endif
else if (strcmp(argv[2], "fieldcount") == 0) {
/* return value for the test suite */