forked from osmocom/wireshark
Tshark: Optional packet summary for Elasticsearch
Currently, the Elasticsearch output exports the packet details and, if -x is specified, the raw hex data. This change adds the option of exporting the packet summary as well. The default stays the same (packet details only), but now the existing -P switch turns on printing of the packet summary. It also turns off printing packet details, which can be turned back on with -V to print both, and combined with -x to print all three: summary, details and raw hex. The packet summary is especially useful when exploring and visualizing the data in Kibana, e.g. by displaying the summary "Info" field/column in a table, as in the Wireshark GUI. Change-Id: I2030490cfdd905572397bc3d5457ba49d805a5c4 Reviewed-on: https://code.wireshark.org/review/22716 Petri-Dish: Michael Mann <mmann78@netscape.net> Tested-by: Petri Dish Buildbot <buildbot-no-reply@wireshark.org> Reviewed-by: Michael Mann <mmann78@netscape.net>
This commit is contained in:
parent
ddf6526256
commit
af09db8bd3
|
@ -771,11 +771,14 @@ options are one of:
|
||||||
|
|
||||||
B<ek> Newline delimited JSON format for bulk import into Elasticsearch.
|
B<ek> Newline delimited JSON format for bulk import into Elasticsearch.
|
||||||
It can be used with B<-j> or B<-J> including the JSON filter or with
|
It can be used with B<-j> or B<-J> including the JSON filter or with
|
||||||
B<-x> flag to include raw hex-encoded packet data.
|
B<-x> to include raw hex-encoded packet data.
|
||||||
|
If B<-P> is specified it will print the packet summary only, with both
|
||||||
|
B<-P> and B<-V> it will print the packet summary and packet details.
|
||||||
|
If neither B<-P> or B<-V> are used it will print the packet details only.
|
||||||
Example of usage to import data into Elasticsearch:
|
Example of usage to import data into Elasticsearch:
|
||||||
|
|
||||||
tshark -T ek -j "http tcp ip" -x -r file.pcap > file.json
|
tshark -T ek -j "http tcp ip" -P -V -x -r file.pcap > file.json
|
||||||
curl -XPUT http://elasticsearch:9200/_bulk --data-binary @file.json
|
curl -H "Content-Type: application/x-ndjson" -XPOST http://elasticsearch:9200/_bulk --data-binary "@file.json"
|
||||||
|
|
||||||
B<fields> The values of fields specified with the B<-e> option, in a
|
B<fields> The values of fields specified with the B<-e> option, in a
|
||||||
form specified by the B<-E> option. For example,
|
form specified by the B<-E> option. For example,
|
||||||
|
|
26
epan/print.c
26
epan/print.c
|
@ -128,6 +128,7 @@ static void write_json_proto_node_no_value(proto_node *node, write_json_data *da
|
||||||
static const char *proto_node_to_json_key(proto_node *node);
|
static const char *proto_node_to_json_key(proto_node *node);
|
||||||
|
|
||||||
static void print_pdml_geninfo(epan_dissect_t *edt, FILE *fh);
|
static void print_pdml_geninfo(epan_dissect_t *edt, FILE *fh);
|
||||||
|
static void write_ek_summary(column_info *cinfo, FILE *fh);
|
||||||
|
|
||||||
static void proto_tree_get_node_field_values(proto_node *node, gpointer data);
|
static void proto_tree_get_node_field_values(proto_node *node, gpointer data);
|
||||||
|
|
||||||
|
@ -352,7 +353,8 @@ write_pdml_proto_tree(output_fields_t* fields, gchar **protocolfilter, pf_flags
|
||||||
|
|
||||||
void
|
void
|
||||||
write_ek_proto_tree(output_fields_t* fields,
|
write_ek_proto_tree(output_fields_t* fields,
|
||||||
gboolean print_hex, gchar **protocolfilter,
|
gboolean print_summary, gboolean print_hex,
|
||||||
|
gchar **protocolfilter,
|
||||||
pf_flags protocolfilter_flags, epan_dissect_t *edt,
|
pf_flags protocolfilter_flags, epan_dissect_t *edt,
|
||||||
FILE *fh)
|
FILE *fh)
|
||||||
{
|
{
|
||||||
|
@ -373,7 +375,12 @@ write_ek_proto_tree(output_fields_t* fields,
|
||||||
|
|
||||||
fprintf(fh, "{\"index\" : {\"_index\": \"packets-%s\", \"_type\": \"pcap_file\", \"_score\": null}}\n", ts);
|
fprintf(fh, "{\"index\" : {\"_index\": \"packets-%s\", \"_type\": \"pcap_file\", \"_score\": null}}\n", ts);
|
||||||
/* Timestamp added for time indexing in Elasticsearch */
|
/* Timestamp added for time indexing in Elasticsearch */
|
||||||
fprintf(fh, "{\"timestamp\" : \"%" G_GUINT64_FORMAT "%03d\", \"layers\" : {", (guint64)edt->pi.abs_ts.secs, edt->pi.abs_ts.nsecs/1000000);
|
fprintf(fh, "{\"timestamp\" : \"%" G_GUINT64_FORMAT "%03d\"", (guint64)edt->pi.abs_ts.secs, edt->pi.abs_ts.nsecs/1000000);
|
||||||
|
|
||||||
|
if (print_summary)
|
||||||
|
write_ek_summary(edt->pi.cinfo, fh);
|
||||||
|
|
||||||
|
fprintf(fh, ", \"layers\" : {");
|
||||||
|
|
||||||
if (fields == NULL || fields->fields == NULL) {
|
if (fields == NULL || fields->fields == NULL) {
|
||||||
/* Write out all fields */
|
/* Write out all fields */
|
||||||
|
@ -1157,6 +1164,21 @@ ek_check_protocolfilter(gchar **protocolfilter, const char *str)
|
||||||
* Finds a node's descendants to be printed as EK/JSON attributes.
|
* Finds a node's descendants to be printed as EK/JSON attributes.
|
||||||
*/
|
*/
|
||||||
static void
|
static void
|
||||||
|
write_ek_summary(column_info *cinfo, FILE *fh)
|
||||||
|
{
|
||||||
|
gint i;
|
||||||
|
|
||||||
|
for (i = 0; i < cinfo->num_cols; i++) {
|
||||||
|
fputs(", \"", fh);
|
||||||
|
print_escaped_ek(fh, g_ascii_strdown(cinfo->columns[i].col_title, -1));
|
||||||
|
fputs("\": \"", fh);
|
||||||
|
print_escaped_json(fh, cinfo->columns[i].col_data);
|
||||||
|
fputs("\"", fh);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Write out a tree's data, and any child nodes, as JSON for EK */
|
||||||
|
static void
|
||||||
ek_fill_attr(proto_node *node, GSList **attr_list, GHashTable *attr_table, write_json_data *pdata)
|
ek_fill_attr(proto_node *node, GSList **attr_list, GHashTable *attr_table, write_json_data *pdata)
|
||||||
{
|
{
|
||||||
field_info *fi = NULL;
|
field_info *fi = NULL;
|
||||||
|
|
|
@ -115,6 +115,7 @@ WS_DLL_PUBLIC void write_json_proto_tree(output_fields_t* fields,
|
||||||
WS_DLL_PUBLIC void write_json_finale(FILE *fh);
|
WS_DLL_PUBLIC void write_json_finale(FILE *fh);
|
||||||
|
|
||||||
WS_DLL_PUBLIC void write_ek_proto_tree(output_fields_t* fields,
|
WS_DLL_PUBLIC void write_ek_proto_tree(output_fields_t* fields,
|
||||||
|
gboolean print_summary,
|
||||||
gboolean print_hex_data,
|
gboolean print_hex_data,
|
||||||
gchar **protocolfilter,
|
gchar **protocolfilter,
|
||||||
pf_flags protocolfilter_flags,
|
pf_flags protocolfilter_flags,
|
||||||
|
|
82
tshark.c
82
tshark.c
|
@ -193,7 +193,7 @@ typedef enum {
|
||||||
static output_action_e output_action;
|
static output_action_e output_action;
|
||||||
static gboolean do_dissection; /* TRUE if we have to dissect each packet */
|
static gboolean do_dissection; /* TRUE if we have to dissect each packet */
|
||||||
static gboolean print_packet_info; /* TRUE if we're to print packet information */
|
static gboolean print_packet_info; /* TRUE if we're to print packet information */
|
||||||
static gint print_summary = -1; /* TRUE if we're to print packet summary information */
|
static gboolean print_summary; /* TRUE if we're to print packet summary information */
|
||||||
static gboolean print_details; /* TRUE if we're to print packet details information */
|
static gboolean print_details; /* TRUE if we're to print packet details information */
|
||||||
static gboolean print_hex; /* TRUE if we're to print hex/ascci information */
|
static gboolean print_hex; /* TRUE if we're to print hex/ascci information */
|
||||||
static gboolean line_buffered;
|
static gboolean line_buffered;
|
||||||
|
@ -872,15 +872,6 @@ main(int argc, char *argv[])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Print packet summary information is the default, unless either -V or -x
|
|
||||||
* were specified and -P was not. Note that this is new behavior, which
|
|
||||||
* allows for the possibility of printing only hex/ascii output without
|
|
||||||
* necessarily requiring that either the summary or details be printed too.
|
|
||||||
*/
|
|
||||||
if (print_summary == -1)
|
|
||||||
print_summary = (print_details || print_hex) ? FALSE : TRUE;
|
|
||||||
|
|
||||||
/** Send All g_log messages to our own handler **/
|
/** Send All g_log messages to our own handler **/
|
||||||
|
|
||||||
log_flags =
|
log_flags =
|
||||||
|
@ -1323,8 +1314,8 @@ main(int argc, char *argv[])
|
||||||
print_summary = FALSE; /* Don't allow summary */
|
print_summary = FALSE; /* Don't allow summary */
|
||||||
} else if (strcmp(optarg, "ek") == 0) {
|
} else if (strcmp(optarg, "ek") == 0) {
|
||||||
output_action = WRITE_EK;
|
output_action = WRITE_EK;
|
||||||
print_details = TRUE; /* Need details */
|
if (!print_summary)
|
||||||
print_summary = FALSE; /* Don't allow summary */
|
print_details = TRUE;
|
||||||
} else if (strcmp(optarg, "jsonraw") == 0) {
|
} else if (strcmp(optarg, "jsonraw") == 0) {
|
||||||
output_action = WRITE_JSON_RAW;
|
output_action = WRITE_JSON_RAW;
|
||||||
print_details = TRUE; /* Need details */
|
print_details = TRUE; /* Need details */
|
||||||
|
@ -1480,6 +1471,15 @@ main(int argc, char *argv[])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Print packet summary information is the default if neither -V or -x
|
||||||
|
* were specified. Note that this is new behavior, which allows for the
|
||||||
|
* possibility of printing only hex/ascii output without necessarily
|
||||||
|
* requiring that either the summary or details be printed too.
|
||||||
|
*/
|
||||||
|
if (!print_summary && !print_details && !print_hex)
|
||||||
|
print_summary = TRUE;
|
||||||
|
|
||||||
if (no_duplicate_keys && output_action != WRITE_JSON && output_action != WRITE_JSON_RAW) {
|
if (no_duplicate_keys && output_action != WRITE_JSON && output_action != WRITE_JSON_RAW) {
|
||||||
cmdarg_err("--no-duplicate-keys can only be used with \"-T json\" and \"-T jsonraw\"");
|
cmdarg_err("--no-duplicate-keys can only be used with \"-T json\" and \"-T jsonraw\"");
|
||||||
exit_status = INVALID_OPTION;
|
exit_status = INVALID_OPTION;
|
||||||
|
@ -3899,36 +3899,17 @@ print_columns(capture_file *cf, const epan_dissect_t *edt)
|
||||||
static gboolean
|
static gboolean
|
||||||
print_packet(capture_file *cf, epan_dissect_t *edt)
|
print_packet(capture_file *cf, epan_dissect_t *edt)
|
||||||
{
|
{
|
||||||
if (print_summary || output_fields_has_cols(output_fields)) {
|
if (print_summary || output_fields_has_cols(output_fields))
|
||||||
/* Just fill in the columns. */
|
/* Just fill in the columns. */
|
||||||
epan_dissect_fill_in_columns(edt, FALSE, TRUE);
|
epan_dissect_fill_in_columns(edt, FALSE, TRUE);
|
||||||
|
|
||||||
if (print_summary) {
|
/* Print summary columns and/or protocol tree */
|
||||||
/* Now print them. */
|
|
||||||
switch (output_action) {
|
switch (output_action) {
|
||||||
|
|
||||||
case WRITE_TEXT:
|
case WRITE_TEXT:
|
||||||
if (!print_columns(cf, edt))
|
if (print_summary && !print_columns(cf, edt))
|
||||||
return FALSE;
|
return FALSE;
|
||||||
break;
|
|
||||||
|
|
||||||
case WRITE_XML:
|
|
||||||
write_psml_columns(edt, stdout, dissect_color);
|
|
||||||
return !ferror(stdout);
|
|
||||||
case WRITE_FIELDS: /*No non-verbose "fields" format */
|
|
||||||
case WRITE_JSON:
|
|
||||||
case WRITE_EK:
|
|
||||||
case WRITE_JSON_RAW:
|
|
||||||
g_assert_not_reached();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (print_details) {
|
if (print_details) {
|
||||||
/* Print the information in the protocol tree. */
|
|
||||||
switch (output_action) {
|
|
||||||
|
|
||||||
case WRITE_TEXT:
|
|
||||||
if (!proto_tree_print(print_details ? print_dissections_expanded : print_dissections_none,
|
if (!proto_tree_print(print_details ? print_dissections_expanded : print_dissections_none,
|
||||||
print_hex, edt, output_only_tables, print_stream))
|
print_hex, edt, output_only_tables, print_stream))
|
||||||
return FALSE;
|
return FALSE;
|
||||||
|
@ -3936,32 +3917,61 @@ print_packet(capture_file *cf, epan_dissect_t *edt)
|
||||||
if (!print_line(print_stream, 0, separator))
|
if (!print_line(print_stream, 0, separator))
|
||||||
return FALSE;
|
return FALSE;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case WRITE_XML:
|
case WRITE_XML:
|
||||||
|
if (print_summary) {
|
||||||
|
write_psml_columns(edt, stdout, dissect_color);
|
||||||
|
return !ferror(stdout);
|
||||||
|
}
|
||||||
|
if (print_details) {
|
||||||
write_pdml_proto_tree(output_fields, protocolfilter, protocolfilter_flags, edt, stdout, dissect_color);
|
write_pdml_proto_tree(output_fields, protocolfilter, protocolfilter_flags, edt, stdout, dissect_color);
|
||||||
printf("\n");
|
printf("\n");
|
||||||
return !ferror(stdout);
|
return !ferror(stdout);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
case WRITE_FIELDS:
|
case WRITE_FIELDS:
|
||||||
|
if (print_summary) {
|
||||||
|
/*No non-verbose "fields" format */
|
||||||
|
g_assert_not_reached();
|
||||||
|
}
|
||||||
|
if (print_details) {
|
||||||
write_fields_proto_tree(output_fields, edt, &cf->cinfo, stdout);
|
write_fields_proto_tree(output_fields, edt, &cf->cinfo, stdout);
|
||||||
printf("\n");
|
printf("\n");
|
||||||
return !ferror(stdout);
|
return !ferror(stdout);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
case WRITE_JSON:
|
case WRITE_JSON:
|
||||||
|
if (print_summary)
|
||||||
|
g_assert_not_reached();
|
||||||
|
if (print_details) {
|
||||||
write_json_proto_tree(output_fields, print_dissections_expanded,
|
write_json_proto_tree(output_fields, print_dissections_expanded,
|
||||||
print_hex, protocolfilter, protocolfilter_flags,
|
print_hex, protocolfilter, protocolfilter_flags,
|
||||||
edt, node_children_grouper, stdout);
|
edt, node_children_grouper, stdout);
|
||||||
return !ferror(stdout);
|
return !ferror(stdout);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
case WRITE_JSON_RAW:
|
case WRITE_JSON_RAW:
|
||||||
|
if (print_summary)
|
||||||
|
g_assert_not_reached();
|
||||||
|
if (print_details) {
|
||||||
write_json_proto_tree(output_fields, print_dissections_none, TRUE,
|
write_json_proto_tree(output_fields, print_dissections_none, TRUE,
|
||||||
protocolfilter, protocolfilter_flags,
|
protocolfilter, protocolfilter_flags,
|
||||||
edt, node_children_grouper, stdout);
|
edt, node_children_grouper, stdout);
|
||||||
return !ferror(stdout);
|
return !ferror(stdout);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
case WRITE_EK:
|
case WRITE_EK:
|
||||||
write_ek_proto_tree(output_fields, print_hex, protocolfilter,
|
write_ek_proto_tree(output_fields, print_summary, print_hex, protocolfilter,
|
||||||
protocolfilter_flags, edt, stdout);
|
protocolfilter_flags, edt, stdout);
|
||||||
return !ferror(stdout);
|
return !ferror(stdout);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if (print_hex) {
|
if (print_hex) {
|
||||||
if (print_summary || print_details) {
|
if (print_summary || print_details) {
|
||||||
if (!print_line(print_stream, 0, ""))
|
if (!print_line(print_stream, 0, ""))
|
||||||
|
|
Loading…
Reference in New Issue