json2pcap support added
Modified tshark -T json -x output Added tshark -T jsonraw output json2pcap.py (can be used for basic packet editing by modifying json) The modification in tshark -T json -x and new tshark -T jsonraw output add into hex-data output in JSON also information on which position each field is dissected in the original frame, what is the field length, bitmask (for not byte aligned fields) and type. This information can be used for latter processing. One use-case is json2pcap script which assembles the protocol layers back together from upper to lowers layers, which allows the basic packet modification/editing/rewriting. Change-Id: Ibf948eb8fc7e3b0b51c12df6c3855f705a9c7925 Reviewed-on: https://code.wireshark.org/review/19990 Petri-Dish: Peter Wu <peter@lekensteyn.nl> Reviewed-by: Dario Lombardo <lomato@gmail.com> Petri-Dish: Pascal Quantin <pascal.quantin@gmail.com> Tested-by: Petri Dish Buildbot <buildbot-no-reply@wireshark.org> Reviewed-by: Peter Wu <peter@lekensteyn.nl>
This commit is contained in:
Martin Kacer
Peter Wu
parent
d00dae7af9
commit
cd7d159c2e
|
|
@ -3685,6 +3685,7 @@ Barbu Paul - Gheorghe <barbu.paul.gheorghe[AT]gmail.com> {
|
|||
|
||||
Martin Kacer <kacer.martin[AT]gmail.com> {
|
||||
JSON and Elasticsearch tshark output
|
||||
json2pcap
|
||||
}
|
||||
Ben Stewart <bst[AT]google.com> {
|
||||
SCTE-35 dissector
|
||||
|
|
|
|||
|
|
@ -545,11 +545,18 @@ this option.
|
|||
|
||||
=item -j E<lt>protocol match filterE<gt>
|
||||
|
||||
Protocol match filter used for ek|json|pdml output file types.
|
||||
Protocol match filter used for ek|json|jsonraw|pdml output file types.
|
||||
Parent node containing multiple child nodes is only included,
|
||||
if the name is found in the filter.
|
||||
|
||||
Example: B<-j "http tcp ip">
|
||||
Example: B<-j "ip ip.flags text">
|
||||
|
||||
=item -J E<lt>protocol match filterE<gt>
|
||||
|
||||
Protocol top level filter used for ek|json|jsonraw|pdml output file types.
|
||||
Parent node containing multiple child nodes is included with all children.
|
||||
|
||||
Example: B<-J "http tcp">
|
||||
|
||||
=item -K E<lt>keytabE<gt>
|
||||
|
||||
|
|
@ -741,14 +748,14 @@ was captured
|
|||
|
||||
The default format is relative.
|
||||
|
||||
=item -T ek|fields|json|pdml|ps|psml|text
|
||||
=item -T ek|fields|json|jsonraw|pdml|ps|psml|text
|
||||
|
||||
Set the format of the output when viewing decoded packet data. The
|
||||
options are one of:
|
||||
|
||||
B<ek> Newline delimited JSON format for bulk import into Elasticsearch.
|
||||
It can be used with B<-j> including the JSON filter or with B<-x> flag
|
||||
to include raw packet data.
|
||||
It can be used with B<-j> or B<-J> including the JSON filter or with
|
||||
B<-x> flag to include raw hex-encoded packet data.
|
||||
Example of usage to import data into Elasticsearch:
|
||||
|
||||
tshark -T ek -j "http tcp ip" -x -r file.pcap > file.json
|
||||
|
|
@ -762,13 +769,20 @@ form specified by the B<-E> option. For example,
|
|||
would generate comma-separated values (CSV) output suitable for importing
|
||||
into your favorite spreadsheet program.
|
||||
|
||||
B<json> JSON file format. It can be used with B<-j> including
|
||||
the JSON filter or with B<-x> flag to influde raw packet data.
|
||||
B<json> JSON file format. It can be used with B<-j> or B<-J> including
|
||||
the JSON filter or with B<-x> flag to include raw hex-encoded packet data.
|
||||
Example of usage:
|
||||
|
||||
tshark -T json -r file.pcap
|
||||
tshark -T json -j "http tcp ip" -x -r file.pcap
|
||||
|
||||
B<jsonraw> JSON file format including only raw hex-encoded packet data.
|
||||
It can be used with B<-j> including or B<-J >the JSON filter flag.
|
||||
Example of usage:
|
||||
|
||||
tshark -T jsonraw -r file.pcap
|
||||
tshark -T jsonraw -j "http tcp ip" -x -r file.pcap
|
||||
|
||||
B<pdml> Packet Details Markup Language, an XML-based format for the details of
|
||||
a decoded packet. This information is equivalent to the packet details
|
||||
printed with the B<-V> flag.
|
||||
|
|
|
|||
112
epan/print.c
112
epan/print.c
|
|
@ -60,7 +60,7 @@ typedef struct {
|
|||
FILE *fh;
|
||||
GSList *src_list;
|
||||
gchar **filter;
|
||||
pf_flags filter_flags;
|
||||
pf_flags filter_flags;
|
||||
} write_pdml_data;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -68,8 +68,9 @@ typedef struct {
|
|||
FILE *fh;
|
||||
GSList *src_list;
|
||||
gchar **filter;
|
||||
pf_flags filter_flags;
|
||||
pf_flags filter_flags;
|
||||
gboolean print_hex;
|
||||
gboolean print_text;
|
||||
} write_json_data;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -373,6 +374,10 @@ write_json_proto_tree(output_fields_t* fields, print_args_t *print_args, gchar *
|
|||
data.filter = protocolfilter;
|
||||
data.filter_flags = protocolfilter_flags;
|
||||
data.print_hex = print_args->print_hex;
|
||||
data.print_text = TRUE;
|
||||
if (print_args->print_dissections == print_dissections_none) {
|
||||
data.print_text = FALSE;
|
||||
}
|
||||
|
||||
proto_tree_children_foreach(edt->tree, proto_tree_write_node_json,
|
||||
&data);
|
||||
|
|
@ -721,10 +726,10 @@ proto_tree_write_node_json(proto_node *node, gpointer data)
|
|||
/* dissection with an invisible proto tree? */
|
||||
g_assert(fi);
|
||||
|
||||
print_indent(pdata->level + 3, pdata->fh);
|
||||
|
||||
/* Text label. It's printed as a field with no name. */
|
||||
if (fi->hfinfo->id == hf_text_only) {
|
||||
print_indent(pdata->level + 3, pdata->fh);
|
||||
|
||||
/* Get the text */
|
||||
if (fi->rep) {
|
||||
label_ptr = fi->rep->representation;
|
||||
|
|
@ -754,11 +759,13 @@ proto_tree_write_node_json(proto_node *node, gpointer data)
|
|||
/*
|
||||
* Hex dump -x
|
||||
*/
|
||||
if (pdata->print_hex && fi->length > 0) {
|
||||
if (pdata->print_hex && (!pdata->print_text || fi->length > 0)) {
|
||||
print_indent(pdata->level + 3, pdata->fh);
|
||||
|
||||
fputs("\"", pdata->fh);
|
||||
print_escaped_json(pdata->fh, fi->hfinfo->abbrev);
|
||||
fputs("_raw", pdata->fh);
|
||||
fputs("\": \"", pdata->fh);
|
||||
fputs("\": [\"", pdata->fh);
|
||||
|
||||
if (fi->hfinfo->bitmask!=0) {
|
||||
switch (fi->value.ftype->ftype) {
|
||||
|
|
@ -790,28 +797,47 @@ proto_tree_write_node_json(proto_node *node, gpointer data)
|
|||
default:
|
||||
g_assert_not_reached();
|
||||
}
|
||||
fputs("\",\n", pdata->fh);
|
||||
}
|
||||
else {
|
||||
json_write_field_hex_value(pdata, fi);
|
||||
fputs("\",\n", pdata->fh);
|
||||
}
|
||||
|
||||
print_indent(pdata->level + 3, pdata->fh);
|
||||
/* Dump raw hex-encoded dissected information including position, length, bitmask, type */
|
||||
fprintf(pdata->fh, "\", %" G_GINT32_MODIFIER "d", fi->start);
|
||||
fprintf(pdata->fh, ", %" G_GINT32_MODIFIER "d", fi->length);
|
||||
fprintf(pdata->fh, ", %" G_GUINT64_FORMAT, fi->hfinfo->bitmask);
|
||||
fprintf(pdata->fh, ", %" G_GINT32_MODIFIER "d", (gint32)fi->value.ftype->ftype);
|
||||
|
||||
if (pdata->print_text) {
|
||||
fputs("],\n", pdata->fh);
|
||||
} else {
|
||||
if (node->next == NULL && node->first_child == NULL) {
|
||||
fputs("]\n", pdata->fh);
|
||||
} else {
|
||||
fputs("],\n", pdata->fh);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
fputs("\"", pdata->fh);
|
||||
|
||||
print_escaped_json(pdata->fh, fi->hfinfo->abbrev);
|
||||
|
||||
/* show, value, and unmaskedvalue attributes */
|
||||
switch (fi->hfinfo->type)
|
||||
{
|
||||
case FT_PROTOCOL:
|
||||
if (node->first_child != NULL) {
|
||||
print_indent(pdata->level + 3, pdata->fh);
|
||||
|
||||
fputs("\"", pdata->fh);
|
||||
print_escaped_json(pdata->fh, fi->hfinfo->abbrev);
|
||||
|
||||
fputs("\": {\n", pdata->fh);
|
||||
} else {
|
||||
} else if (pdata->print_text) {
|
||||
print_indent(pdata->level + 3, pdata->fh);
|
||||
|
||||
fputs("\"", pdata->fh);
|
||||
print_escaped_json(pdata->fh, fi->hfinfo->abbrev);
|
||||
|
||||
fputs("\": \"", pdata->fh);
|
||||
if (fi->rep) {
|
||||
print_escaped_json(pdata->fh, fi->rep->representation);
|
||||
|
|
@ -830,8 +856,18 @@ proto_tree_write_node_json(proto_node *node, gpointer data)
|
|||
break;
|
||||
case FT_NONE:
|
||||
if (node->first_child != NULL) {
|
||||
print_indent(pdata->level + 3, pdata->fh);
|
||||
|
||||
fputs("\"", pdata->fh);
|
||||
print_escaped_json(pdata->fh, fi->hfinfo->abbrev);
|
||||
|
||||
fputs("\": {\n", pdata->fh);
|
||||
} else {
|
||||
} else if (pdata->print_text) {
|
||||
print_indent(pdata->level + 3, pdata->fh);
|
||||
|
||||
fputs("\"", pdata->fh);
|
||||
print_escaped_json(pdata->fh, fi->hfinfo->abbrev);
|
||||
|
||||
if (node->next == NULL) {
|
||||
fputs("\": \"\"\n", pdata->fh);
|
||||
} else {
|
||||
|
|
@ -840,27 +876,39 @@ proto_tree_write_node_json(proto_node *node, gpointer data)
|
|||
}
|
||||
break;
|
||||
default:
|
||||
dfilter_string = fvalue_to_string_repr(NULL, &fi->value, FTREPR_DISPLAY, fi->hfinfo->display);
|
||||
if (dfilter_string != NULL) {
|
||||
fputs("\": \"", pdata->fh);
|
||||
print_escaped_json(pdata->fh, dfilter_string);
|
||||
if (node->first_child != NULL) {
|
||||
fputs("\",\n", pdata->fh);
|
||||
print_indent(pdata->level + 3, pdata->fh);
|
||||
if (pdata->print_text) {
|
||||
print_indent(pdata->level + 3, pdata->fh);
|
||||
|
||||
fputs("\"", pdata->fh);
|
||||
print_escaped_json(pdata->fh, fi->hfinfo->abbrev);
|
||||
fputs("_tree\": {\n", pdata->fh);
|
||||
fputs("\"", pdata->fh);
|
||||
print_escaped_json(pdata->fh, fi->hfinfo->abbrev);
|
||||
|
||||
dfilter_string = fvalue_to_string_repr(NULL, &fi->value, FTREPR_DISPLAY, fi->hfinfo->display);
|
||||
if (dfilter_string != NULL) {
|
||||
if (pdata->print_text) {
|
||||
fputs("\": \"", pdata->fh);
|
||||
print_escaped_json(pdata->fh, dfilter_string);
|
||||
if (node->first_child != NULL) {
|
||||
fputs("\",\n", pdata->fh);
|
||||
}
|
||||
}
|
||||
}
|
||||
wmem_free(NULL, dfilter_string);
|
||||
|
||||
if (node->first_child == NULL) {
|
||||
if (node->next == NULL) {
|
||||
fputs("\"\n", pdata->fh);
|
||||
} else {
|
||||
fputs("\",\n", pdata->fh);
|
||||
}
|
||||
}
|
||||
}
|
||||
wmem_free(NULL, dfilter_string);
|
||||
|
||||
if (node->first_child == NULL) {
|
||||
if (node->next == NULL) {
|
||||
fputs("\"\n", pdata->fh);
|
||||
} else {
|
||||
fputs("\",\n", pdata->fh);
|
||||
}
|
||||
if (node->first_child != NULL) {
|
||||
print_indent(pdata->level + 3, pdata->fh);
|
||||
|
||||
fputs("\"", pdata->fh);
|
||||
print_escaped_json(pdata->fh, fi->hfinfo->abbrev);
|
||||
fputs("_tree\": {\n", pdata->fh);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,450 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
#
|
||||
# Copyright 2017, Martin Kacer <kacer.martin[AT]gmail.com>
|
||||
#
|
||||
# Wireshark - Network traffic analyzer
|
||||
# By Gerald Combs <gerald@wireshark.org>
|
||||
# Copyright 1998 Gerald Combs
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
import sys
|
||||
import json
|
||||
import operator
|
||||
import copy
|
||||
import os
|
||||
import binascii
|
||||
import array
|
||||
import argparse
|
||||
import subprocess
|
||||
from collections import OrderedDict
|
||||
|
||||
#
|
||||
# ********* PY TEMPLATES *********
|
||||
#
|
||||
def read_py_function(name):
|
||||
s = ''
|
||||
record = False
|
||||
indent = 0
|
||||
|
||||
file = open(__file__)
|
||||
for line in file:
|
||||
|
||||
ind = len(line) - len(line.lstrip())
|
||||
|
||||
if (line.find("def " + name) != -1):
|
||||
record = True
|
||||
indent = ind
|
||||
elif (record == True and indent == ind and len(line) > 1):
|
||||
record = False
|
||||
|
||||
if (record == True):
|
||||
s = s + line
|
||||
|
||||
file.close()
|
||||
return s
|
||||
|
||||
py_header = """#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# File generated by json2pcap.py
|
||||
# json2pcap.py created by Martin Kacer, 2017
|
||||
|
||||
import os
|
||||
import binascii
|
||||
import array
|
||||
import sys
|
||||
import subprocess
|
||||
from collections import OrderedDict
|
||||
|
||||
# *****************************************************
|
||||
# * PACKET PAYLOAD GENERATED FROM INPUT PCAP *
|
||||
# * Modify this function to edit the packet *
|
||||
# *****************************************************
|
||||
def main():
|
||||
d = OrderedDict()
|
||||
"""
|
||||
|
||||
py_footer = """ generate_pcap(d)
|
||||
|
||||
# *****************************************************
|
||||
# * FUNCTIONS from TEMPLATE *
|
||||
# * Do not edit these functions if not required *
|
||||
# *****************************************************
|
||||
|
||||
"""
|
||||
py_footer = py_footer + read_py_function("to_pcap_file")
|
||||
py_footer = py_footer + read_py_function("hex_to_txt")
|
||||
py_footer = py_footer + read_py_function("to_bytes")
|
||||
py_footer = py_footer + read_py_function("lsb")
|
||||
py_footer = py_footer + read_py_function("rewrite_frame")
|
||||
|
||||
py_footer = py_footer + """
|
||||
def assemble_frame(d):
|
||||
input = d['frame_raw'][1]
|
||||
isFlat = False
|
||||
linux_cooked_header = False;
|
||||
while(isFlat == False):
|
||||
isFlat = True
|
||||
for key, val in d.items():
|
||||
h = str(val[1]) # hex
|
||||
p = val[2] * 2 # position
|
||||
l = val[3] * 2 # length
|
||||
b = val[4] # bitmask
|
||||
t = val[5] # type
|
||||
|
||||
if (key == "sll_raw"):
|
||||
linux_cooked_header = True;
|
||||
|
||||
# only if the node is not parent
|
||||
isParent = False
|
||||
for k, v in d.items():
|
||||
if (v[0] == key):
|
||||
isParent = True
|
||||
isFlat = False
|
||||
break
|
||||
|
||||
if (isParent == False and val[0] is not None):
|
||||
d[val[0]][1] = rewrite_frame(d[val[0]][1], h, p, l, b, t)
|
||||
del d[key]
|
||||
|
||||
output = d['frame_raw'][1]
|
||||
|
||||
# for Linux cooked header replace dest MAC and remove two bytes to reconstruct normal frame using text2pcap
|
||||
if (linux_cooked_header):
|
||||
output = "000000000000" + output[6*2:] # replce dest MAC
|
||||
output = output[:12*2] + "" + output[14*2:] # remove two bytes before Protocol
|
||||
|
||||
return output
|
||||
|
||||
def generate_pcap(d):
|
||||
# 1. Assemble frame
|
||||
input = d['frame_raw'][1]
|
||||
output = assemble_frame(d)
|
||||
print(input)
|
||||
print(output)
|
||||
|
||||
# 2. Testing: compare input and output for not modified json
|
||||
if (input != output):
|
||||
print("Modified frames: ")
|
||||
s1 = input
|
||||
s2 = output
|
||||
print(s1)
|
||||
print(s2)
|
||||
if (len(s1) == len(s2)):
|
||||
d = [i for i in xrange(len(s1)) if s1[i] != s2[i]]
|
||||
print(d)
|
||||
|
||||
# 3. Open TMP file used by text2pcap
|
||||
file = sys.argv[0] + '.tmp'
|
||||
f = open(file,'w')
|
||||
hex_to_txt(output, file)
|
||||
f.close()
|
||||
|
||||
# 4. Generate pcap
|
||||
to_pcap_file(sys.argv[0] + '.tmp', sys.argv[0] + '.pcap')
|
||||
print("Generated " + sys.argv[0] + ".tmp")
|
||||
print("Generated " + sys.argv[0] + ".pcap")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
"""
|
||||
#
|
||||
# ***** End of PY TEMPLATES ******
|
||||
#
|
||||
|
||||
|
||||
|
||||
#
|
||||
# ********** FUNCTIONS ***********
|
||||
#
|
||||
def to_pcap_file(filename, output_pcap_file):
|
||||
subprocess.call(["text2pcap", filename, output_pcap_file])
|
||||
|
||||
def hex_to_txt(hexstring, output_file):
|
||||
h = hexstring.lower()
|
||||
|
||||
file = open(output_file, 'a')
|
||||
|
||||
for i in range(0, len(h), 2):
|
||||
if(i % 32 == 0):
|
||||
file.write(format(i / 2, '06x') + ' ')
|
||||
|
||||
file.write(h[i:i + 2] + ' ')
|
||||
|
||||
if(i % 32 == 30):
|
||||
file.write('\n')
|
||||
|
||||
file.write('\n')
|
||||
file.close()
|
||||
|
||||
def raw_flat_collector(dict):
|
||||
if hasattr(dict, 'items'):
|
||||
for k, v in dict.items():
|
||||
if k.endswith("_raw"):
|
||||
yield k, v
|
||||
else:
|
||||
for val in raw_flat_collector(v):
|
||||
yield val
|
||||
|
||||
|
||||
# d - input dictionary, parsed from json
|
||||
# r - result dictionary
|
||||
# frame_name - parent protocol name
|
||||
# frame_position - parent protocol position
|
||||
def py_generator(d, r, frame_name='frame_raw', frame_position=0):
|
||||
if (d is None or d is None):
|
||||
return
|
||||
|
||||
if hasattr(d, 'items'):
|
||||
for k, v in d.items():
|
||||
# no recursion
|
||||
if k.endswith("_raw"):
|
||||
h = v[0]
|
||||
p = v[1]
|
||||
l = v[2] * 2
|
||||
b = v[3]
|
||||
t = v[4]
|
||||
if (len(h) != l):
|
||||
l = len(h)
|
||||
|
||||
p = p - frame_position
|
||||
|
||||
# Add into result dictionary
|
||||
key = str(k).replace('.', '_')
|
||||
fn = frame_name.replace('.', '_')
|
||||
if (fn == key):
|
||||
fn = None
|
||||
value = [fn , h, p, l, b, t]
|
||||
r[key] = value
|
||||
|
||||
# recursion
|
||||
else:
|
||||
if isinstance(v, dict):
|
||||
fn = frame_name
|
||||
fp = frame_position
|
||||
|
||||
# if there is also preceding raw protocol frame use it
|
||||
# remove tree suffix
|
||||
key = k
|
||||
if key.endswith("_tree"):
|
||||
key = key[:-5]
|
||||
raw_key = key + "_raw"
|
||||
if (raw_key in d):
|
||||
# f = d[raw_key][0]
|
||||
fn = raw_key
|
||||
fp = d[raw_key][1]
|
||||
|
||||
py_generator(v, r, fn, fp)
|
||||
|
||||
# To emulate Python 3.2
|
||||
def to_bytes(n, length, endianess='big'):
|
||||
h = '%x' % n
|
||||
s = ('0' * (len(h) % 2) + h).zfill(length * 2).decode('hex')
|
||||
return s if endianess == 'big' else s[::-1]
|
||||
|
||||
# Returns the index, counting from 0, of the least significant set bit in x
|
||||
def lsb(x):
|
||||
return (x & -x).bit_length() - 1
|
||||
|
||||
# Rewrite frame
|
||||
# h - hex bytes
|
||||
# p - position
|
||||
# l - length
|
||||
# b - bitmask
|
||||
# t - type
|
||||
def rewrite_frame(frame_raw, h, p, l, b, t):
|
||||
# no bitmask
|
||||
if(b == 0):
|
||||
if (len(h) != l):
|
||||
l = len(h)
|
||||
return frame_raw[:p] + h + frame_raw[p + l:]
|
||||
# bitmask
|
||||
else:
|
||||
# get hex string from frame which will be replaced
|
||||
_h = frame_raw[p:p + l]
|
||||
|
||||
# add 0 padding to have correct length
|
||||
if (len(_h) % 2 == 1):
|
||||
_h = '0' + _h
|
||||
if (len(h) % 2 == 1):
|
||||
h = '0' + h
|
||||
|
||||
# Only replace bits defined by mask
|
||||
# new_hex = (old_hex & !mask) | (new_hex & mask)
|
||||
_H = _h.decode("hex")
|
||||
_H = array.array('B', _H)
|
||||
|
||||
M = to_bytes(b, len(_H))
|
||||
M = array.array('B', M)
|
||||
# shift mask aligned to position
|
||||
for i in range(len(M)):
|
||||
if (i + p / 2) < len(M):
|
||||
M[i] = M[i + p / 2]
|
||||
else:
|
||||
M[i] = 0x00
|
||||
|
||||
H = h.decode("hex")
|
||||
H = array.array('B', H)
|
||||
|
||||
# for i in range(len(_H)):
|
||||
# print "{0:08b}".format(_H[i]),
|
||||
# print
|
||||
# for i in range(len(M)):
|
||||
# print "{0:08b}".format(M[i]),
|
||||
# print
|
||||
|
||||
j = 0;
|
||||
for i in range(len(_H)):
|
||||
if (M[i] != 0):
|
||||
v = H[j] << lsb(M[i])
|
||||
# print "Debug: {0:08b}".format(v),
|
||||
_H[i] = (_H[i] & ~M[i]) | (v & M[i])
|
||||
# print "Debug: " + str(_H[i]),
|
||||
j = j + 1;
|
||||
|
||||
# for i in range(len(_H)):
|
||||
# print "{0:08b}".format(_H[i]),
|
||||
# print
|
||||
|
||||
masked_h = binascii.hexlify(_H)
|
||||
|
||||
return frame_raw[:p] + masked_h + frame_raw[p + l:]
|
||||
|
||||
#
|
||||
# ************ MAIN **************
|
||||
#
|
||||
parser = argparse.ArgumentParser(description="""
|
||||
Utility to generate pcap from json format.
|
||||
|
||||
Packet modification:
|
||||
In input json it is possible to modify the raw values of decoded fields.
|
||||
The output pcap will include the modified values. The algorithm of
|
||||
generating the output pcap is to get all raw hex fields from input json and
|
||||
then assembling them by layering from longest (less decoded fields) to
|
||||
shortest (more decoded fields). It means if the modified raw field is
|
||||
shorter field (more decoded field) it takes precedence against modification
|
||||
in longer field (less decoded field). If the json includes duplicated raw
|
||||
fields with same position and length, the behavior is not deterministic.
|
||||
For manual packet editing it is always possible to remove any not required
|
||||
raw fields from json, only frame_raw is field mandatory for reconstruction.
|
||||
|
||||
Packet modification with -p switch:
|
||||
The python script is generated instead of pcap. This python script when
|
||||
executed will generate the pcap of 1st packet from input json. The
|
||||
generated code includes the decoded fields and the function to assembly the
|
||||
packet. This enables to modify the script and programmatically edit or
|
||||
encode the packet variables. The assembling algorithm is different, because
|
||||
the decoded packet fields are relative and points to parent node with their
|
||||
position (compared to input json which has absolute positions).
|
||||
|
||||
""", formatter_class=argparse.RawTextHelpFormatter)
|
||||
parser.add_argument('infile', nargs='+', help='json generated by tshark -T jsonraw or by tshark -T json -x')
|
||||
parser.add_argument('-p', '--python', help='generate python payload instead of pcap (only 1st packet)', default=False, action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
# read JSON
|
||||
infile = args.infile[0]
|
||||
|
||||
with open(infile) as data_file:
|
||||
json = json.load(data_file, object_pairs_hook=OrderedDict)
|
||||
|
||||
input_frame_raw = ''
|
||||
frame_raw = ''
|
||||
|
||||
# Generate pcap
|
||||
if args.python == False:
|
||||
# open TMP file used by text2pcap
|
||||
file = infile + '.tmp'
|
||||
f = open(file, 'w')
|
||||
|
||||
# Iterate over packets in JSON
|
||||
for packet in json:
|
||||
list = []
|
||||
linux_cooked_header = False;
|
||||
|
||||
# get flat raw fields into list
|
||||
for raw in raw_flat_collector(packet['_source']['layers']):
|
||||
if (raw[0] == "frame_raw"):
|
||||
frame_raw = raw[1][0]
|
||||
input_frame_raw = copy.copy(frame_raw)
|
||||
else:
|
||||
list.append(raw[1])
|
||||
if (raw[0] == "sll_raw"):
|
||||
linux_cooked_header = True
|
||||
|
||||
# sort list
|
||||
sorted_list = sorted(list, key=operator.itemgetter(1), reverse=False)
|
||||
sorted_list = sorted(sorted_list, key=operator.itemgetter(2), reverse=True)
|
||||
# print("Debug: " + str(sorted_list))
|
||||
|
||||
# rewrite frame
|
||||
for raw in sorted_list:
|
||||
h = str(raw[0]) # hex
|
||||
p = raw[1] * 2 # position
|
||||
l = raw[2] * 2 # length
|
||||
b = raw[3] # bitmask
|
||||
t = raw[4] # type
|
||||
|
||||
# print("Debug: " + str(raw))
|
||||
frame_raw = rewrite_frame(frame_raw, h, p, l, b, t)
|
||||
|
||||
# for Linux cooked header replace dest MAC and remove two bytes to reconstruct normal frame using text2pcap
|
||||
if (linux_cooked_header):
|
||||
frame_raw = "000000000000" + frame_raw[6 * 2:] # replce dest MAC
|
||||
frame_raw = frame_raw[:12 * 2] + "" + frame_raw[14 * 2:] # remove two bytes before Protocol
|
||||
|
||||
# Testing: remove comment to compare input and output for not modified json
|
||||
if (input_frame_raw != frame_raw):
|
||||
print("Modified frames: ")
|
||||
s1 = input_frame_raw
|
||||
s2 = frame_raw
|
||||
print(s1)
|
||||
print(s2)
|
||||
if (len(s1) == len(s2)):
|
||||
d = [i for i in xrange(len(s1)) if s1[i] != s2[i]]
|
||||
print(d)
|
||||
|
||||
hex_to_txt(frame_raw, file)
|
||||
|
||||
f.close()
|
||||
to_pcap_file(infile + '.tmp', sys.argv[1] + '.pcap')
|
||||
os.remove(infile + '.tmp')
|
||||
|
||||
# Generate python payload only for first packet
|
||||
else:
|
||||
file = infile + '.py'
|
||||
f = open(file, 'w')
|
||||
|
||||
for packet in json:
|
||||
f.write(py_header)
|
||||
|
||||
r = OrderedDict({})
|
||||
py_generator(packet['_source']['layers'], r)
|
||||
|
||||
for key, value in r.iteritems() :
|
||||
f.write(" d['" + key + "'] =",)
|
||||
f.write(" " + str(value) + "\n")
|
||||
|
||||
f.write(py_footer)
|
||||
|
||||
# Currently only first packet is used from pcap
|
||||
f.close
|
||||
|
||||
print("Generated " + infile + '.py')
|
||||
|
||||
break
|
||||
70
tshark.c
70
tshark.c
|
|
@ -169,7 +169,8 @@ typedef enum {
|
|||
WRITE_TEXT, /* summary or detail text */
|
||||
WRITE_XML, /* PDML or PSML */
|
||||
WRITE_FIELDS, /* User defined list of fields */
|
||||
WRITE_JSON, /* JSON */
|
||||
WRITE_JSON, /* JSON */
|
||||
WRITE_JSON_RAW, /* JSON only raw hex */
|
||||
WRITE_EK /* JSON bulk insert to Elasticsearch */
|
||||
/* Add CSV and the like here */
|
||||
} output_action_e;
|
||||
|
|
@ -386,7 +387,7 @@ print_usage(FILE *output)
|
|||
fprintf(output, " -P print packet summary even when writing to a file\n");
|
||||
fprintf(output, " -S <separator> the line separator to print between packets\n");
|
||||
fprintf(output, " -x add output of hex and ASCII dump (Packet Bytes)\n");
|
||||
fprintf(output, " -T pdml|ps|psml|json|ek|text|fields\n");
|
||||
fprintf(output, " -T pdml|ps|psml|json|jsonraw|ek|text|fields\n");
|
||||
fprintf(output, " format of text output (def: text)\n");
|
||||
fprintf(output, " -j <protocolfilter> protocols layers filter if -T ek|pdml|json selected\n");
|
||||
fprintf(output, " (e.g. \"ip ip.flags text\", filter does not expand child\n");
|
||||
|
|
@ -1320,31 +1321,38 @@ main(int argc, char *argv[])
|
|||
output_action = WRITE_EK;
|
||||
print_details = TRUE; /* Need details */
|
||||
print_summary = FALSE; /* Don't allow summary */
|
||||
} else if (strcmp(optarg, "jsonraw") == 0) {
|
||||
output_action = WRITE_JSON_RAW;
|
||||
print_details = TRUE; /* Need details */
|
||||
print_summary = FALSE; /* Don't allow summary */
|
||||
}
|
||||
else {
|
||||
cmdarg_err("Invalid -T parameter \"%s\"; it must be one of:", optarg); /* x */
|
||||
cmdarg_err_cont("\t\"fields\" The values of fields specified with the -e option, in a form\n"
|
||||
"\t specified by the -E option.\n"
|
||||
"\t\"pdml\" Packet Details Markup Language, an XML-based format for the\n"
|
||||
"\t details of a decoded packet. This information is equivalent to\n"
|
||||
"\t the packet details printed with the -V flag.\n"
|
||||
"\t\"ps\" PostScript for a human-readable one-line summary of each of\n"
|
||||
"\t the packets, or a multi-line view of the details of each of\n"
|
||||
"\t the packets, depending on whether the -V flag was specified.\n"
|
||||
"\t\"psml\" Packet Summary Markup Language, an XML-based format for the\n"
|
||||
"\t summary information of a decoded packet. This information is\n"
|
||||
"\t equivalent to the information shown in the one-line summary\n"
|
||||
"\t printed by default.\n"
|
||||
"\t\"json\" Packet Summary, an JSON-based format for the details\n"
|
||||
"\t summary information of a decoded packet. This information is \n"
|
||||
"\t equivalent to the packet details printed with the -V flag.\n"
|
||||
"\t\"ek\" Packet Summary, an EK JSON-based format for the bulk insert \n"
|
||||
"\t into elastic search cluster. This information is \n"
|
||||
"\t equivalent to the packet details printed with the -V flag.\n"
|
||||
"\t\"text\" Text of a human-readable one-line summary of each of the\n"
|
||||
"\t packets, or a multi-line view of the details of each of the\n"
|
||||
"\t packets, depending on whether the -V flag was specified.\n"
|
||||
"\t This is the default.");
|
||||
cmdarg_err_cont("\t\"fields\" The values of fields specified with the -e option, in a form\n"
|
||||
"\t specified by the -E option.\n"
|
||||
"\t\"pdml\" Packet Details Markup Language, an XML-based format for the\n"
|
||||
"\t details of a decoded packet. This information is equivalent to\n"
|
||||
"\t the packet details printed with the -V flag.\n"
|
||||
"\t\"ps\" PostScript for a human-readable one-line summary of each of\n"
|
||||
"\t the packets, or a multi-line view of the details of each of\n"
|
||||
"\t the packets, depending on whether the -V flag was specified.\n"
|
||||
"\t\"psml\" Packet Summary Markup Language, an XML-based format for the\n"
|
||||
"\t summary information of a decoded packet. This information is\n"
|
||||
"\t equivalent to the information shown in the one-line summary\n"
|
||||
"\t printed by default.\n"
|
||||
"\t\"json\" Packet Summary, an JSON-based format for the details\n"
|
||||
"\t summary information of a decoded packet. This information is \n"
|
||||
"\t equivalent to the packet details printed with the -V flag.\n"
|
||||
"\t\"jsonraw\" Packet Details, a JSON-based format for machine parsing\n"
|
||||
"\t including only raw hex decoded fields (same as -T json -x but\n"
|
||||
"\t without text decoding, only raw fields included). \n"
|
||||
"\t\"ek\" Packet Details, an EK JSON-based format for the bulk insert \n"
|
||||
"\t into elastic search cluster. This information is \n"
|
||||
"\t equivalent to the packet details printed with the -V flag.\n"
|
||||
"\t\"text\" Text of a human-readable one-line summary of each of the\n"
|
||||
"\t packets, or a multi-line view of the details of each of the\n"
|
||||
"\t packets, depending on whether the -V flag was specified.\n"
|
||||
"\t This is the default.");
|
||||
exit_status = INVALID_OPTION;
|
||||
goto clean_exit;
|
||||
}
|
||||
|
|
@ -1553,8 +1561,8 @@ main(int argc, char *argv[])
|
|||
}
|
||||
|
||||
if (print_hex) {
|
||||
if (output_action != WRITE_TEXT && output_action != WRITE_JSON && output_action != WRITE_EK) {
|
||||
cmdarg_err("Raw packet hex data can only be printed as text, PostScript, JSON or EK JSON");
|
||||
if (output_action != WRITE_TEXT && output_action != WRITE_JSON && output_action != WRITE_JSON_RAW && output_action != WRITE_EK) {
|
||||
cmdarg_err("Raw packet hex data can only be printed as text, PostScript, JSON, JSONRAW or EK JSON");
|
||||
exit_status = INVALID_OPTION;
|
||||
goto clean_exit;
|
||||
}
|
||||
|
|
@ -3678,6 +3686,7 @@ write_preamble(capture_file *cf)
|
|||
return !ferror(stdout);
|
||||
|
||||
case WRITE_JSON:
|
||||
case WRITE_JSON_RAW:
|
||||
write_json_preamble(stdout);
|
||||
return !ferror(stdout);
|
||||
|
||||
|
|
@ -3989,6 +3998,7 @@ print_packet(capture_file *cf, epan_dissect_t *edt)
|
|||
case WRITE_FIELDS: /*No non-verbose "fields" format */
|
||||
case WRITE_JSON:
|
||||
case WRITE_EK:
|
||||
case WRITE_JSON_RAW:
|
||||
g_assert_not_reached();
|
||||
break;
|
||||
}
|
||||
|
|
@ -4028,10 +4038,17 @@ print_packet(capture_file *cf, epan_dissect_t *edt)
|
|||
printf("\n");
|
||||
return !ferror(stdout);
|
||||
case WRITE_JSON:
|
||||
print_args.print_dissections = print_dissections_expanded;
|
||||
print_args.print_hex = print_hex;
|
||||
write_json_proto_tree(output_fields, &print_args, protocolfilter, protocolfilter_flags, edt, stdout);
|
||||
printf("\n");
|
||||
return !ferror(stdout);
|
||||
case WRITE_JSON_RAW:
|
||||
print_args.print_dissections = print_dissections_none;
|
||||
print_args.print_hex = TRUE;
|
||||
write_json_proto_tree(output_fields, &print_args, protocolfilter, protocolfilter_flags, edt, stdout);
|
||||
printf("\n");
|
||||
return !ferror(stdout);
|
||||
case WRITE_EK:
|
||||
print_args.print_hex = print_hex;
|
||||
write_ek_proto_tree(output_fields, &print_args, protocolfilter, protocolfilter_flags, edt, stdout);
|
||||
|
|
@ -4071,6 +4088,7 @@ write_finale(void)
|
|||
return !ferror(stdout);
|
||||
|
||||
case WRITE_JSON:
|
||||
case WRITE_JSON_RAW:
|
||||
write_json_finale(stdout);
|
||||
return !ferror(stdout);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue