#!/usr/bin/env python3 # # Copyright 2020, Martin Kacer and contributors # # Wireshark - Network traffic analyzer # By Gerald Combs # Copyright 1998 Gerald Combs # # SPDX-License-Identifier: GPL-2.0-or-later import sys import ijson import operator import copy import binascii import array import argparse import string import random import math import hashlib import re from collections import OrderedDict from scapy import all as scapy # Field anonymization class class AnonymizedField: ''' The Anonymization field object specifying anonymization :filed arg: field name :type arg: anonymization type [0 masking 0xff, 1 anonymization shake_256] :start arg: If specified, the anonymization starts at given byte number :end arg: If specified, the anonymization ends at given byte number ''' def __init__(self, field, type): self.field = field self.type = type self.start = None self.end = None match = re.search(r'(\S+)\[(-?\d+)?:(-?\d+)?\]', field) if match: self.field = match.group(1) self.start = match.group(2) if self.start is not None: self.start = int(self.start) self.end = match.group(3) if self.end is not None: self.end = int(self.end) # Returns the new field value after anonymization def anonymize_field_shake256(self, field, type, salt): shake = hashlib.shake_256(str(field + ':' + salt).encode('utf-8')) # String type, output should be ASCII if type in [26, 27, 28]: length = math.ceil(len(field)/4) shake_hash = shake.hexdigest(length) ret_string = array.array('B', str.encode(shake_hash)) ret_string = ''.join('{:02x}'.format(x) for x in ret_string) # Other types, output could be HEX else: length = math.ceil(len(field)/2) shake_hash = shake.hexdigest(length) ret_string = shake_hash # Correct the string length if (len(ret_string) < len(field)): ret_string = ret_string.ljust(len(field)) if (len(ret_string) > len(field)): ret_string = ret_string[:len(field)] return ret_string def anonymize_field(self, _h, _t, salt): s = 0 e = None if self.start: s = self.start if self.end: e = self.end if e < 0: e = len(_h) + e else: e = len(_h) h = _h[s:e] if self.type == 0: h = 'f' * len(h) elif self.type == 1: h = self.anonymize_field_shake256(h, _t, salt) h_mask = '0' * len(_h[0:s]) + 'f' * len(h) + '0' * len(_h[e:]) h = _h[0:s] + h + _h[e:] return [h, h_mask] def make_unique(key, dct): counter = 0 unique_key = key while unique_key in dct: counter += 1 unique_key = '{}_{}'.format(key, counter) return unique_key def parse_object_pairs(pairs): dct = OrderedDict() for key, value in pairs: if key in dct: key = make_unique(key, dct) dct[key] = value return dct # # ********* PY TEMPLATES ********* # def read_py_function(name): s = '' record = False indent = 0 file = open(__file__) for line in file: ind = len(line) - len(line.lstrip()) if line.find("def " + name) != -1: record = True indent = ind elif record and indent == ind and len(line) > 1: record = False if record: s = s + line file.close() return s py_header = """#!/usr/bin/env python # -*- coding: utf-8 -*- # File generated by json2pcap.py # json2pcap.py created by Martin Kacer, 2020 import os import binascii import array import sys import subprocess from collections import OrderedDict from scapy import all as scapy # ***************************************************** # * PACKET PAYLOAD GENERATED FROM INPUT PCAP * # * Modify this function to edit the packet * # ***************************************************** def main(): d = OrderedDict() """ py_footer = """ generate_pcap(d) # ***************************************************** # * FUNCTIONS from TEMPLATE * # * Do not edit these functions if not required * # ***************************************************** """ py_footer = py_footer + read_py_function("to_bytes") py_footer = py_footer + read_py_function("lsb") py_footer = py_footer + read_py_function("multiply_strings") py_footer = py_footer + read_py_function("rewrite_frame") py_footer = py_footer + read_py_function("assemble_frame") py_footer = py_footer + read_py_function("generate_pcap") py_footer = py_footer + """ if __name__ == '__main__': main() """ # # ***** End of PY TEMPLATES ****** # # # ********** FUNCTIONS *********** # def raw_flat_collector(dict): if hasattr(dict, 'items'): for k, v in dict.items(): if k.endswith("_raw"): yield k, v else: for val in raw_flat_collector(v): yield val # d - input dictionary, parsed from json # r - result dictionary # frame_name - parent protocol name # frame_position - parent protocol position def py_generator(d, r, frame_name='frame_raw', frame_position=0): if (d is None or d is None): return if hasattr(d, 'items'): for k, v in d.items(): # no recursion if k.endswith("_raw") or "_raw_" in k: if isinstance(v[1], (list, tuple)) or isinstance(v[2], (list, tuple)): #i = 1; for _v in v: h = _v[0] p = _v[1] l = _v[2] * 2 b = _v[3] t = _v[4] if (len(h) != l): l = len(h) p = p - frame_position # Add into result dictionary key = str(k).replace('.', '_') key = make_unique(key, r) fn = frame_name.replace('.', '_') if (fn == key): fn = None value = [fn, h, p, l, b, t] r[key] = value else: h = v[0] p = v[1] l = v[2] * 2 b = v[3] t = v[4] if (len(h) != l): l = len(h) p = p - frame_position # Add into result dictionary key = str(k).replace('.', '_') key = make_unique(key, r) fn = frame_name.replace('.', '_') if (fn == key): fn = None value = [fn , h, p, l, b, t] r[key] = value # recursion else: if isinstance(v, dict): fn = frame_name fp = frame_position # if there is also preceding raw protocol frame use it # remove tree suffix key = k if (key.endswith("_tree") or ("_tree_" in key)): key = key.replace('_tree', '') raw_key = key + "_raw" if (raw_key in d): # f = d[raw_key][0] fn = raw_key fp = d[raw_key][1] py_generator(v, r, fn, fp) elif isinstance(v, (list, tuple)): fn = frame_name fp = frame_position # if there is also preceding raw protocol frame use it # remove tree suffix key = k if (key.endswith("_tree") or ("_tree_" in key)): key = key.replace('_tree', '') raw_key = key + "_raw" if (raw_key in d): fn = raw_key fp = d[raw_key][1] for _v in v: py_generator(_v, r, frame_name, frame_position) # To emulate Python 3.2 def to_bytes(n, length, endianess='big'): h = '%x' % n s = bytearray.fromhex(('0' * (len(h) % 2) + h).zfill(length * 2)) return s if endianess == 'big' else s[::-1] # Returns the index, counting from 0, of the least significant set bit in x def lsb(x): return (x & -x).bit_length() - 1 # Replace parts of original_string by new_string, only if mask in the byte is not ff def multiply_strings(original_string, new_string, mask): ret_string = new_string if mask is None: return ret_string for i in range(0, min(len(original_string), len(new_string), len(mask)), 2): if mask[i:i + 2] == 'ff': #print("ff") ret_string = ret_string[:i] + original_string[i:i + 2] + ret_string[i + 2:] return ret_string # Rewrite frame # h - hex bytes # p - position # l - length # b - bitmask # t - type # frame_amask - optional, anonymization mask (00 - not anonymized byte, ff - anonymized byte) def rewrite_frame(frame_raw, h, p, l, b, t, frame_amask=None): if p < 0 or l < 0 or h is None: return frame_raw # no bitmask if(b == 0): if (len(h) != l): l = len(h) frame_raw_new = frame_raw[:p] + h + frame_raw[p + l:] return multiply_strings(frame_raw, frame_raw_new, frame_amask) # bitmask else: # get hex string from frame which will be replaced _h = frame_raw[p:p + l] # add 0 padding to have correct length if (len(_h) % 2 == 1): _h = '0' + _h if (len(h) % 2 == 1): h = '0' + h # Only replace bits defined by mask # new_hex = (old_hex & !mask) | (new_hex & mask) _H = bytearray.fromhex(_h) _H = array.array('B', _H) M = to_bytes(b, len(_H)) M = array.array('B', M) # shift mask aligned to position for i in range(len(M)): if (i + p / 2) < len(M): M[i] = M[i + int(p / 2)] else: M[i] = 0x00 H = bytearray.fromhex(h) H = array.array('B', H) # for i in range(len(_H)): # print "{0:08b}".format(_H[i]), # print # for i in range(len(M)): # print "{0:08b}".format(M[i]), # print j = 0 for i in range(len(_H)): if (M[i] != 0): v = H[j] << lsb(M[i]) # print "Debug: {0:08b}".format(v), _H[i] = (_H[i] & ~M[i]) | (v & M[i]) # print "Debug: " + str(_H[i]), j = j + 1 # for i in range(len(_H)): # print "{0:08b}".format(_H[i]), # print masked_h = binascii.hexlify(_H) masked_h = masked_h.decode('ascii') frame_raw_new = frame_raw[:p] + str(masked_h) + frame_raw[p + l:] return multiply_strings(frame_raw, frame_raw_new, frame_amask) def assemble_frame(d, frame_time): input = d['frame_raw'][1] isFlat = False linux_cooked_header = False while not isFlat: isFlat = True _d = d.copy() for key, val in _d.items(): h = str(val[1]) # hex p = val[2] * 2 # position l = val[3] * 2 # length b = val[4] # bitmask t = val[5] # type if (key == "sll_raw"): linux_cooked_header = True # only if the node is not parent isParent = False for k, v in d.items(): if (v[0] == key): isParent = True isFlat = False break if not isParent and val[0] is not None: d[val[0]][1] = rewrite_frame(d[val[0]][1], h, p, l, b, t) del d[key] output = d['frame_raw'][1] # for Linux cooked header replace dest MAC and remove two bytes to reconstruct normal frame if (linux_cooked_header): output = "000000000000" + output[6*2:] # replce dest MAC output = output[:12*2] + "" + output[14*2:] # remove two bytes before Protocol return output def generate_pcap(d): # 1. Assemble frame input = d['frame_raw'][1] output = assemble_frame(d, None) print(input) print(output) # 2. Testing: compare input and output for not modified json if (input != output): print("Modified frames: ") s1 = input s2 = output print(s1) print(s2) if (len(s1) == len(s2)): d = [i for i in range(len(s1)) if s1[i] != s2[i]] print(d) # 3. Generate pcap outfile = sys.argv[0] + ".pcap" pcap_out = scapy.PcapWriter(outfile, append=False, sync=False) new_packet = scapy.Packet(bytearray.fromhex(output)) pcap_out.write(new_packet) print("Generated " + outfile) # # ************ MAIN ************** # VERSION = "1.1" parser = argparse.ArgumentParser(description=""" json2pcap {version} Utility to generate pcap from json format. Packet modification: In input json it is possible to modify the raw values of decoded fields. The output pcap will include the modified values. The algorithm of generating the output pcap is to get all raw hex fields from input json and then assembling them by layering from longest (less decoded fields) to shortest (more decoded fields). It means if the modified raw field is shorter field (more decoded field) it takes precedence against modification in longer field (less decoded field). If the json includes duplicated raw fields with same position and length, the behavior is not deterministic. For manual packet editing it is always possible to remove any not required raw fields from json, only frame_raw is field mandatory for reconstruction. Packet modification with -p switch: The python script is generated instead of pcap. This python script when executed will generate the pcap of 1st packet from input json. The generated code includes the decoded fields and the function to assembly the packet. This enables to modify the script and programmatically edit or encode the packet variables. The assembling algorithm is different, because the decoded packet fields are relative and points to parent node with their position (compared to input json which has absolute positions). Pcap masking and anonymization with -m and -a switch: The script allows to mask or anonymize the selected json raw fields. If the The fields are selected and located on lower protocol layers, they are not The overwritten by upper fields which are not marked by these switches. The pcap masking and anonymization can be performed in the following way: tshark -r orig.pcap -T json -x | \ python json2pcap.py -m "ip.src_raw" -a "ip.dst_raw" -o anonymized.pcap In this example the ip.src_raw field is masked with ffffffff by byte values and ip.dst_raw is hashed by randomly generated salt. Additionally the following syntax is valid to anonymize portion of field tshark -r orig.pcap -T json -x | \ python json2pcap.py -m "ip.src_raw[2:]" -a "ip.dst_raw[:-2]" -o anonymized.pcap Where the src_ip first byte is preserved and dst_ip last byte is preserved. And the same can be achieved by tshark -r orig.pcap -T json -x | \ python json2pcap.py -m "ip.src_raw[2:8]" -a "ip.dst_raw[0:6]" -o anonymized.pcap Masking and anonymization limitations are mainly the following: - In case the tshark is performing reassembling from multiple frames, the backward pcap reconstruction is not properly performed and can result in malformed frames. - The new values in the fields could violate the field format, as the json2pcap is no performing correct protocol encoding with respect to allowed values of the target field and field encoding. """.format(version=VERSION), formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('--version', action='version', version='%(prog)s ' + VERSION) parser.add_argument('-i', '--infile', nargs='?', help='json generated by tshark -T json -x\nor by tshark -T jsonraw (not preserving frame timestamps).\nIf no inpout file is specified script reads from stdin.') parser.add_argument('-o', '--outfile', required=True, help='output pcap filename') parser.add_argument('-p', '--python', help='generate python payload instead of pcap (only 1st packet)', default=False, action='store_true') parser.add_argument('-m', '--mask', help='mask the specific raw field (e.g. -m "ip.src_raw" -m "ip.dst_raw[2:6]")', action='append', metavar='MASKED_FIELD') parser.add_argument('-a', '--anonymize', help='anonymize the specific raw field (e.g. -a "ip.src_raw[2:]" -a "ip.dst_raw[:-2]")', action='append', metavar='ANONYMIZED_FIELD') parser.add_argument('-s', '--salt', help='salt use for anonymization. If no value is provided it is randomized.', default=None) parser.add_argument('-v', '--verbose', help='verbose output', default=False, action='store_true') args = parser.parse_args() # read JSON infile = args.infile outfile = args.outfile # Read from input file if infile: data_file = open(infile) # Read from pipe else: data_file = sys.stdin # Parse anonymization fields anonymize = {} if args.mask: for m in args.mask: if '_raw' not in m: print("Error: The specified fields by -m switch should be raw fields. " + m + " does not have _raw suffix") sys.exit() af = AnonymizedField(m, 0) anonymize[af.field] = af if args.anonymize: for a in args.anonymize: if '_raw' not in a: print("Error: The specified fields by -a switch should be raw fields. " + a + " does not have _raw suffix") sys.exit() af = AnonymizedField(a, 1) anonymize[af.field] = af input_frame_raw = '' frame_raw = '' frame_time = None salt = args.salt if salt is None: # generate random salt if no salt was provided salt = ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(10)) # Generate pcap if args.python is False: pcap_out = scapy.PcapWriter(outfile, append=False, sync=False) # Iterate over packets in JSON for packet in ijson.items(data_file, "item", buf_size=200000): _list = [] linux_cooked_header = False # get flat raw fields into _list for raw in raw_flat_collector(packet['_source']['layers']): if len(raw) >= 2: if (raw[0] == "frame_raw"): frame_raw = raw[1][0] frame_amask = "0"*len(frame_raw) # initialize anonymization mask input_frame_raw = copy.copy(frame_raw) frame_time = None if 'frame.time_epoch' in packet['_source']['layers']['frame']: frame_time = packet['_source']['layers']['frame']['frame.time_epoch'] else: # add into value list into raw[5] the field name if isinstance(raw[1], list): raw[1].append(raw[0]) _list.append(raw[1]) if (raw[0] == "sll_raw"): linux_cooked_header = True # sort _list sorted_list = sorted(_list, key=operator.itemgetter(1), reverse=False) sorted_list = sorted(sorted_list, key=operator.itemgetter(2), reverse=True) # print("Debug: " + str(sorted_list)) # rewrite frame for raw in sorted_list: if len(raw) >= 6: h = str(raw[0]) # hex p = raw[1] * 2 # position l = raw[2] * 2 # length b = raw[3] # bitmask t = raw[4] # type # raw[5] # field_name (added by script) h_mask = h # hex for anonymization mask # anonymize fields if (raw[5] in anonymize): [h, h_mask] = anonymize[raw[5]].anonymize_field(h, t, salt) if (isinstance(p, (list, tuple)) or isinstance(l, (list, tuple))): for r in raw: _h = str(r[0]) # hex _p = r[1] * 2 # position _l = r[2] * 2 # length _b = r[3] # bitmask _t = r[4] # type # raw[5] # field_name (added by script) _h_mask = _h # hex for anonymization mask # anonymize fields if (raw[5] in anonymize): [_h, _h_mask] = anonymize[raw[5]].anonymize_field(_h, _t, salt) # print("Debug: " + str(raw)) frame_raw = rewrite_frame(frame_raw, _h, _p, _l, _b, _t, frame_amask) # update anonymization mask if (raw[5] in anonymize): frame_amask = rewrite_frame(frame_amask, _h_mask, _p, _l, _b, _t) else: # print("Debug: " + str(raw)) frame_raw = rewrite_frame(frame_raw, h, p, l, b, t, frame_amask) # update anonymization mask if (raw[5] in anonymize): frame_amask = rewrite_frame(frame_amask, h_mask, p, l, b, t) # for Linux cooked header replace dest MAC and remove two bytes to reconstruct normal frame using text2pcap if (linux_cooked_header): frame_raw = "000000000000" + frame_raw[6 * 2:] # replce dest MAC frame_raw = frame_raw[:12 * 2] + "" + frame_raw[14 * 2:] # remove two bytes before Protocol # Testing: remove comment to compare input and output for not modified json if (args.verbose and input_frame_raw != frame_raw): print("Modified frames: ") s1 = input_frame_raw s2 = frame_raw print(s1) print(s2) if (len(s1) == len(s2)): d = [i for i in range(len(s1)) if s1[i] != s2[i]] print(d) new_packet = scapy.Packet(bytearray.fromhex(frame_raw)) if frame_time: new_packet.time = float(frame_time) pcap_out.write(new_packet) # Generate python payload only for first packet else: py_outfile = outfile + '.py' f = open(py_outfile, 'w') #for packet in json: for packet in ijson.items(data_file, "item", buf_size=200000): f.write(py_header) r = OrderedDict({}) #print "packet = " + str(packet['_source']['layers']) py_generator(packet['_source']['layers'], r) for key, value in r.items(): f.write(" d['" + key + "'] =",) f.write(" " + str(value) + "\n") f.write(py_footer) # Currently only first packet is used from pcap f.close print("Generated " + py_outfile) break