2009-09-24 19:02:49 +00:00
|
|
|
#!/usr/bin/python
|
|
|
|
#
|
|
|
|
# Tool to index protocols that appears in the given capture files
|
|
|
|
#
|
|
|
|
# Copyright 2009, Kovarththanan Rajaratnam <kovarththanan.rajaratnam@gmail.com>
|
|
|
|
#
|
|
|
|
# $Id$
|
|
|
|
#
|
|
|
|
# Wireshark - Network traffic analyzer
|
|
|
|
# By Gerald Combs <gerald@wireshark.org>
|
|
|
|
# Copyright 1998 Gerald Combs
|
|
|
|
#
|
|
|
|
# This program is free software; you can redistribute it and/or
|
|
|
|
# modify it under the terms of the GNU General Public License
|
|
|
|
# as published by the Free Software Foundation; either version 2
|
|
|
|
# of the License, or (at your option) any later version.
|
|
|
|
#
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
# GNU General Public License for more details.
|
|
|
|
#
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
|
|
# along with this program; if not, write to the Free Software
|
|
|
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
|
|
#
|
|
|
|
|
|
|
|
from optparse import OptionParser
|
2009-09-25 19:41:54 +00:00
|
|
|
import multiprocessing
|
2009-09-24 19:02:49 +00:00
|
|
|
import sys
|
|
|
|
import os
|
|
|
|
import subprocess
|
|
|
|
import re
|
2009-09-25 15:02:57 +00:00
|
|
|
import pickle
|
2009-09-26 11:05:59 +00:00
|
|
|
import tempfile
|
|
|
|
import filecmp
|
2009-10-11 10:39:21 +00:00
|
|
|
import random
|
2009-09-25 15:02:57 +00:00
|
|
|
|
2009-09-26 09:03:26 +00:00
|
|
|
def extract_protos_from_file_proces(tshark, file):
|
2009-09-25 21:01:28 +00:00
|
|
|
try:
|
|
|
|
cmd = [tshark, "-Tfields", "-e", "frame.protocols", "-r", file]
|
|
|
|
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
|
|
(stdout, stderr) = p.communicate()
|
|
|
|
if p.returncode != 0:
|
|
|
|
return (file, {})
|
2009-09-25 15:02:57 +00:00
|
|
|
|
2009-09-25 21:01:28 +00:00
|
|
|
proto_hash = {}
|
|
|
|
for line in stdout.splitlines():
|
|
|
|
if not re.match(r'^[\w:-]+$', line):
|
|
|
|
continue
|
2009-09-25 15:02:57 +00:00
|
|
|
|
2009-09-25 21:01:28 +00:00
|
|
|
for proto in line.split(':'):
|
|
|
|
proto_hash[proto] = 1 + proto_hash.setdefault(proto, 0)
|
2009-09-25 15:02:57 +00:00
|
|
|
|
2009-09-25 21:01:28 +00:00
|
|
|
return (file, proto_hash)
|
|
|
|
except KeyboardInterrupt:
|
|
|
|
return None
|
2009-09-24 19:02:49 +00:00
|
|
|
|
2009-09-26 09:03:26 +00:00
|
|
|
def extract_protos_from_file(tshark, num_procs, max_files, cap_files, cap_hash, index_file_name):
|
|
|
|
pool = multiprocessing.Pool(num_procs)
|
|
|
|
results = [pool.apply_async(extract_protos_from_file_proces, [tshark, file]) for file in cap_files]
|
|
|
|
try:
|
|
|
|
for (cur_item_idx,result_async) in enumerate(results):
|
|
|
|
file_result = result_async.get()
|
|
|
|
action = "SKIPPED" if file_result[1] is {} else "PROCESSED"
|
|
|
|
print "%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result[0], os.path.getsize(file_result[0]))
|
|
|
|
cap_hash.update(dict([file_result]))
|
|
|
|
except KeyboardInterrupt:
|
|
|
|
print "%s was interrupted by user" % (sys.argv[0])
|
|
|
|
pool.terminate()
|
|
|
|
exit(1)
|
|
|
|
|
|
|
|
index_file = open(index_file_name, "w")
|
|
|
|
pickle.dump(cap_hash, index_file)
|
|
|
|
index_file.close()
|
|
|
|
exit(0)
|
|
|
|
|
2009-09-26 11:05:59 +00:00
|
|
|
def dissect_file_process(tshark, tmpdir, file):
|
2009-09-26 09:03:26 +00:00
|
|
|
try:
|
2009-09-26 11:05:59 +00:00
|
|
|
(handle_o, tmpfile_o) = tempfile.mkstemp(suffix='_stdout', dir=tmpdir)
|
|
|
|
(handle_e, tmpfile_e) = tempfile.mkstemp(suffix='_stderr', dir=tmpdir)
|
2009-09-26 09:03:26 +00:00
|
|
|
cmd = [tshark, "-nxVr", file]
|
2009-09-26 11:05:59 +00:00
|
|
|
p = subprocess.Popen(cmd, stdout=handle_o, stderr=handle_e)
|
2009-09-26 09:03:26 +00:00
|
|
|
(stdout, stderr) = p.communicate()
|
|
|
|
if p.returncode == 0:
|
2009-09-26 11:05:59 +00:00
|
|
|
return (file, True, tmpfile_o, tmpfile_e)
|
2009-09-26 09:03:26 +00:00
|
|
|
else:
|
2009-09-26 11:05:59 +00:00
|
|
|
return (file, False, tmpfile_o, tmpfile_e)
|
2009-09-26 09:03:26 +00:00
|
|
|
|
|
|
|
except KeyboardInterrupt:
|
|
|
|
return False
|
|
|
|
|
2009-09-26 11:05:59 +00:00
|
|
|
finally:
|
|
|
|
os.close(handle_o)
|
|
|
|
os.close(handle_e)
|
|
|
|
|
2009-10-11 10:45:48 +00:00
|
|
|
def dissect_files(tshark, tmpdir, num_procs, max_files, cap_files):
|
2009-09-26 09:03:26 +00:00
|
|
|
pool = multiprocessing.Pool(num_procs)
|
2009-09-26 11:05:59 +00:00
|
|
|
results = [pool.apply_async(dissect_file_process, [tshark, tmpdir, file]) for file in cap_files]
|
2009-09-26 09:03:26 +00:00
|
|
|
try:
|
|
|
|
for (cur_item_idx,result_async) in enumerate(results):
|
|
|
|
file_result = result_async.get()
|
|
|
|
action = "FAILED" if file_result[1] is False else "PASSED"
|
|
|
|
print "%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result[0], os.path.getsize(file_result[0]))
|
|
|
|
except KeyboardInterrupt:
|
|
|
|
print "%s was interrupted by user" % (sys.argv[0])
|
|
|
|
pool.terminate()
|
|
|
|
exit(1)
|
|
|
|
|
2009-10-11 10:45:48 +00:00
|
|
|
def compare_files(tshark_bin, tmpdir, tshark_cmp, num_procs, max_files, cap_files):
|
2009-09-26 09:48:46 +00:00
|
|
|
pool = multiprocessing.Pool(num_procs)
|
2009-09-26 11:05:59 +00:00
|
|
|
results_bin = [pool.apply_async(dissect_file_process, [tshark_bin, tmpdir, file]) for file in cap_files]
|
|
|
|
results_cmp = [pool.apply_async(dissect_file_process, [tshark_cmp, tmpdir, file]) for file in cap_files]
|
2009-09-26 09:48:46 +00:00
|
|
|
try:
|
|
|
|
for (cur_item_idx,(result_async_bin, result_async_cmp)) in enumerate(zip(results_bin, results_cmp)):
|
|
|
|
file_result_bin = result_async_bin.get()
|
|
|
|
file_result_cmp = result_async_cmp.get()
|
2009-09-26 11:05:59 +00:00
|
|
|
if file_result_cmp[1] is False or file_result_bin[1] is False:
|
|
|
|
action = "FAILED (exitcode)"
|
|
|
|
if not filecmp.cmp(file_result_bin[2], file_result_cmp[2]):
|
|
|
|
action = "FAILED (stdout)"
|
|
|
|
if not filecmp.cmp(file_result_bin[3], file_result_cmp[3]):
|
|
|
|
action = "FAILED (stderr)"
|
|
|
|
else:
|
|
|
|
action = "PASSED"
|
2009-09-26 11:48:13 +00:00
|
|
|
os.remove(file_result_bin[2])
|
|
|
|
os.remove(file_result_cmp[2])
|
|
|
|
os.remove(file_result_bin[3])
|
|
|
|
os.remove(file_result_cmp[3])
|
2009-09-26 11:05:59 +00:00
|
|
|
|
2009-09-26 09:48:46 +00:00
|
|
|
print "%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result_bin[0], os.path.getsize(file_result_bin[0]))
|
|
|
|
print "%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result_cmp[0], os.path.getsize(file_result_cmp[0]))
|
|
|
|
except KeyboardInterrupt:
|
|
|
|
print "%s was interrupted by user" % (sys.argv[0])
|
|
|
|
pool.terminate()
|
|
|
|
exit(1)
|
|
|
|
|
2009-09-26 03:48:04 +00:00
|
|
|
def list_all_proto(cap_hash):
|
2009-09-25 17:28:53 +00:00
|
|
|
proto_hash = {}
|
|
|
|
for files_hash in cap_hash.itervalues():
|
|
|
|
for proto,count in files_hash.iteritems():
|
|
|
|
proto_hash[proto] = count + proto_hash.setdefault(proto, 0)
|
|
|
|
|
2009-09-26 09:03:26 +00:00
|
|
|
return proto_hash
|
2009-09-25 17:28:53 +00:00
|
|
|
|
2009-09-26 03:48:04 +00:00
|
|
|
def list_all_files(cap_hash):
|
2009-09-25 17:43:01 +00:00
|
|
|
files = cap_hash.keys()
|
|
|
|
files.sort()
|
|
|
|
|
2009-09-26 09:03:26 +00:00
|
|
|
return files
|
2009-09-25 17:43:01 +00:00
|
|
|
|
2009-09-26 03:48:04 +00:00
|
|
|
def list_all_proto_files(cap_hash, proto_comma_delit):
|
|
|
|
protos = [ x.strip() for x in proto_comma_delit.split(',') ]
|
|
|
|
files = []
|
|
|
|
for (file, files_hash) in cap_hash.iteritems():
|
|
|
|
for proto in files_hash.iterkeys():
|
|
|
|
if proto in protos:
|
|
|
|
files.append(file)
|
|
|
|
break
|
|
|
|
|
2009-09-26 09:03:26 +00:00
|
|
|
return files
|
2009-09-26 03:48:04 +00:00
|
|
|
|
2009-09-25 17:43:01 +00:00
|
|
|
def index_file_action(options):
|
2009-09-26 03:48:04 +00:00
|
|
|
return options.list_all_proto or \
|
|
|
|
options.list_all_files or \
|
2009-09-26 09:03:26 +00:00
|
|
|
options.list_all_proto_files or \
|
|
|
|
options.dissect_files
|
2009-09-25 17:43:01 +00:00
|
|
|
|
2009-09-25 17:47:25 +00:00
|
|
|
def find_capture_files(paths, cap_hash):
|
|
|
|
cap_files = []
|
|
|
|
for path in paths:
|
|
|
|
if os.path.isdir(path):
|
|
|
|
path = os.path.normpath(path)
|
|
|
|
for root, dirs, files in os.walk(path):
|
|
|
|
cap_files += [os.path.join(root, name) for name in files if os.path.join(root, name) not in cap_hash]
|
|
|
|
elif path not in cap_hash:
|
|
|
|
cap_files.append(path)
|
|
|
|
return cap_files
|
|
|
|
|
2009-09-26 07:13:48 +00:00
|
|
|
def find_tshark_executable(bin_dir):
|
|
|
|
for file in ["tshark.exe", "tshark"]:
|
|
|
|
tshark = os.path.join(bin_dir, file)
|
|
|
|
if os.access(tshark, os.X_OK):
|
|
|
|
return tshark
|
|
|
|
|
|
|
|
return None
|
|
|
|
|
2009-09-24 19:02:49 +00:00
|
|
|
def main():
|
2009-09-25 17:28:53 +00:00
|
|
|
parser = OptionParser(usage="usage: %prog [options] index_file [file_1|dir_1 [.. file_n|dir_n]]")
|
2009-09-26 09:03:26 +00:00
|
|
|
parser.add_option("-d", "--dissect-files", dest="dissect_files", default=False, action="store_true",
|
|
|
|
help="Dissect all matching files")
|
2009-09-26 03:48:04 +00:00
|
|
|
parser.add_option("-m", "--max-files", dest="max_files", default=sys.maxint, type="int",
|
|
|
|
help="Max number of files to process")
|
2009-09-26 09:48:46 +00:00
|
|
|
parser.add_option("-b", "--binary-dir", dest="bin_dir", default=os.getcwd(),
|
2009-09-26 03:48:04 +00:00
|
|
|
help="Directory containing tshark executable")
|
2009-09-26 09:48:46 +00:00
|
|
|
parser.add_option("-c", "--compare-dir", dest="compare_dir", default=None,
|
|
|
|
help="Directory containing tshark executable which is used for comparison")
|
2009-10-18 08:33:40 +00:00
|
|
|
parser.add_option("-j", dest="num_procs", default=multiprocessing.cpu_count(), type=int,
|
2009-09-26 03:48:04 +00:00
|
|
|
help="Max number of processes to spawn")
|
2009-10-11 10:39:21 +00:00
|
|
|
parser.add_option("-r", "--randomize", default=False, action="store_true",
|
|
|
|
help="Randomize the file list order")
|
2009-09-26 03:48:04 +00:00
|
|
|
parser.add_option("", "--list-all-proto", dest="list_all_proto", default=False, action="store_true",
|
|
|
|
help="List all protocols in index file")
|
|
|
|
parser.add_option("", "--list-all-files", dest="list_all_files", default=False, action="store_true",
|
|
|
|
help="List all files in index file")
|
|
|
|
parser.add_option("", "--list-all-proto-files", dest="list_all_proto_files", default=False,
|
|
|
|
metavar="PROTO_1[, .. PROTO_N]",
|
|
|
|
help="List all files in index file containing the given protocol")
|
2009-09-24 19:02:49 +00:00
|
|
|
|
|
|
|
(options, args) = parser.parse_args()
|
|
|
|
|
|
|
|
if len(args) == 0:
|
2009-09-25 15:41:17 +00:00
|
|
|
parser.error("index_file is a required argument")
|
2009-09-24 19:02:49 +00:00
|
|
|
|
2009-09-25 17:43:01 +00:00
|
|
|
if len(args) == 1 and not index_file_action(options):
|
2009-09-24 19:02:49 +00:00
|
|
|
parser.error("one capture file/directory must be specified")
|
|
|
|
|
2009-09-26 09:03:26 +00:00
|
|
|
if options.dissect_files and not options.list_all_files and not options.list_all_proto_files:
|
|
|
|
parser.error("--list-all-files or --list-all-proto-files must be specified")
|
|
|
|
|
2009-09-26 09:48:46 +00:00
|
|
|
if options.dissect_files and not options.compare_dir is None:
|
|
|
|
parser.error("--dissect-files and --compare-dir cannot be specified at the same time")
|
|
|
|
|
2009-09-25 15:41:17 +00:00
|
|
|
index_file_name = args.pop(0)
|
2009-09-26 09:03:26 +00:00
|
|
|
paths = args
|
2009-09-26 03:48:04 +00:00
|
|
|
cap_hash = {}
|
2009-09-25 15:41:17 +00:00
|
|
|
try:
|
2009-09-25 17:03:44 +00:00
|
|
|
index_file = open(index_file_name, "r")
|
|
|
|
print "index file:", index_file.name, "[OPENED]",
|
2009-09-25 15:41:17 +00:00
|
|
|
cap_hash = pickle.load(index_file)
|
2009-09-25 17:03:44 +00:00
|
|
|
index_file.close()
|
|
|
|
print len(cap_hash), "files"
|
2009-09-25 15:41:17 +00:00
|
|
|
except IOError:
|
2009-09-25 17:03:44 +00:00
|
|
|
print "index file:", index_file_name, "[NEW]"
|
2009-09-25 15:41:17 +00:00
|
|
|
|
2009-09-26 03:48:04 +00:00
|
|
|
if options.list_all_proto:
|
2009-09-26 09:03:26 +00:00
|
|
|
print list_all_proto(cap_hash)
|
2009-09-26 03:48:04 +00:00
|
|
|
exit(0)
|
|
|
|
|
2009-09-26 09:03:26 +00:00
|
|
|
indexed_files = []
|
2009-09-26 03:48:04 +00:00
|
|
|
if options.list_all_files:
|
2009-09-26 09:03:26 +00:00
|
|
|
indexed_files = list_all_files(cap_hash)
|
|
|
|
print indexed_files
|
2009-09-25 17:28:53 +00:00
|
|
|
|
2009-09-26 03:48:04 +00:00
|
|
|
if options.list_all_proto_files:
|
2009-09-26 09:03:26 +00:00
|
|
|
indexed_files = list_all_proto_files(cap_hash, options.list_all_proto_files)
|
|
|
|
print indexed_files
|
2009-09-25 17:43:01 +00:00
|
|
|
|
2009-09-26 09:48:46 +00:00
|
|
|
tshark_bin = find_tshark_executable(options.bin_dir)
|
|
|
|
if not tshark_bin is None:
|
|
|
|
print "tshark:", tshark_bin, "[FOUND]"
|
2009-09-25 17:28:53 +00:00
|
|
|
else:
|
2009-09-26 09:48:46 +00:00
|
|
|
print "tshark:", tshark_bin, "[MISSING]"
|
2009-09-25 17:28:53 +00:00
|
|
|
exit(1)
|
|
|
|
|
2009-09-26 09:48:46 +00:00
|
|
|
if not options.compare_dir is None:
|
|
|
|
tshark_cmp = find_tshark_executable(options.compare_dir)
|
|
|
|
if not tshark_cmp is None:
|
|
|
|
print "tshark:", tshark_cmp, "[FOUND]"
|
|
|
|
else:
|
|
|
|
print "tshark:", tshark_cmp, "[MISSING]"
|
|
|
|
exit(1)
|
|
|
|
|
|
|
|
if options.dissect_files or options.compare_dir:
|
2009-09-26 09:03:26 +00:00
|
|
|
cap_files = indexed_files
|
|
|
|
elif options.list_all_proto_files or options.list_all_files:
|
|
|
|
exit(0)
|
|
|
|
else:
|
|
|
|
cap_files = find_capture_files(paths, cap_hash)
|
|
|
|
|
2009-10-11 10:39:21 +00:00
|
|
|
if options.randomize:
|
|
|
|
random.shuffle(cap_files)
|
|
|
|
else:
|
|
|
|
cap_files.sort()
|
|
|
|
|
2009-09-25 19:41:54 +00:00
|
|
|
options.max_files = min(options.max_files, len(cap_files))
|
2009-09-26 11:05:59 +00:00
|
|
|
print "%u total files, %u working files" % (len(cap_files), options.max_files)
|
2009-09-24 19:02:49 +00:00
|
|
|
cap_files = cap_files[:options.max_files]
|
2009-10-11 10:45:48 +00:00
|
|
|
tmpdir = tempfile.mkdtemp()
|
|
|
|
print "Temporary working dir: %s" % tmpdir
|
2009-09-24 19:02:49 +00:00
|
|
|
|
2009-09-26 09:48:46 +00:00
|
|
|
if options.compare_dir:
|
2009-10-11 10:45:48 +00:00
|
|
|
compare_files(tshark_bin, tmpdir, tshark_cmp, options.num_procs, options.max_files, cap_files)
|
2009-09-26 09:48:46 +00:00
|
|
|
elif options.dissect_files:
|
2009-10-11 10:45:48 +00:00
|
|
|
dissect_files(tshark_bin, tmpdir, options.num_procs, options.max_files, cap_files)
|
2009-09-26 09:03:26 +00:00
|
|
|
else:
|
2009-10-11 10:45:48 +00:00
|
|
|
extract_protos_from_file(tshark_bin, tmpdir, options.num_procs, options.max_files, cap_files, cap_hash, index_file_name)
|
2009-09-24 19:02:49 +00:00
|
|
|
|
2009-10-11 10:45:48 +00:00
|
|
|
os.rmdir(tmpdir)
|
2009-09-24 19:02:49 +00:00
|
|
|
if __name__ == "__main__":
|
|
|
|
main()
|