forked from osmocom/wireshark
indexcap.py: fix tmpdir handling, python 3 compat
Add Python 3 compatibility to indexcap.py and only create a tmpdir for actions that need it. Only remove tmpdir for the compare action and try to remove the temp dir even if an exception occurs. In list_protos_in_cap.sh, rename BIN_DIR to WS_BIN_PATH (matching test/config.sh) and allow it to be overridden from the environment. Tested with Python 2.6.6, 2.7.9, 3.2.6, 3.4.3 as follows (with tshark built using cmake and '-b /tmp/wsbuild/run'): tools/indexcap.py out pop-ssl.pcapng; tools/indexcap.py out pop-ssl.pcapng --list-all-proto; tools/indexcap.py out pop-ssl.pcapng --list-all-files; tools/indexcap.py out pop-ssl.pcapng --list-all-proto-files=ssl; rm out; Python 2.5.6 does not work (there is no multiprocessing module). The dict output in Python 3.4.3 is different (has the hash function changed?) Change-Id: I592d8c4458a20d5088d815c6dd1cf90c9d3df9d3 Reviewed-on: https://code.wireshark.org/review/7792 Reviewed-by: Jeff Morriss <jeff.morriss.ws@gmail.com> Reviewed-by: Anders Broman <a.broman58@gmail.com>
This commit is contained in:
parent
68698db8cc
commit
1ba4191c70
|
@ -1,4 +1,4 @@
|
||||||
#!/usr/bin/python
|
#!/usr/bin/env python
|
||||||
#
|
#
|
||||||
# Tool to index protocols that appears in the given capture files
|
# Tool to index protocols that appears in the given capture files
|
||||||
#
|
#
|
||||||
|
@ -41,6 +41,8 @@ def extract_protos_from_file_proces(tshark, file):
|
||||||
cmd = [tshark, "-Tfields", "-e", "frame.protocols", "-r", file]
|
cmd = [tshark, "-Tfields", "-e", "frame.protocols", "-r", file]
|
||||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
(stdout, stderr) = p.communicate()
|
(stdout, stderr) = p.communicate()
|
||||||
|
if sys.version_info[0] >= 3:
|
||||||
|
stdout = stdout.decode('utf-8')
|
||||||
if p.returncode != 0:
|
if p.returncode != 0:
|
||||||
return (file, {})
|
return (file, {})
|
||||||
|
|
||||||
|
@ -63,14 +65,14 @@ def extract_protos_from_file(tshark, num_procs, max_files, cap_files, cap_hash,
|
||||||
for (cur_item_idx,result_async) in enumerate(results):
|
for (cur_item_idx,result_async) in enumerate(results):
|
||||||
file_result = result_async.get()
|
file_result = result_async.get()
|
||||||
action = "SKIPPED" if file_result[1] is {} else "PROCESSED"
|
action = "SKIPPED" if file_result[1] is {} else "PROCESSED"
|
||||||
print "%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result[0], os.path.getsize(file_result[0]))
|
print("%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result[0], os.path.getsize(file_result[0])))
|
||||||
cap_hash.update(dict([file_result]))
|
cap_hash.update(dict([file_result]))
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
print "%s was interrupted by user" % (sys.argv[0])
|
print("%s was interrupted by user" % (sys.argv[0]))
|
||||||
pool.terminate()
|
pool.terminate()
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
index_file = open(index_file_name, "w")
|
index_file = open(index_file_name, "wb")
|
||||||
pickle.dump(cap_hash, index_file)
|
pickle.dump(cap_hash, index_file)
|
||||||
index_file.close()
|
index_file.close()
|
||||||
exit(0)
|
exit(0)
|
||||||
|
@ -101,9 +103,9 @@ def dissect_files(tshark, tmpdir, num_procs, max_files, cap_files):
|
||||||
for (cur_item_idx,result_async) in enumerate(results):
|
for (cur_item_idx,result_async) in enumerate(results):
|
||||||
file_result = result_async.get()
|
file_result = result_async.get()
|
||||||
action = "FAILED" if file_result[1] is False else "PASSED"
|
action = "FAILED" if file_result[1] is False else "PASSED"
|
||||||
print "%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result[0], os.path.getsize(file_result[0]))
|
print("%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result[0], os.path.getsize(file_result[0])))
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
print "%s was interrupted by user" % (sys.argv[0])
|
print("%s was interrupted by user" % (sys.argv[0]))
|
||||||
pool.terminate()
|
pool.terminate()
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
|
@ -128,23 +130,23 @@ def compare_files(tshark_bin, tmpdir, tshark_cmp, num_procs, max_files, cap_file
|
||||||
os.remove(file_result_bin[3])
|
os.remove(file_result_bin[3])
|
||||||
os.remove(file_result_cmp[3])
|
os.remove(file_result_cmp[3])
|
||||||
|
|
||||||
print "%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result_bin[0], os.path.getsize(file_result_bin[0]))
|
print("%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result_bin[0], os.path.getsize(file_result_bin[0])))
|
||||||
print "%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result_cmp[0], os.path.getsize(file_result_cmp[0]))
|
print("%s [%u/%u] %s %u bytes" % (action, cur_item_idx+1, max_files, file_result_cmp[0], os.path.getsize(file_result_cmp[0])))
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
print "%s was interrupted by user" % (sys.argv[0])
|
print("%s was interrupted by user" % (sys.argv[0]))
|
||||||
pool.terminate()
|
pool.terminate()
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
def list_all_proto(cap_hash):
|
def list_all_proto(cap_hash):
|
||||||
proto_hash = {}
|
proto_hash = {}
|
||||||
for files_hash in cap_hash.itervalues():
|
for files_hash in cap_hash.values():
|
||||||
for proto,count in files_hash.iteritems():
|
for proto,count in files_hash.items():
|
||||||
proto_hash[proto] = count + proto_hash.setdefault(proto, 0)
|
proto_hash[proto] = count + proto_hash.setdefault(proto, 0)
|
||||||
|
|
||||||
return proto_hash
|
return proto_hash
|
||||||
|
|
||||||
def list_all_files(cap_hash):
|
def list_all_files(cap_hash):
|
||||||
files = cap_hash.keys()
|
files = list(cap_hash.keys())
|
||||||
files.sort()
|
files.sort()
|
||||||
|
|
||||||
return files
|
return files
|
||||||
|
@ -152,8 +154,8 @@ def list_all_files(cap_hash):
|
||||||
def list_all_proto_files(cap_hash, proto_comma_delit):
|
def list_all_proto_files(cap_hash, proto_comma_delit):
|
||||||
protos = [ x.strip() for x in proto_comma_delit.split(',') ]
|
protos = [ x.strip() for x in proto_comma_delit.split(',') ]
|
||||||
files = []
|
files = []
|
||||||
for (file, files_hash) in cap_hash.iteritems():
|
for (file, files_hash) in cap_hash.items():
|
||||||
for proto in files_hash.iterkeys():
|
for proto in files_hash.keys():
|
||||||
if proto in protos:
|
if proto in protos:
|
||||||
files.append(file)
|
files.append(file)
|
||||||
break
|
break
|
||||||
|
@ -189,7 +191,7 @@ def main():
|
||||||
parser = OptionParser(usage="usage: %prog [options] index_file [file_1|dir_1 [.. file_n|dir_n]]")
|
parser = OptionParser(usage="usage: %prog [options] index_file [file_1|dir_1 [.. file_n|dir_n]]")
|
||||||
parser.add_option("-d", "--dissect-files", dest="dissect_files", default=False, action="store_true",
|
parser.add_option("-d", "--dissect-files", dest="dissect_files", default=False, action="store_true",
|
||||||
help="Dissect all matching files")
|
help="Dissect all matching files")
|
||||||
parser.add_option("-m", "--max-files", dest="max_files", default=sys.maxint, type="int",
|
parser.add_option("-m", "--max-files", dest="max_files", default=sys.maxsize, type="int",
|
||||||
help="Max number of files to process")
|
help="Max number of files to process")
|
||||||
parser.add_option("-b", "--binary-dir", dest="bin_dir", default=os.getcwd(),
|
parser.add_option("-b", "--binary-dir", dest="bin_dir", default=os.getcwd(),
|
||||||
help="Directory containing tshark executable")
|
help="Directory containing tshark executable")
|
||||||
|
@ -225,40 +227,40 @@ def main():
|
||||||
paths = args
|
paths = args
|
||||||
cap_hash = {}
|
cap_hash = {}
|
||||||
try:
|
try:
|
||||||
index_file = open(index_file_name, "r")
|
index_file = open(index_file_name, "rb")
|
||||||
print "index file:", index_file.name, "[OPENED]",
|
print("index file: %s [OPENED]" % index_file.name)
|
||||||
cap_hash = pickle.load(index_file)
|
cap_hash = pickle.load(index_file)
|
||||||
index_file.close()
|
index_file.close()
|
||||||
print len(cap_hash), "files"
|
print("%d files" % len(cap_hash))
|
||||||
except IOError:
|
except IOError:
|
||||||
print "index file:", index_file_name, "[NEW]"
|
print("index file: %s [NEW]" % index_file_name)
|
||||||
|
|
||||||
if options.list_all_proto:
|
if options.list_all_proto:
|
||||||
print list_all_proto(cap_hash)
|
print(list_all_proto(cap_hash))
|
||||||
exit(0)
|
exit(0)
|
||||||
|
|
||||||
indexed_files = []
|
indexed_files = []
|
||||||
if options.list_all_files:
|
if options.list_all_files:
|
||||||
indexed_files = list_all_files(cap_hash)
|
indexed_files = list_all_files(cap_hash)
|
||||||
print indexed_files
|
print(indexed_files)
|
||||||
|
|
||||||
if options.list_all_proto_files:
|
if options.list_all_proto_files:
|
||||||
indexed_files = list_all_proto_files(cap_hash, options.list_all_proto_files)
|
indexed_files = list_all_proto_files(cap_hash, options.list_all_proto_files)
|
||||||
print indexed_files
|
print(indexed_files)
|
||||||
|
|
||||||
tshark_bin = find_tshark_executable(options.bin_dir)
|
tshark_bin = find_tshark_executable(options.bin_dir)
|
||||||
if not tshark_bin is None:
|
if not tshark_bin is None:
|
||||||
print "tshark:", tshark_bin, "[FOUND]"
|
print("tshark: %s [FOUND]" % tshark_bin)
|
||||||
else:
|
else:
|
||||||
print "tshark:", tshark_bin, "[MISSING]"
|
print("tshark: %s [MISSING]" % tshark_bin)
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
if not options.compare_dir is None:
|
if not options.compare_dir is None:
|
||||||
tshark_cmp = find_tshark_executable(options.compare_dir)
|
tshark_cmp = find_tshark_executable(options.compare_dir)
|
||||||
if not tshark_cmp is None:
|
if not tshark_cmp is None:
|
||||||
print "tshark:", tshark_cmp, "[FOUND]"
|
print("tshark: %s [FOUND]" % tshark_cmp)
|
||||||
else:
|
else:
|
||||||
print "tshark:", tshark_cmp, "[MISSING]"
|
print("tshark: %s [MISSING]" % tshark_cmp)
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
if options.dissect_files or options.compare_dir:
|
if options.dissect_files or options.compare_dir:
|
||||||
|
@ -274,18 +276,21 @@ def main():
|
||||||
cap_files.sort()
|
cap_files.sort()
|
||||||
|
|
||||||
options.max_files = min(options.max_files, len(cap_files))
|
options.max_files = min(options.max_files, len(cap_files))
|
||||||
print "%u total files, %u working files" % (len(cap_files), options.max_files)
|
print("%u total files, %u working files" % (len(cap_files), options.max_files))
|
||||||
cap_files = cap_files[:options.max_files]
|
cap_files = cap_files[:options.max_files]
|
||||||
tmpdir = tempfile.mkdtemp()
|
if options.compare_dir or options.dissect_files:
|
||||||
print "Temporary working dir: %s" % tmpdir
|
tmpdir = tempfile.mkdtemp()
|
||||||
|
print("Temporary working dir: %s" % tmpdir)
|
||||||
if options.compare_dir:
|
try:
|
||||||
compare_files(tshark_bin, tmpdir, tshark_cmp, options.num_procs, options.max_files, cap_files)
|
if options.compare_dir:
|
||||||
elif options.dissect_files:
|
compare_files(tshark_bin, tmpdir, tshark_cmp, options.num_procs, options.max_files, cap_files)
|
||||||
dissect_files(tshark_bin, tmpdir, options.num_procs, options.max_files, cap_files)
|
elif options.dissect_files:
|
||||||
else:
|
dissect_files(tshark_bin, tmpdir, options.num_procs, options.max_files, cap_files)
|
||||||
extract_protos_from_file(tshark_bin, options.num_procs, options.max_files, cap_files, cap_hash, index_file_name)
|
else:
|
||||||
|
extract_protos_from_file(tshark_bin, options.num_procs, options.max_files, cap_files, cap_hash, index_file_name)
|
||||||
os.rmdir(tmpdir)
|
finally:
|
||||||
|
# Dissection may result in a non-empty directory.
|
||||||
|
if options.compare_dir:
|
||||||
|
os.rmdir(tmpdir)
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
|
@ -32,13 +32,13 @@
|
||||||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||||
|
|
||||||
# Directory containing binaries. Default current directory.
|
# Directory containing binaries. Default current directory.
|
||||||
BIN_DIR=.
|
WS_BIN_PATH=${WS_BIN_PATH:-.}
|
||||||
|
|
||||||
# Tweak the following to your liking. Editcap must support "-E".
|
# Tweak the following to your liking. Editcap must support "-E".
|
||||||
TSHARK="$BIN_DIR/tshark"
|
TSHARK="$WS_BIN_PATH/tshark"
|
||||||
CAPINFOS="$BIN_DIR/capinfos"
|
CAPINFOS="$WS_BIN_PATH/capinfos"
|
||||||
|
|
||||||
if [ "$BIN_DIR" = "." ]; then
|
if [ "$WS_BIN_PATH" = "." ]; then
|
||||||
export WIRESHARK_RUN_FROM_BUILD_DIRECTORY=
|
export WIRESHARK_RUN_FROM_BUILD_DIRECTORY=
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
@ -67,7 +67,7 @@ do
|
||||||
"$CAPINFOS" "$CF" > /dev/null 2>&1 && FOUND=1
|
"$CAPINFOS" "$CF" > /dev/null 2>&1 && FOUND=1
|
||||||
if [ $FOUND -eq 1 ]
|
if [ $FOUND -eq 1 ]
|
||||||
then
|
then
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
@ -82,7 +82,7 @@ fi
|
||||||
|
|
||||||
for CF in "$@" ; do
|
for CF in "$@" ; do
|
||||||
if [ "$OSTYPE" == "cygwin" ] ; then
|
if [ "$OSTYPE" == "cygwin" ] ; then
|
||||||
CF=`cygpath --windows "$CF"`
|
CF=`cygpath --windows "$CF"`
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ ! -f "$CF" ] ; then
|
if [ ! -f "$CF" ] ; then
|
||||||
|
@ -93,15 +93,15 @@ for CF in "$@" ; do
|
||||||
"$CAPINFOS" "$CF" > /dev/null
|
"$CAPINFOS" "$CF" > /dev/null
|
||||||
RETVAL=$?
|
RETVAL=$?
|
||||||
if [ $RETVAL -ne 0 ] ; then
|
if [ $RETVAL -ne 0 ] ; then
|
||||||
echo "Not a valid capture file (or some other problem)" 1>&2
|
echo "Not a valid capture file (or some other problem)" 1>&2
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
printf "%s: " "$CF"
|
printf "%s: " "$CF"
|
||||||
|
|
||||||
# Extract the protocol names.
|
# Extract the protocol names.
|
||||||
$TSHARK -T fields -eframe.protocols -nr "$CF" 2>/dev/null | tr ':\r' '\n' \
|
$TSHARK -T fields -eframe.protocols -nr "$CF" 2>/dev/null | \
|
||||||
| sort -u | tr '\n\r' ' '
|
tr ':\r' '\n' | sort -u | tr '\n\r' ' '
|
||||||
|
|
||||||
printf "\n"
|
printf "\n"
|
||||||
done
|
done
|
||||||
|
|
Loading…
Reference in New Issue