Add duplicate frame detection (and removal) to editcap. (Some switches

generate duplicate packets when a mirror/SPAN port is misconfigured).

svn path=/trunk/; revision=18800

Makefile.common

@@ -201,7 +201,8 @@ mergecap_SOURCES = \
 
 # editcap specifics
 editcap_SOURCES = \
-	editcap.c
+	editcap.c	\
+	epan/crypt-md5.c
 
 # dftest specifics
 dftest_SOURCES =	\

doc/editcap.pod

@@ -8,6 +8,7 @@ editcap - Edit and/or translate the format of capture files
 B<editcap>
 S<[ B<-c> E<lt>packets per fileE<gt> ]>
 S<[ B<-C> E<lt>choplenE<gt> ]>
+S<[ B<-d> ]>
 S<[ B<-E> E<lt>error probabilityE<gt> ]>
 S<[ B<-F> E<lt>file formatE<gt> ]>
 S<[ B<-A> E<lt>start timeE<gt> ]>
@@ -71,6 +72,12 @@ Each packet is chopped at the packet end by a few <choplen> bytes of data.
 This is useful in the rare case that the conversion between two file 
 formats leaves some random bytes at the end of each packet.
 
+=item -d
+
+Attempts to remove duplicate packets.  The length and MD5 sum of the 
+current packet are compared to the previous four packets.  If a match 
+is found, the packet is skipped.
+
 =item -E  E<lt>error probabilityE<gt>
 
 Sets the probabilty that bytes in the output file are randomly changed.

docbook/wsug_src/WSUG_app_tools.xml

@@ -173,38 +173,67 @@ Usage: capinfos [-t] [-c] [-s] [-d] [-u] [-a] [-e] [-y]
       <title>Help information available from editcap</title>
       <programlisting>
 $ editcap.exe -h
-Usage: editcap [-r] [-h] [-v] [-T &lt;encap type>] [-E &lt;probability>]
+Editcap 0.99.3
-               [-F &lt;capture type>]> [-s &lt;snaplen>] [-t &lt;time adjustment>]
+Edit and/or translate the format of capture files.
-               &lt;infile> &lt;outfile> [ &lt;record#>[-&lt;record#>] ... ]
+See http://www.wireshark.org for more information.
-  where
+
-        -E &lt;probability> specifies the probability (between 0 and 1)
+Usage: editcap [options] ... &lt;infile&gt; &lt;outfile&gt; [ &lt;packet#&gt;[-&lt;packet#&gt;] ... ]
-            that a particular byte will will have an error.
+
-        -F &lt;capture type> specifies the capture file type to write:
+A single packet or a range of packets can be selected.
-            libpcap - libpcap (tcpdump, Wireshark, etc.)
+
-            rh6_1libpcap - RedHat Linux 6.1 libpcap (tcpdump)
+Packets:
-            suse6_3libpcap - SuSE Linux 6.3 libpcap (tcpdump)
+  -C &lt;choplen&gt;           chop each packet at the end by &lt;choplen&gt; bytes
-            modlibpcap - modified libpcap (tcpdump)
+  -d                     remove duplicate packets
-            nokialibpcap - Nokia libpcap (tcpdump)
+  -E &lt;error probability&gt; set the probability (between 0.0 and 1.0 incl.)
-            lanalyzer - Novell LANalyzer
+                         that a particular packet byte will be randomly changed
-            ngsniffer - Network Associates Sniffer (DOS-based)
+  -r                     keep the selected packets, default is to delete them
-            snoop - Sun snoop
+  -s &lt;snaplen&gt;           truncate packets to max. &lt;snaplen&gt; bytes of data
-            netmon1 - Microsoft Network Monitor 1.x
+  -t &lt;time adjustment&gt;   adjust the timestamp of selected packets,
-            netmon2 - Microsoft Network Monitor 2.x
+                         &lt;time adjustment&gt; is in relative seconds (e.g. -0.5)
-            ngwsniffer_1_1 - Network Associates Sniffer (Windows-based) 1.1
+  -A &lt;start time&gt;        don't output packets whose timestamp is before the
-            ngwsniffer_2_0 - Network Associates Sniffer (Windows-based) 2.00x
+                         given time (format as YYYY-MM-DD hh:mm:ss)
-            nettl - HP-UX nettl trace
+  -B &lt;stop time&gt;         don't output packets whose timestamp is after the
-            visual - Visual Networks traffic capture
+                         given time (format as YYYY-MM-DD hh:mm:ss)
+
+Output File(s):
+  -c &lt;packets per file&gt;  split the packet output to different files,
+                         with a maximum of &lt;packets per file&gt; each
+  -F &lt;capture type&gt;      set the output file type, default is libpcap
+                         an empty "-F" option will list the file types
+  -T &lt;encap type&gt;        set the output file encapsulation type,
+                         default is the same as the input file
+                         an empty "-T" option will list the encapsulation types
+
+Miscellaneous:
+  -h                     display this help and exit
+  -v                     verbose output
+
+$ editcap.exe -F
+editcap.exe: option requires an argument -- F
+editcap.exe: The available capture file types for "F":
+    libpcap - Wireshark/tcpdump/... - libpcap
+    nseclibpcap - Wireshark - nanosecond libpcap
+    modlibpcap - Modified tcpdump - libpcap
+    nokialibpcap - Nokia tcpdump - libpcap
+    rh6_1libpcap - RedHat 6.1 tcpdump - libpcap
+    suse6_3libpcap - SuSE 6.3 tcpdump - libpcap
     5views - Accellent 5Views capture
-            niobserverv9 - Network Instruments Observer version 9
+    dct2000 - Catapult DCT2000 trace (.out format)
-            default is libpcap
+    nettl - HP-UX nettl trace
-        -h produces this help listing.
+    netmon1 - Microsoft NetMon 1.x
-        -r specifies that the records specified should be kept, not deleted,
+    netmon2 - Microsoft NetMon 2.x
-                           default is to delete
+    ngsniffer - NA Sniffer (DOS)
-        -s &lt;snaplen> specifies that packets should be truncated to
+    ngwsniffer_1_1 - NA Sniffer (Windows) 1.1
-           &lt;snaplen> bytes of data
+    ngwsniffer_2_0 - NA Sniffer (Windows) 2.00x
-        -t &lt;time adjustment> specifies the time adjustment
+    niobserverv9 - Network Instruments Observer (V9)
-           to be applied to selected packets
+    lanalyzer - Novell LANalyzer
-        -T &lt;encap type> specifies the encapsulation type to use:
+    snoop - Sun snoop
+    rf5 - Tektronix K12xx 32-bit .rf5 format
+    visual - Visual Networks traffic capture
+
+$ editcap.exe -F
+editcap.exe: option requires an argument -- T
+editcap.exe: The available encapsulation types for "T":
     ether - Ethernet
     tr - Token Ring
     slip - SLIP
@@ -284,10 +313,17 @@ Usage: editcap [-r] [-h] [-v] [-T &lt;encap type>] [-E &lt;probability>]
     gcom-tie1 - GCOM TIE1
     gcom-serial - GCOM Serial
     x25-nettl - X25 with nettl headers
-            default is the same as the input file
+    k12 - K12 protocol analyzer
-        -v specifies verbose operation, default is silent
+    juniper-mlppp - Juniper MLPPP
-
+    juniper-mlfr - Juniper MLFR
-            A range of records can be specified as well
+    juniper-ether - Juniper Ethernet
+    juniper-ppp - Juniper PPP
+    juniper-frelay - Juniper Frame-Relay
+    juniper-chdlc - Juniper C-HDLC
+    juniper-ggsn - Juniper GGSN
+    lapd - LAPD
+    dct2000 - Catapult DCT2000
+    ber - ASN.1 Basic Encoding Rules
       </programlisting>
     </example>
 

editcap.c

@@ -48,6 +48,8 @@
 # include "strptime.h"
 #endif
 
+#include "epan/crypt-md5.h"
+
 #include "svnversion.h"
 
 /*
@@ -61,6 +63,19 @@ struct select_item {
 
 };
 
+
+/*
+ * Duplicate frame detection
+ */
+typedef struct _fd_hash_t {
+  md5_byte_t digest[16];
+  guint32 len;
+} fd_hash_t;
+
+#define DUP_DEPTH 5
+fd_hash_t fd_hash[DUP_DEPTH];
+int cur_dup = 0;
+
 #define ONE_MILLION 1000000
 
 /* Weights of different errors we can introduce */
@@ -93,6 +108,7 @@ static double err_prob = 0.0;
 static time_t starttime = 0;
 static time_t stoptime = 0;
 static gboolean check_startstop = FALSE;
+static gboolean dup_detect = FALSE;
 
 /* Add a selection item, a simple parser for now */
 
@@ -228,6 +244,36 @@ set_time_adjustment(char *optarg)
   time_adj.tv.tv_usec = val;
 }
 
+static gboolean
+is_duplicate(guint8* fd, guint32 len) {
+  int i;
+  md5_state_t ms;
+
+  cur_dup++;
+  if (cur_dup >= DUP_DEPTH)
+    cur_dup = 0;
+
+  /* Calculate our digest */
+  md5_init(&ms);
+  md5_append(&ms, fd, len);
+  md5_finish(&ms, fd_hash[cur_dup].digest);
+
+  fd_hash[cur_dup].len = len;
+
+  /* Look for duplicates */
+  for (i = 0; i < DUP_DEPTH; i++) {
+    if (i == cur_dup)
+      continue;
+
+    if (fd_hash[i].len == fd_hash[cur_dup].len &&
+        memcmp(fd_hash[i].digest, fd_hash[cur_dup].digest, 16) == 0) {
+      return TRUE;
+    }
+  }
+
+  return FALSE;
+}
+
 static void usage(void)
 {
   fprintf(stderr, "Editcap %s"
@@ -244,6 +290,7 @@ static void usage(void)
   fprintf(stderr, "\n");
   fprintf(stderr, "Packets:\n");
   fprintf(stderr, "  -C <choplen>           chop each packet at the end by <choplen> bytes\n");
+  fprintf(stderr, "  -d                     remove duplicate packets\n");
   fprintf(stderr, "  -E <error probability> set the probability (between 0.0 and 1.0 incl.)\n");
   fprintf(stderr, "                         that a particular packet byte will be randomly changed\n");
   fprintf(stderr, "  -r                     keep the selected packets, default is to delete them\n");
@@ -319,7 +366,7 @@ int main(int argc, char *argv[])
 
   /* Process the options first */
 
-  while ((opt = getopt(argc, argv, "A:B:c:C:E:F:hrs:t:T:v")) !=-1) {
+  while ((opt = getopt(argc, argv, "A:B:c:C:dE:F:hrs:t:T:v")) !=-1) {
 
     switch (opt) {
 
@@ -366,6 +413,14 @@ int main(int argc, char *argv[])
       }
       break;
 
+    case 'd':
+      dup_detect = TRUE;
+      for (i = 0; i < DUP_DEPTH; i++) {
+	memset(&fd_hash[i].digest, 0, 16);
+	fd_hash[i].len = 0;
+      }
+      break;
+
     case '?':              /* Bad options if GNU getopt */
       switch(optopt) {
       case'F':
@@ -619,6 +674,16 @@ int main(int argc, char *argv[])
           phdr = &snap_phdr;
         }
 
+	if (dup_detect) {
+	  buf = wtap_buf_ptr(wth);
+	  if (is_duplicate(buf, phdr->caplen)) {
+            if (verbose)
+              printf("Skipping duplicate: %u\n", count);
+            count++;
+	    continue;
+          }
+	}
+
         if (err_prob > 0.0) {
           buf = wtap_buf_ptr(wth);
           for (i = 0; i < (int) phdr->caplen; i++) {