Follow stream: Modify YAML format, add timestamps and peers

Modify YAML output format so it includes information about peers and
absolute timestamps for each packet.
This also adds yaml output to tshark: -z follow,tcp,yaml,X

doc/tshark.pod

@@ -1195,6 +1195,7 @@ I<mode> specifies the output mode.  It can be one of:
   ebcdic EBCDIC output with dots for non-printable characters
   hex    Hexadecimal and ASCII data with offsets
   raw    Hexadecimal data
+  yaml   YAML format
 
 Since the output in B<ascii> or B<ebcdic> mode may contain newlines, the length
 of each section of output plus a newline precedes each section of output.

docbook/release-notes.adoc

@@ -69,6 +69,9 @@ They previously shipped with Npcap 1.20.
 
 * Follow stream is now able to follow SIP call by SIP.Call-ID
 
+* Follow stream YAML output format’s has been changed to add timestamps and peers information (for more details see the user’s guide,
+{wireshark-users-guide-url}/ChAdvFollowStreamSection.html[Following Protocol Streams])
+
 // === Removed Features and Support
 
 // === Removed Dissectors

docbook/wsug_src/WSUG_chapter_advanced.adoc

@@ -97,6 +97,78 @@ menu:UTF-16[]:: Like ASCII, but decode the data as UTF-16.
 
 menu:YAML[]:: This allows you to load the stream as YAML.
 
+The YAML output is divided into 2 main sections:
+
+* The `peers` section where for each `peer` you found the peer index, the `host` address and the `port` number.
+
+* The `packets` section where for each `packet` you found the packet number in the original capture, the `peer` index,
+the packet `index` for this peer, the `timestamp` in seconds and the `data` in base64 encoding.
+
+.Follow Stream YAML output
+====
+[source,yaml]
+----
+peers:
+  - peer: 0
+    host: 127.0.0.1
+    port: 54048
+  - peer: 1
+    host: 127.0.10.1
+    port: 5000
+packets:
+  - packet: 1
+    peer: 0
+    index: 0
+    timestamp: 1599485409.693955274
+    data: !!binary |
+      aGVsbG8K
+  - packet: 3
+    peer: 1
+    index: 0
+    timestamp: 1599485423.885866692
+    data: !!binary |
+      Ym9uam91cgo=
+----
+====
+
+The same example but in old YAML format (before version 3.5):
+[source,yaml]
+----
+# Packet 1
+peer0_0: !!binary |
+  aGVsbG8K
+# Packet 3
+peer1_0: !!binary |
+  Ym9uam91cgo=
+----
+
+How the old format data can be found in the new format:
+[options="header"]
+|===
+|New YAML format |Old YAML format | 
+a|
+----
+...
+packets:
+  - packet: AAA
+    peer: BBB
+    index: CCC
+    data: !!binary \|
+      DDD
+----
+a|
+----
+# Packet AAA
+peerBBB_CCC !!binary \|
+  DDD
+----
+a|
+ AAA: packet number in the original capture
+ BBB: peer index
+ CCC: packet index for this peer
+ DDD: data in base64 encoding
+|===
+
 menu:Raw[]:: This allows you to load the unaltered stream data into a different
   program for further examination. The display will look the same as the ASCII
   setting, but “Save As” will result in a binary file.

epan/dissectors/packet-tcp.c

@@ -1089,6 +1089,7 @@ check_follow_fragments(follow_info_t *follow_info, gboolean is_server, guint32 a
 
                     follow_record->is_server = is_server;
                     follow_record->packet_num = fragment->packet_num;
+                    follow_record->abs_ts = fragment->abs_ts;
                     follow_record->seq = follow_info->seq[is_server] + new_frag_size;
 
                     follow_record->data = g_byte_array_append(g_byte_array_new(),
@@ -1222,6 +1223,7 @@ follow_tcp_tap_listener(void *tapdata, packet_info *pinfo,
     follow_record = g_new0(follow_record_t, 1);
     follow_record->is_server = is_server;
     follow_record->packet_num = pinfo->fd->num;
+    follow_record->abs_ts = pinfo->fd->abs_ts;
     follow_record->seq = sequence;  /* start of fragment, used by check_follow_fragments. */
     follow_record->data = g_byte_array_append(g_byte_array_new(),
                                               tvb_get_ptr(follow_data->tvb, data_offset, data_length),

epan/dissectors/packet-tls.c

@@ -463,6 +463,7 @@ ssl_follow_tap_listener(void *tapdata, packet_info *pinfo, epan_dissect_t *edt _
 
         follow_record->is_server = (from == FROM_SERVER);
         follow_record->packet_num = pinfo->num;
+        follow_record->abs_ts = pinfo->abs_ts;
 
         follow_record->data = g_byte_array_sized_new(appl_data->data_len);
         follow_record->data = g_byte_array_append(follow_record->data,

epan/follow.c

@@ -190,6 +190,7 @@ follow_tvb_tap_listener(void *tapdata, packet_info *pinfo,
                                               tvb_get_ptr(next_tvb, 0, -1),
                                               tvb_captured_length(next_tvb));
     follow_record->packet_num = pinfo->fd->num;
+    follow_record->abs_ts = pinfo->fd->abs_ts;
 
     if (follow_info->client_port == 0) {
         follow_info->client_port = pinfo->srcport;

epan/follow.h

@@ -82,6 +82,7 @@ typedef struct {
     gboolean is_server;
     guint32 packet_num;
     guint32 seq; /* TCP only */
+    nstime_t abs_ts; /**< Packet absolute time stamp */
     GByteArray *data;
 } follow_record_t;
 

ui/cli/tap-follow.c

@@ -53,6 +53,7 @@ typedef struct _cli_follow_info {
 #define STR_ASCII       ",ascii"
 #define STR_EBCDIC      ",ebcdic"
 #define STR_RAW         ",raw"
+#define STR_YAML        ",yaml"
 
 WS_NORETURN static void follow_exit(const char *strp)
 {
@@ -68,6 +69,7 @@ static const char * follow_str_type(cli_follow_info_t* cli_follow_info)
   case SHOW_ASCII:      return "ascii";
   case SHOW_EBCDIC:     return "ebcdic";
   case SHOW_RAW:        return "raw";
+  case SHOW_YAML:       return "yaml";
   default:
     g_assert_not_reached();
     break;
@@ -168,22 +170,43 @@ static void follow_draw(void *contextp)
   GList             *cur;
   follow_record_t   *follow_record;
   guint             chunk;
+  gchar             *b64encoded;
+  const guint32     base64_raw_len = 57; /* Encodes to 76 bytes, common in RFCs */
 
-  printf("\n%s", separator);
+  /* Print header */
-  printf("Follow: %s,%s\n", proto_get_protocol_filter_name(get_follow_proto_id(cli_follow_info->follower)), follow_str_type(cli_follow_info));
+  switch (cli_follow_info->show_type)
-  printf("Filter: %s\n", follow_info->filter_out_filter);
+  {
+    case SHOW_YAML:
+      printf("peers:\n");
+      printf("  - peer: 0\n");
+      address_to_str_buf(&follow_info->client_ip, buf, sizeof buf);
+      printf("    host: %s\n", buf);
+      printf("    port: %d\n", follow_info->client_port);
+      printf("  - peer: 1\n");
+      address_to_str_buf(&follow_info->server_ip, buf, sizeof buf);
+      printf("    host: %s\n", buf);
+      printf("    port: %d\n", follow_info->server_port);
+      printf("packets:\n");
+      break;
 
-  address_to_str_buf(&follow_info->client_ip, buf, sizeof buf);
+    default:
-  if (follow_info->client_ip.type == AT_IPv6)
+      printf("\n%s", separator);
-    printf("Node 0: [%s]:%u\n", buf, follow_info->client_port);
+      printf("Follow: %s,%s\n", proto_get_protocol_filter_name(get_follow_proto_id(cli_follow_info->follower)), follow_str_type(cli_follow_info));
-  else
+      printf("Filter: %s\n", follow_info->filter_out_filter);
-    printf("Node 0: %s:%u\n", buf, follow_info->client_port);
 
-  address_to_str_buf(&follow_info->server_ip, buf, sizeof buf);
+      address_to_str_buf(&follow_info->client_ip, buf, sizeof buf);
-  if (follow_info->client_ip.type == AT_IPv6)
+      if (follow_info->client_ip.type == AT_IPv6)
-    printf("Node 1: [%s]:%u\n", buf, follow_info->server_port);
+        printf("Node 0: [%s]:%u\n", buf, follow_info->client_port);
-  else
+      else
-    printf("Node 1: %s:%u\n", buf, follow_info->server_port);
+        printf("Node 0: %s:%u\n", buf, follow_info->client_port);
+
+      address_to_str_buf(&follow_info->server_ip, buf, sizeof buf);
+      if (follow_info->server_ip.type == AT_IPv6)
+        printf("Node 1: [%s]:%u\n", buf, follow_info->server_port);
+      else
+        printf("Node 1: %s:%u\n", buf, follow_info->server_port);
+      break;
+  }
 
   for (cur = g_list_last(follow_info->payload), chunk = 1;
        cur != NULL;
@@ -202,9 +225,11 @@ static void follow_draw(void *contextp)
       continue;
     }
 
+    /* Print start of line */
     switch (cli_follow_info->show_type)
     {
     case SHOW_HEXDUMP:
+    case SHOW_YAML:
       break;
 
     case SHOW_ASCII:
@@ -218,10 +243,12 @@ static void follow_draw(void *contextp)
         putchar('\t');
       }
       break;
+
     default:
       g_assert_not_reached();
     }
 
+    /* Print data */
     switch (cli_follow_info->show_type)
     {
     case SHOW_HEXDUMP:
@@ -271,12 +298,38 @@ static void follow_draw(void *contextp)
       g_free(buffer);
       break;
 
+    case SHOW_YAML:
+      printf("  - packet: %d\n", follow_record->packet_num);
+      printf("    peer: %d\n", follow_record->is_server ? 1 : 0);
+      printf("    timestamp: %" G_GINT64_MODIFIER "u.%09d\n", follow_record->abs_ts.secs, follow_record->abs_ts.nsecs);
+      printf("    data: !!binary |\n");
+      ii = 0;
+      while (ii < follow_record->data->len) {
+          guint32 len = ii + base64_raw_len < follow_record->data->len
+                ? base64_raw_len
+                : follow_record->data->len - ii;
+          b64encoded = g_base64_encode(&follow_record->data->data[ii], len);
+          printf("      %s\n", b64encoded);
+          g_free(b64encoded);
+          ii += len;
+      }
+      break;
+
     default:
       g_assert_not_reached();
     }
   }
 
-  printf("%s", separator);
+  /* Print footer */
+  switch (cli_follow_info->show_type)
+  {
+    case SHOW_YAML:
+      break;
+
+    default:
+      printf("%s", separator);
+      break;
+  }
 }
 
 static gboolean follow_arg_strncmp(const char **opt_argp, const char *strp)
@@ -312,6 +365,10 @@ follow_arg_mode(const char **opt_argp, follow_info_t *follow_info)
   {
     cli_follow_info->show_type = SHOW_RAW;
   }
+  else if (follow_arg_strncmp(opt_argp, STR_YAML))
+  {
+    cli_follow_info->show_type = SHOW_YAML;
+  }
   else
   {
     follow_exit("Invalid display mode.");

ui/qt/follow_stream_dialog.cpp

@@ -554,7 +554,7 @@ FollowStreamDialog::followStream()
 }
 
 const int FollowStreamDialog::max_document_length_ = 500 * 1000 * 1000; // Just a guess
-void FollowStreamDialog::addText(QString text, gboolean is_from_server, guint32 packet_num)
+void FollowStreamDialog::addText(QString text, gboolean is_from_server, guint32 packet_num, gboolean colorize)
 {
     if (truncated_) {
         return;
@@ -569,8 +569,12 @@ void FollowStreamDialog::addText(QString text, gboolean is_from_server, guint32
     setUpdatesEnabled(false);
     int cur_pos = ui->teStreamContent->verticalScrollBar()->value();
     ui->teStreamContent->moveCursor(QTextCursor::End);
+
     QTextCharFormat tcf = ui->teStreamContent->currentCharFormat();
-    if (is_from_server) {
+    if (!colorize) {
+        tcf.setBackground(palette().window().color());
+        tcf.setForeground(palette().windowText().color());
+    } else if (is_from_server) {
         tcf.setForeground(ColorUtils::fromColorT(prefs.st_server_fg));
         tcf.setBackground(ColorUtils::fromColorT(prefs.st_server_bg));
     } else {
@@ -654,7 +658,7 @@ static inline void sanitize_buffer(char *buffer, size_t nchars) {
 
 frs_return_t
 FollowStreamDialog::showBuffer(char *buffer, size_t nchars, gboolean is_from_server, guint32 packet_num,
-                                guint32 *global_pos)
+                                nstime_t abs_ts, guint32 *global_pos)
 {
     gchar initbuf[256];
     guint32 current_pos;
@@ -795,17 +799,53 @@ FollowStreamDialog::showBuffer(char *buffer, size_t nchars, gboolean is_from_ser
         const int base64_raw_len = 57; // Encodes to 76 bytes, common in RFCs
         current_pos = 0;
 
+        if (last_packet_ == 0) {
+            // Header with general info about peers
+            const char *hostname0 = address_to_name(&follow_info_.client_ip);
+            const char *hostname1 = address_to_name(&follow_info_.server_ip);
+
+            char *port0 = get_follow_port_to_display(follower_)(NULL, follow_info_.client_port);
+            char *port1 = get_follow_port_to_display(follower_)(NULL, follow_info_.server_port);
+
+            addText("peers:\n", false, 0, false);
+
+            addText(QString(
+                "  - peer: 0\n"
+                "    host: %1\n"
+                "    port: %2\n")
+                .arg(hostname0)
+                .arg(port0), false, 0);
+
+            addText(QString(
+                "  - peer: 1\n"
+                "    host: %1\n"
+                "    port: %2\n")
+                .arg(hostname1)
+                .arg(port1), true, 0);
+
+            wmem_free(NULL, port0);
+            wmem_free(NULL, port1);
+
+            addText("packets:\n", false, 0, false);
+        }
+
         if (packet_num != last_packet_) {
-            yaml_text.append(QString("# Packet %1\npeer%2_%3: !!binary |\n")
+            yaml_text.append(QString("  - packet: %1\n")
-                    .arg(packet_num)
+                    .arg(packet_num));
-                    .arg(is_from_server ? 1 : 0)
+            yaml_text.append(QString("    peer: %1\n")
+                    .arg(is_from_server ? 1 : 0));
+            yaml_text.append(QString("    index: %1\n")
                     .arg(is_from_server ? server_buffer_count_++ : client_buffer_count_++));
+            yaml_text.append(QString("    timestamp: %1.%2\n")
+                    .arg(abs_ts.secs)
+                    .arg(abs_ts.nsecs, 9, 10, QChar('0')));
+            yaml_text.append(QString("    data: !!binary |\n"));
         }
         while (current_pos < nchars) {
             int len = current_pos + base64_raw_len < nchars ? base64_raw_len : (int) nchars - current_pos;
             QByteArray base64_data(&buffer[current_pos], len);
 
-            yaml_text += "  " + base64_data.toBase64() + "\n";
+            yaml_text += "      " + base64_data.toBase64() + "\n";
 
             current_pos += len;
             (*global_pos) += len;
@@ -1160,6 +1200,7 @@ FollowStreamDialog::readFollowStream()
                         follow_record->data->len,
                         follow_record->is_server,
                         follow_record->packet_num,
+                        follow_record->abs_ts,
                         global_pos);
             if (frs_return == FRS_PRINT_ERROR)
                 return frs_return;

ui/qt/follow_stream_dialog.h

@@ -83,14 +83,14 @@ private:
     void updateWidgets() { updateWidgets(false); } // Needed for WiresharkDialog?
     frs_return_t
     showBuffer(char *buffer, size_t nchars, gboolean is_from_server,
-                guint32 packet_num, guint32 *global_pos);
+                guint32 packet_num, nstime_t abs_ts, guint32 *global_pos);
 
     frs_return_t readStream();
     frs_return_t readFollowStream();
     frs_return_t readSslStream();
 
     void followStream();
-    void addText(QString text, gboolean is_from_server, guint32 packet_num);
+    void addText(QString text, gboolean is_from_server, guint32 packet_num, gboolean colorize = true);
 
     Ui::FollowStreamDialog  *ui;