osmocom
/
wireshark
11
0
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Follow stream: Modify YAML format, add timestamps and peers 

Modify YAML output format so it includes information about peers and
absolute timestamps for each packet.
This also adds yaml output to tshark: -z follow,tcp,yaml,X
This commit is contained in:
Toff 2020-09-09 17:06:15 +02:00 committed by Wireshark GitLab Utility
parent 48ba793ef6
commit 2df04e5bb0
10 changed files with 202 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/tshark.pod
View File

@ -1195,6 +1195,7 @@ I<mode> specifies the output mode. It can be one of:
ebcdic EBCDIC output with dots for non-printable characters
hex Hexadecimal and ASCII data with offsets
raw Hexadecimal data
yaml YAML format
Since the output in B<ascii> or B<ebcdic> mode may contain newlines, the length
of each section of output plus a newline precedes each section of output.

3
docbook/release-notes.adoc
View File

@ -69,6 +69,9 @@ They previously shipped with Npcap 1.20.
* Follow stream is now able to follow SIP call by SIP.Call-ID
* Follow stream YAML output format’s has been changed to add timestamps and peers information (for more details see the user’s guide,
{wireshark-users-guide-url}/ChAdvFollowStreamSection.html[Following Protocol Streams])
// === Removed Features and Support
// === Removed Dissectors

72
docbook/wsug_src/WSUG_chapter_advanced.adoc
View File

@ -97,6 +97,78 @@ menu:UTF-16[]:: Like ASCII, but decode the data as UTF-16.
menu:YAML[]:: This allows you to load the stream as YAML.
The YAML output is divided into 2 main sections:
* The `peers` section where for each `peer` you found the peer index, the `host` address and the `port` number.
* The `packets` section where for each `packet` you found the packet number in the original capture, the `peer` index,
the packet `index` for this peer, the `timestamp` in seconds and the `data` in base64 encoding.
.Follow Stream YAML output
====
[source,yaml]
----
peers:
- peer: 0
host: 127.0.0.1
port: 54048
- peer: 1
host: 127.0.10.1
port: 5000
packets:
- packet: 1
peer: 0
index: 0
timestamp: 1599485409.693955274
data: !!binary |
aGVsbG8K
- packet: 3
peer: 1
index: 0
timestamp: 1599485423.885866692
data: !!binary |
Ym9uam91cgo=
----
====
The same example but in old YAML format (before version 3.5):
[source,yaml]
----
# Packet 1
peer0_0: !!binary |
aGVsbG8K
# Packet 3
peer1_0: !!binary |
Ym9uam91cgo=
----
How the old format data can be found in the new format:
[options="header"]
|===
|New YAML format |Old YAML format |
a|
----
...
packets:
- packet: AAA
peer: BBB
index: CCC
data: !!binary \|
DDD
----
a|
----
# Packet AAA
peerBBB_CCC !!binary \|
DDD
----
a|
AAA: packet number in the original capture
BBB: peer index
CCC: packet index for this peer
DDD: data in base64 encoding
|===
menu:Raw[]:: This allows you to load the unaltered stream data into a different
program for further examination. The display will look the same as the ASCII
setting, but “Save As” will result in a binary file.

2
epan/dissectors/packet-tcp.c
View File

@ -1089,6 +1089,7 @@ check_follow_fragments(follow_info_t *follow_info, gboolean is_server, guint32 a
follow_record->is_server = is_server;
follow_record->packet_num = fragment->packet_num;
follow_record->abs_ts = fragment->abs_ts;
follow_record->seq = follow_info->seq[is_server] + new_frag_size;
follow_record->data = g_byte_array_append(g_byte_array_new(),
@ -1222,6 +1223,7 @@ follow_tcp_tap_listener(void *tapdata, packet_info *pinfo,
follow_record = g_new0(follow_record_t, 1);
follow_record->is_server = is_server;
follow_record->packet_num = pinfo->fd->num;
follow_record->abs_ts = pinfo->fd->abs_ts;
follow_record->seq = sequence; /* start of fragment, used by check_follow_fragments. */
follow_record->data = g_byte_array_append(g_byte_array_new(),
tvb_get_ptr(follow_data->tvb, data_offset, data_length),

1
epan/dissectors/packet-tls.c
View File

@ -463,6 +463,7 @@ ssl_follow_tap_listener(void *tapdata, packet_info *pinfo, epan_dissect_t *edt _
follow_record->is_server = (from == FROM_SERVER);
follow_record->packet_num = pinfo->num;
follow_record->abs_ts = pinfo->abs_ts;
follow_record->data = g_byte_array_sized_new(appl_data->data_len);
follow_record->data = g_byte_array_append(follow_record->data,

1
epan/follow.c
View File

@ -190,6 +190,7 @@ follow_tvb_tap_listener(void *tapdata, packet_info *pinfo,
tvb_get_ptr(next_tvb, 0, -1),
tvb_captured_length(next_tvb));
follow_record->packet_num = pinfo->fd->num;
follow_record->abs_ts = pinfo->fd->abs_ts;
if (follow_info->client_port == 0) {
follow_info->client_port = pinfo->srcport;

1
epan/follow.h
View File

@ -82,6 +82,7 @@ typedef struct {
gboolean is_server;
guint32 packet_num;
guint32 seq; /* TCP only */
nstime_t abs_ts; /**< Packet absolute time stamp */
GByteArray *data;
} follow_record_t;

85
ui/cli/tap-follow.c
View File

@ -53,6 +53,7 @@ typedef struct _cli_follow_info {
#define STR_ASCII ",ascii"
#define STR_EBCDIC ",ebcdic"
#define STR_RAW ",raw"
#define STR_YAML ",yaml"
WS_NORETURN static void follow_exit(const char *strp)
{
@ -68,6 +69,7 @@ static const char * follow_str_type(cli_follow_info_t* cli_follow_info)
case SHOW_ASCII: return "ascii";
case SHOW_EBCDIC: return "ebcdic";
case SHOW_RAW: return "raw";
case SHOW_YAML: return "yaml";
default:
g_assert_not_reached();
break;
@ -168,22 +170,43 @@ static void follow_draw(void *contextp)
GList *cur;
follow_record_t *follow_record;
guint chunk;
gchar *b64encoded;
const guint32 base64_raw_len = 57; /* Encodes to 76 bytes, common in RFCs */
printf("\n%s", separator);
printf("Follow: %s,%s\n", proto_get_protocol_filter_name(get_follow_proto_id(cli_follow_info->follower)), follow_str_type(cli_follow_info));
printf("Filter: %s\n", follow_info->filter_out_filter);
/* Print header */
switch (cli_follow_info->show_type)
{
case SHOW_YAML:
printf("peers:\n");
printf(" - peer: 0\n");
address_to_str_buf(&follow_info->client_ip, buf, sizeof buf);
printf(" host: %s\n", buf);
printf(" port: %d\n", follow_info->client_port);
printf(" - peer: 1\n");
address_to_str_buf(&follow_info->server_ip, buf, sizeof buf);
printf(" host: %s\n", buf);
printf(" port: %d\n", follow_info->server_port);
printf("packets:\n");
break;
address_to_str_buf(&follow_info->client_ip, buf, sizeof buf);
if (follow_info->client_ip.type == AT_IPv6)
printf("Node 0: [%s]:%u\n", buf, follow_info->client_port);
else
printf("Node 0: %s:%u\n", buf, follow_info->client_port);
default:
printf("\n%s", separator);
printf("Follow: %s,%s\n", proto_get_protocol_filter_name(get_follow_proto_id(cli_follow_info->follower)), follow_str_type(cli_follow_info));
printf("Filter: %s\n", follow_info->filter_out_filter);
address_to_str_buf(&follow_info->server_ip, buf, sizeof buf);
if (follow_info->client_ip.type == AT_IPv6)
printf("Node 1: [%s]:%u\n", buf, follow_info->server_port);
else
printf("Node 1: %s:%u\n", buf, follow_info->server_port);
address_to_str_buf(&follow_info->client_ip, buf, sizeof buf);
if (follow_info->client_ip.type == AT_IPv6)
printf("Node 0: [%s]:%u\n", buf, follow_info->client_port);
else
printf("Node 0: %s:%u\n", buf, follow_info->client_port);
address_to_str_buf(&follow_info->server_ip, buf, sizeof buf);
if (follow_info->server_ip.type == AT_IPv6)
printf("Node 1: [%s]:%u\n", buf, follow_info->server_port);
else
printf("Node 1: %s:%u\n", buf, follow_info->server_port);
break;
}
for (cur = g_list_last(follow_info->payload), chunk = 1;
cur != NULL;
@ -202,9 +225,11 @@ static void follow_draw(void *contextp)
continue;
}
/* Print start of line */
switch (cli_follow_info->show_type)
{
case SHOW_HEXDUMP:
case SHOW_YAML:
break;
case SHOW_ASCII:
@ -218,10 +243,12 @@ static void follow_draw(void *contextp)
putchar('\t');
}
break;
default:
g_assert_not_reached();
}
/* Print data */
switch (cli_follow_info->show_type)
{
case SHOW_HEXDUMP:
@ -271,12 +298,38 @@ static void follow_draw(void *contextp)
g_free(buffer);
break;
case SHOW_YAML:
printf(" - packet: %d\n", follow_record->packet_num);
printf(" peer: %d\n", follow_record->is_server ? 1 : 0);
printf(" timestamp: %" G_GINT64_MODIFIER "u.%09d\n", follow_record->abs_ts.secs, follow_record->abs_ts.nsecs);
printf(" data: !!binary |\n");
ii = 0;
while (ii < follow_record->data->len) {
guint32 len = ii + base64_raw_len < follow_record->data->len
? base64_raw_len
: follow_record->data->len - ii;
b64encoded = g_base64_encode(&follow_record->data->data[ii], len);
printf(" %s\n", b64encoded);
g_free(b64encoded);
ii += len;
}
break;
default:
g_assert_not_reached();
}
}
printf("%s", separator);
/* Print footer */
switch (cli_follow_info->show_type)
{
case SHOW_YAML:
break;
default:
printf("%s", separator);
break;
}
}
static gboolean follow_arg_strncmp(const char **opt_argp, const char *strp)
@ -312,6 +365,10 @@ follow_arg_mode(const char **opt_argp, follow_info_t *follow_info)
{
cli_follow_info->show_type = SHOW_RAW;
}
else if (follow_arg_strncmp(opt_argp, STR_YAML))
{
cli_follow_info->show_type = SHOW_YAML;
}
else
{
follow_exit("Invalid display mode.");

55
ui/qt/follow_stream_dialog.cpp
View File

@ -554,7 +554,7 @@ FollowStreamDialog::followStream()
}
const int FollowStreamDialog::max_document_length_ = 500 * 1000 * 1000; // Just a guess
void FollowStreamDialog::addText(QString text, gboolean is_from_server, guint32 packet_num)
void FollowStreamDialog::addText(QString text, gboolean is_from_server, guint32 packet_num, gboolean colorize)
{
if (truncated_) {
return;
@ -569,8 +569,12 @@ void FollowStreamDialog::addText(QString text, gboolean is_from_server, guint32
setUpdatesEnabled(false);
int cur_pos = ui->teStreamContent->verticalScrollBar()->value();
ui->teStreamContent->moveCursor(QTextCursor::End);
QTextCharFormat tcf = ui->teStreamContent->currentCharFormat();
if (is_from_server) {
if (!colorize) {
tcf.setBackground(palette().window().color());
tcf.setForeground(palette().windowText().color());
} else if (is_from_server) {
tcf.setForeground(ColorUtils::fromColorT(prefs.st_server_fg));
tcf.setBackground(ColorUtils::fromColorT(prefs.st_server_bg));
} else {
@ -654,7 +658,7 @@ static inline void sanitize_buffer(char *buffer, size_t nchars) {
frs_return_t
FollowStreamDialog::showBuffer(char *buffer, size_t nchars, gboolean is_from_server, guint32 packet_num,
guint32 *global_pos)
nstime_t abs_ts, guint32 *global_pos)
{
gchar initbuf[256];
guint32 current_pos;
@ -795,17 +799,53 @@ FollowStreamDialog::showBuffer(char *buffer, size_t nchars, gboolean is_from_ser
const int base64_raw_len = 57; // Encodes to 76 bytes, common in RFCs
current_pos = 0;
if (last_packet_ == 0) {
// Header with general info about peers
const char *hostname0 = address_to_name(&follow_info_.client_ip);
const char *hostname1 = address_to_name(&follow_info_.server_ip);
char *port0 = get_follow_port_to_display(follower_)(NULL, follow_info_.client_port);
char *port1 = get_follow_port_to_display(follower_)(NULL, follow_info_.server_port);
addText("peers:\n", false, 0, false);
addText(QString(
" - peer: 0\n"
" host: %1\n"
" port: %2\n")
.arg(hostname0)
.arg(port0), false, 0);
addText(QString(
" - peer: 1\n"
" host: %1\n"
" port: %2\n")
.arg(hostname1)
.arg(port1), true, 0);
wmem_free(NULL, port0);
wmem_free(NULL, port1);
addText("packets:\n", false, 0, false);
}
if (packet_num != last_packet_) {
yaml_text.append(QString("# Packet %1\npeer%2_%3: !!binary |\n")
.arg(packet_num)
.arg(is_from_server ? 1 : 0)
yaml_text.append(QString(" - packet: %1\n")
.arg(packet_num));
yaml_text.append(QString(" peer: %1\n")
.arg(is_from_server ? 1 : 0));
yaml_text.append(QString(" index: %1\n")
.arg(is_from_server ? server_buffer_count_++ : client_buffer_count_++));
yaml_text.append(QString(" timestamp: %1.%2\n")
.arg(abs_ts.secs)
.arg(abs_ts.nsecs, 9, 10, QChar('0')));
yaml_text.append(QString(" data: !!binary |\n"));
}
while (current_pos < nchars) {
int len = current_pos + base64_raw_len < nchars ? base64_raw_len : (int) nchars - current_pos;
QByteArray base64_data(&buffer[current_pos], len);
yaml_text += " " + base64_data.toBase64() + "\n";
yaml_text += " " + base64_data.toBase64() + "\n";
current_pos += len;
(*global_pos) += len;
@ -1160,6 +1200,7 @@ FollowStreamDialog::readFollowStream()
follow_record->data->len,
follow_record->is_server,
follow_record->packet_num,
follow_record->abs_ts,
global_pos);
if (frs_return == FRS_PRINT_ERROR)
return frs_return;

4
ui/qt/follow_stream_dialog.h
View File

@ -83,14 +83,14 @@ private:
void updateWidgets() { updateWidgets(false); } // Needed for WiresharkDialog?
frs_return_t
showBuffer(char *buffer, size_t nchars, gboolean is_from_server,
guint32 packet_num, guint32 *global_pos);
guint32 packet_num, nstime_t abs_ts, guint32 *global_pos);
frs_return_t readStream();
frs_return_t readFollowStream();
frs_return_t readSslStream();
void followStream();
void addText(QString text, gboolean is_from_server, guint32 packet_num);
void addText(QString text, gboolean is_from_server, guint32 packet_num, gboolean colorize = true);
Ui::FollowStreamDialog *ui;