osmocom
/
wireshark
11
0
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

HTTP2/gRPC: Support streaming mode reassembly 

If working in streaming RPC mode, many grpc messages will be
contained in one http2 stream, the stream will end very late
(for example ETCD watch stream).

So we could not rely on old http2 reassembly mode which call
sub-dissector only END_STREAM appeared. We need a reassembly
mode that call subdissector which support streaming mode as
soon as the message in STREAM is available.

Please refer to comments of
reassemble_http2_data_according_to_subdissector() function
of epan/dissectors/packet-http2.c for more detail.

See the linked bug for streaming mode gRPC capture files.

Ping-Bug: 16160
Change-Id: Id9e5337a0e3ca9f8c8119d74d2c1fe4cc263afc3
Reviewed-on: https://code.wireshark.org/review/23988
Petri-Dish: Peter Wu <peter@lekensteyn.nl>
Tested-by: Petri Dish Buildbot
Reviewed-by: Peter Wu <peter@lekensteyn.nl>
This commit is contained in:
Huang Qiangxiong 2017-10-19 16:07:17 +00:00 committed by Peter Wu
parent accd563aaf
commit 0b0bbb8060
4 changed files with 523 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
AUTHORS.src
View File

@ -3720,6 +3720,12 @@ Connor Newton <c.newton[AT]samsung.com> {
Wi-Fi Alliance Neighbor Awareness Networking (NAN) dissector
}
Huang Qiangxiong <qiangxiong.huang[AT]qq.com> {
Protobuf dissector
gRPC dissector
HTTP2 dissector: add streaming mode reassembly and dissecting DATA according to content-type features.
}
and by:
Georgi Guninski <guninski[AT]guninski.com>

2
docbook/release-notes.adoc
View File

@ -62,6 +62,8 @@ since version 3.0.0:
* The “Enabled Protocols” Dialog now only enables, disables and inverts protocols based on the set filter selection. The protocol type (standard or heuristic) may also be choosen as a filter value.
* The “Analyze -> Apply as Filter” and “Analyze -> Prepare a Filter” packet list and detail popup menus now show a preview of their respective filters.
* Protobuf files (*.proto) can now be configured to enable more precise parsing of serialized Protobuf data (such as gRPC).
* HTTP2 support streaming mode reassembly. To use this feature, subdissectors can register itself to "streaming_content_type" dissector table and return pinfo->desegment_len and pinfo->desegment_offset to tell HTTP2 when to start and how many additional bytes requires when next called.
* The message of stream gRPC method can now be parsed with supporting of HTTP2 streaming mode reassembly feature.
// === Removed Features and Support

90
epan/dissectors/packet-grpc.c
View File

@ -11,7 +11,7 @@
/*
* The information used comes from:
* https://grpc.io/docs/guides/wire.html
* https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md
*
* This GRPC dissector must be invoked by HTTP2 dissector.
*
@ -82,6 +82,10 @@ static gboolean grpc_decompress_body = FALSE;
/* detect json automaticlly */
static gboolean grpc_detect_json_automatically = TRUE;
/* tell http2 to use streaming mode reassembly for grpc dissector */
static gboolean grpc_enable_streaming_reassembly_mode = TRUE;
/* whether embed GRPC messages under HTTP2 protocol tree items */
static gboolean grpc_embedded_under_http2 = FALSE;
void proto_register_grpc(void);
void proto_reg_handoff_grpc(void);
@ -282,33 +286,46 @@ dissect_grpc(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree, void *data _U_
gboolean is_request;
guint tvb_len = tvb_reported_length(tvb);
col_set_str(pinfo->cinfo, COL_PROTOCOL, "GRPC");
col_append_str(pinfo->cinfo, COL_INFO, " (GRPC)");
if (!grpc_embedded_under_http2 && proto_tree_get_parent_tree(tree)) {
tree = proto_tree_get_parent_tree(tree);
}
/* http2 had reassembled the http2.data.data, so we need not reassemble again.
reassembled http2.data.data may contain one or more grpc messages. */
while (offset < tvb_len)
{
ti = proto_tree_add_item(tree, proto_grpc, tvb, offset, -1, ENC_NA);
grpc_tree = proto_item_add_subtree(ti, ett_grpc_message);
if (tvb_len - offset < GRPC_MESSAGE_HEAD_LEN) {
/* need at least 5 bytes for dissecting a grpc message */
proto_tree_add_expert_format(grpc_tree, pinfo, &ei_grpc_body_malformed, tvb, offset, -1,
"Malformed message data: only %u bytes left, need at least %u bytes.", tvb_len - offset, GRPC_MESSAGE_HEAD_LEN);
if (pinfo->can_desegment) {
pinfo->desegment_offset = offset;
pinfo->desegment_len = GRPC_MESSAGE_HEAD_LEN - (tvb_len - offset);
return offset;
}
proto_tree_add_expert_format(tree, pinfo, &ei_grpc_body_malformed, tvb, offset, -1,
"GRPC Malformed message data: only %u bytes left, need at least %u bytes.", tvb_len - offset, GRPC_MESSAGE_HEAD_LEN);
break;
}
message_length = tvb_get_ntohl(tvb, offset + 1);
if (tvb_len - offset < GRPC_MESSAGE_HEAD_LEN + message_length) {
/* remaining bytes are not enough for dissecting the message body */
proto_tree_add_expert_format(grpc_tree, pinfo, &ei_grpc_body_malformed, tvb, offset, -1,
"Malformed message data: only %u bytes left, need at least %u bytes.", tvb_len - offset, GRPC_MESSAGE_HEAD_LEN + message_length);
if (pinfo->can_desegment) {
pinfo->desegment_offset = offset;
pinfo->desegment_len = GRPC_MESSAGE_HEAD_LEN + message_length - (tvb_len - offset);
return offset;
}
proto_tree_add_expert_format(tree, pinfo, &ei_grpc_body_malformed, tvb, offset, -1,
"GRPC Malformed message data: only %u bytes left, need at least %u bytes.", tvb_len - offset, GRPC_MESSAGE_HEAD_LEN + message_length);
break;
}
proto_item_set_len(ti, message_length + GRPC_MESSAGE_HEAD_LEN);
/* ready to add information into protocol columns and tree */
if (offset == 0) { /* change columns only when there is at least one grpc message will be parsed */
col_set_str(pinfo->cinfo, COL_PROTOCOL, "GRPC");
col_append_str(pinfo->cinfo, COL_INFO, " (GRPC)");
col_set_fence(pinfo->cinfo, COL_PROTOCOL);
}
ti = proto_tree_add_item(tree, proto_grpc, tvb, offset, message_length + GRPC_MESSAGE_HEAD_LEN, ENC_NA);
grpc_tree = proto_item_add_subtree(ti, ett_grpc_message);
/* http2_path contains: "/" Service-Name "/" {method name} */
http2_path = http2_get_header_value(pinfo, HTTP2_HEADER_PATH, FALSE);
@ -376,7 +393,7 @@ proto_register_grpc(void)
proto_register_field_array(proto_grpc, hf, array_length(hf));
proto_register_subtree_array(ett, array_length(ett));
grpc_module = prefs_register_protocol(proto_grpc, NULL);
grpc_module = prefs_register_protocol(proto_grpc, proto_reg_handoff_grpc);
prefs_register_bool_preference(grpc_module, "detect_json_automaticlly",
"Always check whether the message is JSON regardless of content-type.",
@ -388,6 +405,24 @@ proto_register_grpc(void)
"message according to content-type).",
&grpc_detect_json_automatically);
prefs_register_bool_preference(grpc_module, "streaming_reassembly_mode",
"Turn on streaming reassembly mode. Required for parsing streaming RPCs.",
"If turned on, http2 will reassemble gRPC message as soon as possible. "
"Or else the gRPC message will be reassembled at the end of each HTTP2 "
"STREAM. If your .proto files contains streaming RPCs (declaring RPC "
"operation input/output message type with 'stream' label), you need "
"to keep this option on.",
&grpc_enable_streaming_reassembly_mode);
prefs_register_bool_preference(grpc_module, "embeded_under_http2",
"Embed gRPC messages under HTTP2 protocol tree items.",
"Embed gRPC messages under HTTP2 protocol tree items.",
&grpc_embedded_under_http2);
prefs_register_static_text_preference(grpc_module, "service_definition",
"Please refer to preferences of Protobuf for specifying gRPC Service Definitions (*.proto).",
"Including specifying .proto files search paths, etc.");
expert_grpc = expert_register_protocol(proto_grpc);
expert_register_field_array(expert_grpc, ei, array_length(ei));
@ -405,9 +440,30 @@ proto_register_grpc(void)
void
proto_reg_handoff_grpc(void)
{
dissector_add_string("media_type", "application/grpc", grpc_handle);
dissector_add_string("media_type", "application/grpc+proto", grpc_handle);
dissector_add_string("media_type", "application/grpc+json", grpc_handle);
static gboolean initialized = FALSE;
char *del_table, *add_table;
char *content_types[] = {
"application/grpc",
"application/grpc+proto",
"application/grpc+json",
NULL /* end flag */
};
int i;
add_table = grpc_enable_streaming_reassembly_mode ? "streaming_content_type" : "media_type";
del_table = grpc_enable_streaming_reassembly_mode ? "media_type" : "streaming_content_type";
/* register/deregister grpc_handle to/from tables */
for (i = 0; content_types[i]; i++) {
if (initialized) {
dissector_delete_string(del_table, content_types[i], grpc_handle);
}
dissector_add_string(add_table, content_types[i], grpc_handle);
}
if (!initialized) {
initialized = TRUE;
}
}
/*

455
epan/dissectors/packet-http2.c
View File

@ -48,6 +48,7 @@
#include "packet-tcp.h"
#include "wsutil/pint.h"
#include "wsutil/strtoi.h"
#include "wsutil/str_util.h"
#ifdef HAVE_NGHTTP2
#define http2_header_repr_type_VALUE_STRING_LIST(XXX) \
@ -77,6 +78,30 @@ static gboolean http2_decompress_body = FALSE;
static dissector_table_t media_type_dissector_table;
#endif
/* Some protocols on top of http2 require http2 streams to remain open. For example, the stream
* mode gRPC method (like ETCD watch stream). These kinds of subdissectors need http2 dissector
* to reassemble http2.data.data according to the desegment_offset and desegment_len fields of the
* pinfo returned from them. Subdissectors can register its content-type in this table.
* Note: DESEGMENT_ONE_MORE_SEGMENT is not supported. Subdissector can set pinfo->desegment_len
* to missing nbytes of the head length if entire length of message is undetermined. */
static dissector_table_t streaming_content_type_dissector_table;
/* The type of reassembly mode contains:
* - HTTP2_DATA_REASSEMBLY_MODE_END_STREAM Complete reassembly at the end of stream. (default)
* - HTTP2_DATA_REASSEMBLY_MODE_STREAMING Determined by the desegment_offset and desegment_len fields returned from subdissector.
*/
enum http2_data_reassembly_mode_t {
HTTP2_DATA_REASSEMBLY_MODE_END_STREAM = 0, /* default */
HTTP2_DATA_REASSEMBLY_MODE_STREAMING = 1
};
static enum http2_data_reassembly_mode_t
http2_get_data_reassembly_mode(const gchar* content_type)
{
return dissector_get_string_handle(streaming_content_type_dissector_table, content_type) ?
HTTP2_DATA_REASSEMBLY_MODE_STREAMING : HTTP2_DATA_REASSEMBLY_MODE_END_STREAM;
}
/* Decompressed header field */
typedef struct {
/* one of http2_header_repr_type */
@ -138,10 +163,41 @@ typedef struct {
#ifdef HAVE_NGHTTP2
typedef guint64 http2_frame_num_t;
/* One instance of this structure is created for each pdu that spans across
* multiple DATA segments. (MSP) */
typedef struct _http2_multisegment_pdu_t {
http2_frame_num_t first_frame;
http2_frame_num_t last_frame;
guint start_offset_at_first_frame;
guint end_offset_at_last_frame;
gint length; /* length of this MSP */
/* generated by create_streaming_reassembly_id(), used as reassembly_id */
guint32 streaming_reassembly_id;
/* pointer to previous multisegment_pdu */
struct _http2_multisegment_pdu_t* prev_msp;
} http2_multisegment_pdu_t;
/* struct for per-stream, per-direction streaming DATA frame reassembly */
typedef struct {
/* This tree is indexed by http2 frame num and keeps track of all pdus
* spanning multiple segments in DATAs for this direction of http2 stream. */
wmem_tree_t *multisegment_pdus;
/* This tree is indexed by http2 frame num and keeps track of the frag_offset
* of the first byte of a DATA frame for fragment_add() after first scan. */
wmem_tree_t *http2fn_frag_offset;
/* how many bytes the current uncompleted MSP still need. (only valid for first scan) */
gint prev_deseg_len;
/* the current uncompleted MSP (only valid for first scan) */
http2_multisegment_pdu_t* last_msp;
} http2_streaming_reassembly_info_t;
/* struct for per-stream, per-direction DATA frame reassembly */
typedef struct {
http2_frame_num_t data_initiated_in;
gboolean has_transfer_encoded_body;
/* streaming_reassembly_info only used for STREAMING reassembly mode */
http2_streaming_reassembly_info_t streaming_reassembly_info;
} http2_data_stream_reassembly_info_t;
/* struct for per-stream, per-direction entity body info */
@ -180,6 +236,7 @@ typedef struct {
http2_oneway_stream_info_t oneway_stream_info[2];
gboolean is_stream_http_connect;
guint32 stream_id;
enum http2_data_reassembly_mode_t reassembly_mode;
} http2_stream_info_t;
#endif
/* struct to hold data per HTTP/2 session */
@ -256,6 +313,7 @@ static int hf_http2_weight_real = -1;
static int hf_http2_stream_dependency = -1;
static int hf_http2_excl_dependency = -1;
/* Data */
static int hf_http2_data_segment = -1;
static int hf_http2_data_data = -1;
static int hf_http2_data_padding = -1;
static int hf_http2_body_fragments = -1;
@ -268,6 +326,7 @@ static int hf_http2_body_fragment_error = -1;
static int hf_http2_body_fragment_count = -1;
static int hf_http2_body_reassembled_in = -1;
static int hf_http2_body_reassembled_length = -1;
static int hf_http2_body_reassembled_data = -1;
/* Headers */
static int hf_http2_headers = -1;
static int hf_http2_headers_padding = -1;
@ -392,6 +451,7 @@ static int hf_http2_headers_www_authenticate = -1;
static expert_field ei_http2_header_size = EI_INIT;
static expert_field ei_http2_header_lines = EI_INIT;
static expert_field ei_http2_body_decompression_failed = EI_INIT;
static expert_field ei_http2_reassembly_error = EI_INIT;
static gint ett_http2 = -1;
static gint ett_http2_header = -1;
@ -418,7 +478,7 @@ static const fragment_items http2_body_fragment_items = {
&hf_http2_body_fragment_count,
&hf_http2_body_reassembled_in,
&hf_http2_body_reassembled_length,
NULL,
&hf_http2_body_reassembled_data,
"Body fragments"
};
@ -906,6 +966,7 @@ http2_cleanup_protocol(void) {
static dissector_handle_t http2_handle;
static reassembly_table http2_body_reassembly_table;
static reassembly_table http2_streaming_reassembly_table;
#define FRAME_HEADER_LENGTH 9
#define MAGIC_FRAME_LENGTH 24
@ -1064,6 +1125,7 @@ get_stream_info(http2_session_t *http2_session)
stream_info->oneway_stream_info[0].header_stream_info.stream_header_list = wmem_list_new(wmem_file_scope());
stream_info->oneway_stream_info[1].header_stream_info.stream_header_list = wmem_list_new(wmem_file_scope());
stream_info->stream_id = stream_id;
stream_info->reassembly_mode = HTTP2_DATA_REASSEMBLY_MODE_END_STREAM;
wmem_map_insert(stream_map, GINT_TO_POINTER(stream_id), stream_info);
}
@ -1534,8 +1596,10 @@ populate_http_header_tracking(tvbuff_t *tvb, packet_info *pinfo, http2_session_t
if (strcmp(header_name, HTTP2_HEADER_CONTENT_TYPE) == 0) {
http2_data_stream_body_info_t *body_info = get_data_stream_body_info(pinfo);
if (body_info->content_type == NULL) {
http2_stream_info_t *stream_info = get_stream_info(h2session);
body_info->content_type = get_content_type_only(header_value, header_value_length);
body_info->content_type_parameters = get_content_type_parameters_only(header_value, header_value_length);
stream_info->reassembly_mode = http2_get_data_reassembly_mode(body_info->content_type);
}
}
}
@ -2126,21 +2190,36 @@ get_body_uncompression_info(packet_info *pinfo)
return BODY_UNCOMPRESSION_NONE;
}
static guint32
create_streaming_reassembly_id(void)
{
static guint32 global_streaming_reassembly_id = 0;
return ++global_streaming_reassembly_id;
}
static http2_streaming_reassembly_info_t*
get_streaming_reassembly_info(packet_info *pinfo)
{
return &(get_oneway_stream_info(pinfo, FALSE)->data_stream_reassembly_info.streaming_reassembly_info);
}
/* Try to dissect reassembled http2.data.data according to content_type. */
static void
dissect_body_data(proto_tree *tree, packet_info *pinfo, tvbuff_t *tvb,
const gint start, gint length, const guint encoding)
const gint start, gint length, const guint encoding, gboolean streaming_mode)
{
http2_data_stream_body_info_t *body_info = get_data_stream_body_info(pinfo);
gchar *content_type = body_info->content_type;
http_message_info_t metadata_used_for_media_type_handle = { HTTP_OTHERS, body_info->content_type_parameters, NULL, NULL };
proto_tree_add_item(tree, hf_http2_data_data, tvb, start, length, encoding);
if (!streaming_mode)
proto_tree_add_item(tree, hf_http2_data_data, tvb, start, length, encoding);
if (content_type != NULL) {
/* add it to STREAM level */
proto_tree *ptree = proto_tree_get_parent_tree(tree);
dissector_try_string(media_type_dissector_table, content_type, tvb_new_subset_length(tvb, start, length), pinfo,
dissector_try_string((streaming_mode ? streaming_content_type_dissector_table : media_type_dissector_table),
content_type, tvb_new_subset_length(tvb, start, length), pinfo,
ptree, &metadata_used_for_media_type_handle);
}
}
@ -2177,14 +2256,14 @@ dissect_http2_data_full_body(tvbuff_t *tvb, packet_info *pinfo, proto_tree *http
guint uncompressed_length = tvb_captured_length(uncompressed_tvb);
add_new_data_source(pinfo, uncompressed_tvb, "Uncompressed entity body");
proto_item_append_text(compressed_proto_item, " -> %u bytes", uncompressed_length);
dissect_body_data(compressed_entity_tree, pinfo, uncompressed_tvb, 0, uncompressed_length, ENC_NA);
dissect_body_data(compressed_entity_tree, pinfo, uncompressed_tvb, 0, uncompressed_length, ENC_NA, FALSE);
} else {
proto_tree_add_expert(compressed_entity_tree, pinfo, &ei_http2_body_decompression_failed, tvb, 0, datalen);
dissect_body_data(compressed_entity_tree, pinfo, tvb, 0, datalen, ENC_NA);
dissect_body_data(compressed_entity_tree, pinfo, tvb, 0, datalen, ENC_NA, FALSE);
}
} else {
dissect_body_data(http2_tree, pinfo, tvb, 0, datalen, ENC_NA);
dissect_body_data(http2_tree, pinfo, tvb, 0, datalen, ENC_NA, FALSE);
}
}
@ -2294,9 +2373,318 @@ dissect_http2_data_partial_body(tvbuff_t *tvb, packet_info *pinfo, proto_tree *h
proto_tree_add_item(http2_tree, hf_http2_data_data, tvb, offset, length, ENC_NA);
}
/* Streaming reassembly of HTTP2 DATA will handle many cases. The most complicated one is:
*
* +--------------------------------------------- HTTP2 DATA payload ---------------------------------------------+
* | Part1: end of a multisegment PDU | Part2: one or more non-fragment PDUs | Part3: begin of a multisegment PDU |
* +----------------------------------+--------------------------------------+------------------------------------+
* Note: we use short name 'MSP' for 'Multisegment PDU', and 'NFP' for 'Non-fragment PDU'.
*
* In this case, one HTTP2 DATA payload contains:
* Part1: At the begin of the DATA, there is the last part of a multisegment PDU.
* Part2: The middle part of DATA payload contains one or more non-fragment PDUs.
* Part3: At the tail of the DATA, there is the begin of a new multisegment PDU.
* All of three parts are optional. For example, one DATA could contain only Part1, Part2 or Part3; or contain
* Part1 and Part2 without Part3; or contain Part1 and Part3 without Part2; or contain Part2 and Part3 without
* Part1.
*
* And another case is the entire DATA is one of middle parts of a multisegment PDU. We call it MoMSP. Following
* case shows a multisegment PDU composed of Part3 + MoMSP + MoMSP + MoMSP + Part3:
*
* +-------------------------------- One Multisegment PDU ---------------------------------+
* | |
* +----- HTTP2 DATA1 -----+ +------------- HTTP2 DATA2 -----------+ +- DATA3 -+ +- DATA4 -+ +- HTTP2 DATA5 -+
* | Part1 | Part2 | Part3 | | MoMSP: middle of a multisegment PDU | | MoMSP | | MoMSP | | Part1 | Part3 |
* +-------+-------+-------+ +-------------------------------------+ +---------+ +---------+ +---------------+
* | |
* +---------------------------------------------------------------------------------------+
*
* For a DATA composed of Part1 + Part2 + Part3 will call fragment_add() twice on Part1 and Part3; and call
* process_reassembled_data() once for generating tvb of a MSP to which Part1 belongs; and call subdissector twice on
* reassembled MSP of Part1 and Part2 + Part3. After that finds Part3 is a beginning of a MSP at first scan.
*
* The rules are:
* - If a DATA contains Part1, we will need call fragment_add(), process_reassembled_data() and
* subdissector once on it to end a MSP. (May run twice at first scan, because subdissector may only return the
* head length of message by pinfo->desegment_len. We need run second time for subdissector to determine the length
* of entire message).
* - If a DATA contains Part2, we will need only call subdissector once on it. The subdissector need dissect
* all non-fragment PDUs in it. (no desegment_len should output)
* - If a DATA contains Part3, we will need call subdissector once on Part3 or Parts2+3 (because unknown
* during first scan). The subdissector will output desegment_len (!= 0). Then we will call fragment_add()
* with a new reassembly id on Part3 for starting a new MSP.
* - If a DATA only contains MoMSP (entire DATA is part of a MSP), we will only call fragment_add() once or twice
* (at first scan) on it. The subdissector will not be called.
*
* In this implementation, only multisegment PDUs are recorded in multisegment_pdus tree with first frame
* number (guint64) as key. Each MSP in the tree has a pointer referred to previous MSP, because we may need
* two MSPs to dissect a DATA contains Part1 + Part3 at the same time. The multisegment_pdus tree is built during
* first scan (pinfo->visited == FALSE) with help of prev_deseg_len and last_msp fields of http2_streaming_reassembly_info_t
* for each direction of a HTTP2 STREAM. The prev_deseg_len record how many bytes of next DATAs belong to previous
* PDU during first scan. The last_msp member of http2_streaming_reassembly_info_t is always point to last MSP which
* is created during scan previous or early DATAs. Since subdissector might return only the head length of entire
* message (by pinfo->desegment_len) when there is not enough data to determine the message length, we need reopen
* reassembly fragments for adding more bytes during scanning next DATA. We have to use fragment_add() instead of
* fragment_add_check() or fragment_add_seq_next().
*
* Subdissector behave: The pinfo->desegment_len should contain the estimated number of additional bytes required
* for completing the PDU. and set pinfo->desegment_offset to the offset in the tvbuff at which the dissector will
* continue processing when next called. Next time the subdissector is called, it will be passed a tvbuff composed
* of the end of the data from the previous tvbuff together with desegment_len more bytes. If the dissector cannot
* tell how many more bytes it will need, it should set pinfo->desegment_len to additional bytes required for parsing
* message head. It will then be called again as soon as more data becomes available. Dissectors MUST NOT set the
* pinfo->desegment_len to DESEGMENT_ONE_MORE_SEGMENT.
*/
static void
reassemble_http2_data_according_to_subdissector(tvbuff_t *tvb, packet_info *pinfo, proto_tree *http2_tree,
guint offset, guint8 flags _U_, gint length)
{
guint orig_offset = offset;
gint orig_length = length;
wmem_tree_key_t key[2];
gint datalen = 0;
gint bytes_belong_to_prev_msp = 0; /* bytes belongs to previous MSP */
guint32 num32[2], reassembly_id = 0, frag_offset = 0;
fragment_head *head = NULL;
gboolean need_more = FALSE;
http2_multisegment_pdu_t *cur_msp = NULL, *prev_msp = NULL;
http2_streaming_reassembly_info_t* reassembly_info = get_streaming_reassembly_info(pinfo);
http2_frame_num_t cur_frame_num = get_http2_frame_num(tvb, pinfo);
guint16 save_can_desegment;
int save_desegment_offset;
guint32 save_desegment_len;
proto_tree_add_bytes_format(http2_tree, hf_http2_data_data, tvb, offset,
length, NULL, "DATA payload (%u byte%s)", length, plurality(length, "", "s"));
num32[0] = (guint32) (cur_frame_num >> 32);
num32[1] = (guint32) cur_frame_num;
key[0].length = 2;
key[0].key = num32;
key[1].length = 0;
key[1].key = NULL;
save_can_desegment = pinfo->can_desegment;
save_desegment_offset = pinfo->desegment_offset;
save_desegment_len = pinfo->desegment_len;
/* calculate how many bytes of this DATA belongs to previous MSP (Part1) */
if (!PINFO_FD_VISITED(pinfo)) {
/* This is first scan. Use information of per-direction stream session */
if (reassembly_info->prev_deseg_len > 0) {
/* part or all of current DATA belong to previous MSP */
bytes_belong_to_prev_msp = MIN(reassembly_info->prev_deseg_len, length);
need_more = (reassembly_info->prev_deseg_len > length);
reassembly_info->prev_deseg_len -= bytes_belong_to_prev_msp;
} /* else { beginning of a new PDU (might be a NFP or MSP) } */
if (bytes_belong_to_prev_msp > 0) {
DISSECTOR_ASSERT(reassembly_info->last_msp != NULL);
reassembly_id = reassembly_info->last_msp->streaming_reassembly_id;
frag_offset = reassembly_info->last_msp->length;
if (reassembly_info->http2fn_frag_offset == NULL) {
reassembly_info->http2fn_frag_offset = wmem_tree_new(wmem_file_scope());
}
wmem_tree_insert32_array(reassembly_info->http2fn_frag_offset, key, GUINT_TO_POINTER(frag_offset));
}
} else {
/* not first scan, use information of multisegment_pdus built during first scan */
cur_msp = (http2_multisegment_pdu_t *) wmem_tree_lookup32_array_le(reassembly_info->multisegment_pdus, key);
if (cur_msp) {
if (cur_msp->first_frame == cur_frame_num) {
/* Current DATA contains a beginning of a MSP. (Part3)
* The cur_msp contains information about the beginning MSP.
* If prev_msp is not null, that means this DATA also contains
* the last part of previous MSP. (Part1) */
prev_msp = cur_msp->prev_msp;
} else {
/* Current DATA is not a first frame of a MSP (not include Part3). */
prev_msp = cur_msp;
cur_msp = NULL;
}
}
if (prev_msp && prev_msp->last_frame >= cur_frame_num) {
if (prev_msp->last_frame == cur_frame_num) {
/* this DATA contains part of previous MSP (contains Part1) */
bytes_belong_to_prev_msp = prev_msp->end_offset_at_last_frame - offset;
} else { /* if (prev_msp->last_frame > cur_frame_num) */
/* this DATA all belongs to previous MSP */
bytes_belong_to_prev_msp = length;
need_more = TRUE;
}
reassembly_id = prev_msp->streaming_reassembly_id;
}
frag_offset = GPOINTER_TO_UINT(wmem_tree_lookup32_array(reassembly_info->http2fn_frag_offset, key));
}
/* handling Part1 or MoMSP (entire DATA being middle part of a MSP) */
while (bytes_belong_to_prev_msp > 0) {
tvbuff_t *reassembled_tvb = NULL;
DISSECTOR_ASSERT(reassembly_id > 0);
pinfo->can_desegment = 2; /* this will be decreased one while passing to subdissector */
pinfo->desegment_offset = 0;
pinfo->desegment_len = 0;
head = fragment_add(&http2_streaming_reassembly_table, tvb, offset, pinfo, reassembly_id, NULL,
frag_offset, bytes_belong_to_prev_msp, need_more);
if (head) {
proto_item_set_generated(
proto_tree_add_uint(http2_tree, hf_http2_body_reassembled_in, tvb, offset, bytes_belong_to_prev_msp, head->reassembled_in)
);
if (!need_more) {
reassembled_tvb = process_reassembled_data(tvb, offset, pinfo, "Reassembled HTTP2 streaming body",
head, &http2_body_fragment_items, NULL,
proto_tree_get_parent_tree(http2_tree));
}
}
proto_tree_add_bytes_format(http2_tree, hf_http2_data_segment, tvb, offset,
bytes_belong_to_prev_msp, NULL, "DATA segment data (%u byte%s)", bytes_belong_to_prev_msp,
plurality(bytes_belong_to_prev_msp, "", "s"));
if (reassembled_tvb) {
datalen = tvb_reported_length(reassembled_tvb);
/* normally, this stage will dissect one or more completed pdus */
dissect_body_data(http2_tree, pinfo, reassembled_tvb, 0, datalen, ENC_NA, TRUE);
}
offset += bytes_belong_to_prev_msp;
length -= bytes_belong_to_prev_msp;
DISSECTOR_ASSERT(length >= 0);
if (!PINFO_FD_VISITED(pinfo)) {
reassembly_info->last_msp->length += bytes_belong_to_prev_msp;
frag_offset = reassembly_info->last_msp->length;
}
if (pinfo->desegment_len) {
/* that must only happen during first scan the reassembly_info->prev_deseg_len might be only the
* head length of entire message. */
DISSECTOR_ASSERT(!PINFO_FD_VISITED(pinfo));
DISSECTOR_ASSERT_HINT(pinfo->desegment_len != DESEGMENT_ONE_MORE_SEGMENT, "Subdissector MUST NOT "
"set pinfo->desegment_len to DESEGMENT_ONE_MORE_SEGMENT. Instead, it can set pinfo->desegment_len "
"to the length of head if the length of entire message is not able to be determined.");
DISSECTOR_ASSERT_HINT(pinfo->desegment_offset == 0, "Subdissector MUST NOT set pinfo->desegment_len greater than "
"the length of one message if the length message is undetermined.");
/* reopen fragments for add more segment */
fragment_set_partial_reassembly(&http2_streaming_reassembly_table, pinfo, reassembly_id, NULL);
bytes_belong_to_prev_msp = MIN(pinfo->desegment_len, (guint) length);
reassembly_info->prev_deseg_len = pinfo->desegment_len - bytes_belong_to_prev_msp;
need_more = (reassembly_info->prev_deseg_len > 0);
} else {
if (!PINFO_FD_VISITED(pinfo) && reassembled_tvb) {
/* completed current msp */
reassembly_info->last_msp->last_frame = cur_frame_num;
reassembly_info->last_msp->end_offset_at_last_frame = offset;
reassembly_info->prev_deseg_len = 0;
}
bytes_belong_to_prev_msp = 0; /* break */
}
}
/* to find and handle Part2, and find Part3 at first scan. */
if (length > 0) {
if (!PINFO_FD_VISITED(pinfo)) {
/* It is first scan, dissect remaining bytes to find there is Part2 only, or Part3 only or Part2 + Part3. */
datalen = length;
DISSECTOR_ASSERT(cur_msp == NULL);
} else {
/* Not first scan */
if (cur_msp) {
/* There's Part3. Let's calculate the length of Part2 between Part1 and Part3 */
datalen = cur_msp->start_offset_at_first_frame - offset; /* if result is zero that means no Part2 */
} else {
/* This DATA is not a beginning of MSP. The remaining bytes all belong to Part2 without Part3 */
datalen = length;
}
}
DISSECTOR_ASSERT(datalen >= 0);
/* Dissect the remaining of this DATA. If (datalen == 0) means remaining only have Part3 without Part2. */
if (datalen > 0) {
/* we dissect if it is not dissected before or it is a non-fragment pdu (between two multisegment pdus) */
pinfo->can_desegment = 2;
pinfo->desegment_offset = 0;
pinfo->desegment_len = 0;
dissect_body_data(http2_tree, pinfo, tvb, offset, datalen, ENC_NA, TRUE);
if (pinfo->desegment_len) {
/* only happen during first scan */
DISSECTOR_ASSERT(!PINFO_FD_VISITED(pinfo) && datalen == length);
offset += pinfo->desegment_offset;
length -= pinfo->desegment_offset;
} else {
/* all remaining bytes are consumed by subdissector */
offset += datalen;
length -= datalen;
}
if (!PINFO_FD_VISITED(pinfo)) {
reassembly_info->prev_deseg_len = pinfo->desegment_len;
}
} /* else all remaining bytes (Part3) belong to a new MSP */
DISSECTOR_ASSERT(length >= 0);
}
/* handling Part3 */
if (length > 0) {
col_append_sep_str(pinfo->cinfo, COL_INFO, " ", "[HTTP2 segment of a reassembled PDU]");
if (!PINFO_FD_VISITED(pinfo)) {
/* create a msp for current frame during first scan */
cur_msp = wmem_new0(wmem_file_scope(), http2_multisegment_pdu_t);
cur_msp->first_frame = cur_frame_num;
cur_msp->start_offset_at_first_frame = offset;
cur_msp->length = length;
cur_msp->streaming_reassembly_id = reassembly_id = create_streaming_reassembly_id();
cur_msp->prev_msp = reassembly_info->last_msp;
reassembly_info->last_msp = cur_msp;
if (reassembly_info->multisegment_pdus == NULL) {
reassembly_info->multisegment_pdus = wmem_tree_new(wmem_file_scope());
}
wmem_tree_insert32_array(reassembly_info->multisegment_pdus, key, cur_msp);
} else {
DISSECTOR_ASSERT(cur_msp && cur_msp->start_offset_at_first_frame == offset);
reassembly_id = cur_msp->streaming_reassembly_id;
}
/* add first fragment of the new MSP to reassembly table */
head = fragment_add(&http2_streaming_reassembly_table, tvb, offset, pinfo, reassembly_id,
NULL, 0, length, TRUE);
if (head) {
proto_item_set_generated(
proto_tree_add_uint(http2_tree, hf_http2_body_reassembled_in, tvb, offset, length,
head->reassembled_in)
);
}
proto_tree_add_bytes_format(http2_tree, hf_http2_data_segment, tvb, offset,
length, NULL, "DATA segment data (%u byte%s)", length, plurality(length, "", "s"));
}
if (IS_HTTP2_END_STREAM(flags) && reassembly_info->prev_deseg_len) {
proto_tree_add_expert_format(http2_tree, pinfo, &ei_http2_reassembly_error, tvb, orig_offset, orig_length,
"Reassembling HTTP2 body for subdissector failed, %d bytes are missing before END_STREAM.",
reassembly_info->prev_deseg_len);
}
pinfo->can_desegment = save_can_desegment;
pinfo->desegment_offset = save_desegment_offset;
pinfo->desegment_len = save_desegment_len;
}
static void
dissect_http2_data_body(tvbuff_t *tvb, packet_info *pinfo, proto_tree *http2_tree, guint offset, guint8 flags, gint length)
{
http2_stream_info_t *stream_info = get_stream_info(get_http2_session(pinfo));
if (stream_info->reassembly_mode == HTTP2_DATA_REASSEMBLY_MODE_STREAMING) {
reassemble_http2_data_according_to_subdissector(tvb, pinfo, http2_tree, offset, flags, length);
return;
}
tvbuff_t *data_tvb = reassemble_http2_data_into_full_frame(tvb, pinfo, http2_tree, offset, flags, length);
if (data_tvb != NULL) {
@ -2408,17 +2796,38 @@ dissect_http2_headers(tvbuff_t *tvb, packet_info *pinfo, proto_tree *http2_tree,
gint headlen;
#ifdef HAVE_NGHTTP2
http2_session_t *h2session;
fragment_head *head;
http2_stream_info_t *info;
http2_streaming_reassembly_info_t *reassembly_info;
h2session = get_http2_session(pinfo);
/* Trailing headers coming after a DATA stream should have END_STREAM set. DATA should be complete
* so try to reassemble DATA fragments if that is the case */
if(IS_HTTP2_END_STREAM(flags) ) {
fragment_head *head = fragment_end_seq_next(&http2_body_reassembly_table, pinfo, get_reassembly_id_from_stream(pinfo), NULL);
if(head) {
tvbuff_t *reassembled_data = process_reassembled_data(tvb, 0, pinfo, "Reassembled body", head,
&http2_body_fragment_items, NULL, http2_tree);
dissect_http2_data_full_body(reassembled_data, pinfo, http2_tree);
if(IS_HTTP2_END_STREAM(flags)) {
info = get_stream_info(h2session);
switch (info->reassembly_mode) {
case HTTP2_DATA_REASSEMBLY_MODE_END_STREAM:
head = fragment_end_seq_next(&http2_body_reassembly_table, pinfo, get_reassembly_id_from_stream(pinfo), NULL);
if(head) {
tvbuff_t *reassembled_data = process_reassembled_data(tvb, 0, pinfo, "Reassembled body", head,
&http2_body_fragment_items, NULL, http2_tree);
dissect_http2_data_full_body(reassembled_data, pinfo, http2_tree);
}
break;
case HTTP2_DATA_REASSEMBLY_MODE_STREAMING:
reassembly_info = get_streaming_reassembly_info(pinfo);
if (reassembly_info->prev_deseg_len) {
proto_tree_add_expert_format(http2_tree, pinfo, &ei_http2_reassembly_error, tvb, offset, tvb_reported_length(tvb),
"Reassembling HTTP2 body for subdissector failed, %d bytes are missing before END_STREAM.",
reassembly_info->prev_deseg_len);
}
break;
default:
/* do nothing */
break;
}
}
@ -3128,6 +3537,11 @@ proto_register_http2(void)
},
/* Data */
{ &hf_http2_data_segment,
{ "DATA segment", "http2.data.segment",
FT_BYTES, BASE_NONE, NULL, 0x0,
"A data segment used in reassembly", HFILL}
},
{ &hf_http2_data_data,
{ "Data", "http2.data.data",
FT_BYTES, BASE_NONE, NULL, 0x0,
@ -3189,6 +3603,11 @@ proto_register_http2(void)
FT_UINT32, BASE_DEC, NULL, 0x0,
"Reassembled body in frame number", HFILL }
},
{ &hf_http2_body_reassembled_data,
{ "Reassembled body data", "http2.body.reassembled.data",
FT_BYTES, BASE_NONE, NULL, 0x0,
"Reassembled body data for multisegment PDU spanning across DATAs", HFILL }
},
/* Headers */
{ &hf_http2_headers,
@ -3444,6 +3863,10 @@ proto_register_http2(void)
{ &ei_http2_body_decompression_failed,
{ "http2.body_decompression_failed", PI_UNDECODED, PI_WARN,
"Body decompression failed", EXPFILL }
},
{ &ei_http2_reassembly_error,
{ "http2.reassembly_error", PI_UNDECODED, PI_WARN,
"Reassembly failed", EXPFILL }
}
};
@ -3511,6 +3934,12 @@ proto_register_http2(void)
reassembly_table_register(&http2_body_reassembly_table,
&addresses_ports_reassembly_table_functions);
reassembly_table_register(&http2_streaming_reassembly_table,
&addresses_ports_reassembly_table_functions);
streaming_content_type_dissector_table =
register_dissector_table("streaming_content_type",
"Data Transmitted over HTTP2 in Streaming Mode", proto_http2, FT_STRING, BASE_NONE);
http2_tap = register_tap("http2");
http2_follow_tap = register_tap("http2_follow");