diff --git a/AUTHORS.src b/AUTHORS.src index 3eb9fc74ff..908660e4b2 100644 --- a/AUTHORS.src +++ b/AUTHORS.src @@ -3720,6 +3720,12 @@ Connor Newton { Wi-Fi Alliance Neighbor Awareness Networking (NAN) dissector } +Huang Qiangxiong { + Protobuf dissector + gRPC dissector + HTTP2 dissector: add streaming mode reassembly and dissecting DATA according to content-type features. +} + and by: Georgi Guninski diff --git a/docbook/release-notes.adoc b/docbook/release-notes.adoc index 8f2268f144..b0d9a41ebb 100644 --- a/docbook/release-notes.adoc +++ b/docbook/release-notes.adoc @@ -62,6 +62,8 @@ since version 3.0.0: * The “Enabled Protocols” Dialog now only enables, disables and inverts protocols based on the set filter selection. The protocol type (standard or heuristic) may also be choosen as a filter value. * The “Analyze -> Apply as Filter” and “Analyze -> Prepare a Filter” packet list and detail popup menus now show a preview of their respective filters. * Protobuf files (*.proto) can now be configured to enable more precise parsing of serialized Protobuf data (such as gRPC). +* HTTP2 support streaming mode reassembly. To use this feature, subdissectors can register itself to "streaming_content_type" dissector table and return pinfo->desegment_len and pinfo->desegment_offset to tell HTTP2 when to start and how many additional bytes requires when next called. +* The message of stream gRPC method can now be parsed with supporting of HTTP2 streaming mode reassembly feature. // === Removed Features and Support diff --git a/epan/dissectors/packet-grpc.c b/epan/dissectors/packet-grpc.c index 419a5d2b4e..3835f54741 100644 --- a/epan/dissectors/packet-grpc.c +++ b/epan/dissectors/packet-grpc.c @@ -11,7 +11,7 @@ /* * The information used comes from: -* https://grpc.io/docs/guides/wire.html +* https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md * * This GRPC dissector must be invoked by HTTP2 dissector. * @@ -82,6 +82,10 @@ static gboolean grpc_decompress_body = FALSE; /* detect json automaticlly */ static gboolean grpc_detect_json_automatically = TRUE; +/* tell http2 to use streaming mode reassembly for grpc dissector */ +static gboolean grpc_enable_streaming_reassembly_mode = TRUE; +/* whether embed GRPC messages under HTTP2 protocol tree items */ +static gboolean grpc_embedded_under_http2 = FALSE; void proto_register_grpc(void); void proto_reg_handoff_grpc(void); @@ -282,33 +286,46 @@ dissect_grpc(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree, void *data _U_ gboolean is_request; guint tvb_len = tvb_reported_length(tvb); - col_set_str(pinfo->cinfo, COL_PROTOCOL, "GRPC"); - col_append_str(pinfo->cinfo, COL_INFO, " (GRPC)"); + if (!grpc_embedded_under_http2 && proto_tree_get_parent_tree(tree)) { + tree = proto_tree_get_parent_tree(tree); + } /* http2 had reassembled the http2.data.data, so we need not reassemble again. reassembled http2.data.data may contain one or more grpc messages. */ while (offset < tvb_len) { - ti = proto_tree_add_item(tree, proto_grpc, tvb, offset, -1, ENC_NA); - grpc_tree = proto_item_add_subtree(ti, ett_grpc_message); - if (tvb_len - offset < GRPC_MESSAGE_HEAD_LEN) { /* need at least 5 bytes for dissecting a grpc message */ - - proto_tree_add_expert_format(grpc_tree, pinfo, &ei_grpc_body_malformed, tvb, offset, -1, - "Malformed message data: only %u bytes left, need at least %u bytes.", tvb_len - offset, GRPC_MESSAGE_HEAD_LEN); + if (pinfo->can_desegment) { + pinfo->desegment_offset = offset; + pinfo->desegment_len = GRPC_MESSAGE_HEAD_LEN - (tvb_len - offset); + return offset; + } + proto_tree_add_expert_format(tree, pinfo, &ei_grpc_body_malformed, tvb, offset, -1, + "GRPC Malformed message data: only %u bytes left, need at least %u bytes.", tvb_len - offset, GRPC_MESSAGE_HEAD_LEN); break; } message_length = tvb_get_ntohl(tvb, offset + 1); if (tvb_len - offset < GRPC_MESSAGE_HEAD_LEN + message_length) { /* remaining bytes are not enough for dissecting the message body */ - - proto_tree_add_expert_format(grpc_tree, pinfo, &ei_grpc_body_malformed, tvb, offset, -1, - "Malformed message data: only %u bytes left, need at least %u bytes.", tvb_len - offset, GRPC_MESSAGE_HEAD_LEN + message_length); + if (pinfo->can_desegment) { + pinfo->desegment_offset = offset; + pinfo->desegment_len = GRPC_MESSAGE_HEAD_LEN + message_length - (tvb_len - offset); + return offset; + } + proto_tree_add_expert_format(tree, pinfo, &ei_grpc_body_malformed, tvb, offset, -1, + "GRPC Malformed message data: only %u bytes left, need at least %u bytes.", tvb_len - offset, GRPC_MESSAGE_HEAD_LEN + message_length); break; } - proto_item_set_len(ti, message_length + GRPC_MESSAGE_HEAD_LEN); + /* ready to add information into protocol columns and tree */ + if (offset == 0) { /* change columns only when there is at least one grpc message will be parsed */ + col_set_str(pinfo->cinfo, COL_PROTOCOL, "GRPC"); + col_append_str(pinfo->cinfo, COL_INFO, " (GRPC)"); + col_set_fence(pinfo->cinfo, COL_PROTOCOL); + } + ti = proto_tree_add_item(tree, proto_grpc, tvb, offset, message_length + GRPC_MESSAGE_HEAD_LEN, ENC_NA); + grpc_tree = proto_item_add_subtree(ti, ett_grpc_message); /* http2_path contains: "/" Service-Name "/" {method name} */ http2_path = http2_get_header_value(pinfo, HTTP2_HEADER_PATH, FALSE); @@ -376,7 +393,7 @@ proto_register_grpc(void) proto_register_field_array(proto_grpc, hf, array_length(hf)); proto_register_subtree_array(ett, array_length(ett)); - grpc_module = prefs_register_protocol(proto_grpc, NULL); + grpc_module = prefs_register_protocol(proto_grpc, proto_reg_handoff_grpc); prefs_register_bool_preference(grpc_module, "detect_json_automaticlly", "Always check whether the message is JSON regardless of content-type.", @@ -388,6 +405,24 @@ proto_register_grpc(void) "message according to content-type).", &grpc_detect_json_automatically); + prefs_register_bool_preference(grpc_module, "streaming_reassembly_mode", + "Turn on streaming reassembly mode. Required for parsing streaming RPCs.", + "If turned on, http2 will reassemble gRPC message as soon as possible. " + "Or else the gRPC message will be reassembled at the end of each HTTP2 " + "STREAM. If your .proto files contains streaming RPCs (declaring RPC " + "operation input/output message type with 'stream' label), you need " + "to keep this option on.", + &grpc_enable_streaming_reassembly_mode); + + prefs_register_bool_preference(grpc_module, "embeded_under_http2", + "Embed gRPC messages under HTTP2 protocol tree items.", + "Embed gRPC messages under HTTP2 protocol tree items.", + &grpc_embedded_under_http2); + + prefs_register_static_text_preference(grpc_module, "service_definition", + "Please refer to preferences of Protobuf for specifying gRPC Service Definitions (*.proto).", + "Including specifying .proto files search paths, etc."); + expert_grpc = expert_register_protocol(proto_grpc); expert_register_field_array(expert_grpc, ei, array_length(ei)); @@ -405,9 +440,30 @@ proto_register_grpc(void) void proto_reg_handoff_grpc(void) { - dissector_add_string("media_type", "application/grpc", grpc_handle); - dissector_add_string("media_type", "application/grpc+proto", grpc_handle); - dissector_add_string("media_type", "application/grpc+json", grpc_handle); + static gboolean initialized = FALSE; + char *del_table, *add_table; + char *content_types[] = { + "application/grpc", + "application/grpc+proto", + "application/grpc+json", + NULL /* end flag */ + }; + int i; + + add_table = grpc_enable_streaming_reassembly_mode ? "streaming_content_type" : "media_type"; + del_table = grpc_enable_streaming_reassembly_mode ? "media_type" : "streaming_content_type"; + + /* register/deregister grpc_handle to/from tables */ + for (i = 0; content_types[i]; i++) { + if (initialized) { + dissector_delete_string(del_table, content_types[i], grpc_handle); + } + dissector_add_string(add_table, content_types[i], grpc_handle); + } + + if (!initialized) { + initialized = TRUE; + } } /* diff --git a/epan/dissectors/packet-http2.c b/epan/dissectors/packet-http2.c index 20aec777a8..e5e76c4687 100644 --- a/epan/dissectors/packet-http2.c +++ b/epan/dissectors/packet-http2.c @@ -48,6 +48,7 @@ #include "packet-tcp.h" #include "wsutil/pint.h" #include "wsutil/strtoi.h" +#include "wsutil/str_util.h" #ifdef HAVE_NGHTTP2 #define http2_header_repr_type_VALUE_STRING_LIST(XXX) \ @@ -77,6 +78,30 @@ static gboolean http2_decompress_body = FALSE; static dissector_table_t media_type_dissector_table; #endif +/* Some protocols on top of http2 require http2 streams to remain open. For example, the stream + * mode gRPC method (like ETCD watch stream). These kinds of subdissectors need http2 dissector + * to reassemble http2.data.data according to the desegment_offset and desegment_len fields of the + * pinfo returned from them. Subdissectors can register its content-type in this table. + * Note: DESEGMENT_ONE_MORE_SEGMENT is not supported. Subdissector can set pinfo->desegment_len + * to missing nbytes of the head length if entire length of message is undetermined. */ +static dissector_table_t streaming_content_type_dissector_table; + +/* The type of reassembly mode contains: + * - HTTP2_DATA_REASSEMBLY_MODE_END_STREAM Complete reassembly at the end of stream. (default) + * - HTTP2_DATA_REASSEMBLY_MODE_STREAMING Determined by the desegment_offset and desegment_len fields returned from subdissector. + */ +enum http2_data_reassembly_mode_t { + HTTP2_DATA_REASSEMBLY_MODE_END_STREAM = 0, /* default */ + HTTP2_DATA_REASSEMBLY_MODE_STREAMING = 1 +}; + +static enum http2_data_reassembly_mode_t +http2_get_data_reassembly_mode(const gchar* content_type) +{ + return dissector_get_string_handle(streaming_content_type_dissector_table, content_type) ? + HTTP2_DATA_REASSEMBLY_MODE_STREAMING : HTTP2_DATA_REASSEMBLY_MODE_END_STREAM; +} + /* Decompressed header field */ typedef struct { /* one of http2_header_repr_type */ @@ -138,10 +163,41 @@ typedef struct { #ifdef HAVE_NGHTTP2 typedef guint64 http2_frame_num_t; + +/* One instance of this structure is created for each pdu that spans across + * multiple DATA segments. (MSP) */ +typedef struct _http2_multisegment_pdu_t { + http2_frame_num_t first_frame; + http2_frame_num_t last_frame; + guint start_offset_at_first_frame; + guint end_offset_at_last_frame; + gint length; /* length of this MSP */ + /* generated by create_streaming_reassembly_id(), used as reassembly_id */ + guint32 streaming_reassembly_id; + /* pointer to previous multisegment_pdu */ + struct _http2_multisegment_pdu_t* prev_msp; +} http2_multisegment_pdu_t; + +/* struct for per-stream, per-direction streaming DATA frame reassembly */ +typedef struct { + /* This tree is indexed by http2 frame num and keeps track of all pdus + * spanning multiple segments in DATAs for this direction of http2 stream. */ + wmem_tree_t *multisegment_pdus; + /* This tree is indexed by http2 frame num and keeps track of the frag_offset + * of the first byte of a DATA frame for fragment_add() after first scan. */ + wmem_tree_t *http2fn_frag_offset; + /* how many bytes the current uncompleted MSP still need. (only valid for first scan) */ + gint prev_deseg_len; + /* the current uncompleted MSP (only valid for first scan) */ + http2_multisegment_pdu_t* last_msp; +} http2_streaming_reassembly_info_t; + /* struct for per-stream, per-direction DATA frame reassembly */ typedef struct { http2_frame_num_t data_initiated_in; gboolean has_transfer_encoded_body; + /* streaming_reassembly_info only used for STREAMING reassembly mode */ + http2_streaming_reassembly_info_t streaming_reassembly_info; } http2_data_stream_reassembly_info_t; /* struct for per-stream, per-direction entity body info */ @@ -180,6 +236,7 @@ typedef struct { http2_oneway_stream_info_t oneway_stream_info[2]; gboolean is_stream_http_connect; guint32 stream_id; + enum http2_data_reassembly_mode_t reassembly_mode; } http2_stream_info_t; #endif /* struct to hold data per HTTP/2 session */ @@ -256,6 +313,7 @@ static int hf_http2_weight_real = -1; static int hf_http2_stream_dependency = -1; static int hf_http2_excl_dependency = -1; /* Data */ +static int hf_http2_data_segment = -1; static int hf_http2_data_data = -1; static int hf_http2_data_padding = -1; static int hf_http2_body_fragments = -1; @@ -268,6 +326,7 @@ static int hf_http2_body_fragment_error = -1; static int hf_http2_body_fragment_count = -1; static int hf_http2_body_reassembled_in = -1; static int hf_http2_body_reassembled_length = -1; +static int hf_http2_body_reassembled_data = -1; /* Headers */ static int hf_http2_headers = -1; static int hf_http2_headers_padding = -1; @@ -392,6 +451,7 @@ static int hf_http2_headers_www_authenticate = -1; static expert_field ei_http2_header_size = EI_INIT; static expert_field ei_http2_header_lines = EI_INIT; static expert_field ei_http2_body_decompression_failed = EI_INIT; +static expert_field ei_http2_reassembly_error = EI_INIT; static gint ett_http2 = -1; static gint ett_http2_header = -1; @@ -418,7 +478,7 @@ static const fragment_items http2_body_fragment_items = { &hf_http2_body_fragment_count, &hf_http2_body_reassembled_in, &hf_http2_body_reassembled_length, - NULL, + &hf_http2_body_reassembled_data, "Body fragments" }; @@ -906,6 +966,7 @@ http2_cleanup_protocol(void) { static dissector_handle_t http2_handle; static reassembly_table http2_body_reassembly_table; +static reassembly_table http2_streaming_reassembly_table; #define FRAME_HEADER_LENGTH 9 #define MAGIC_FRAME_LENGTH 24 @@ -1064,6 +1125,7 @@ get_stream_info(http2_session_t *http2_session) stream_info->oneway_stream_info[0].header_stream_info.stream_header_list = wmem_list_new(wmem_file_scope()); stream_info->oneway_stream_info[1].header_stream_info.stream_header_list = wmem_list_new(wmem_file_scope()); stream_info->stream_id = stream_id; + stream_info->reassembly_mode = HTTP2_DATA_REASSEMBLY_MODE_END_STREAM; wmem_map_insert(stream_map, GINT_TO_POINTER(stream_id), stream_info); } @@ -1534,8 +1596,10 @@ populate_http_header_tracking(tvbuff_t *tvb, packet_info *pinfo, http2_session_t if (strcmp(header_name, HTTP2_HEADER_CONTENT_TYPE) == 0) { http2_data_stream_body_info_t *body_info = get_data_stream_body_info(pinfo); if (body_info->content_type == NULL) { + http2_stream_info_t *stream_info = get_stream_info(h2session); body_info->content_type = get_content_type_only(header_value, header_value_length); body_info->content_type_parameters = get_content_type_parameters_only(header_value, header_value_length); + stream_info->reassembly_mode = http2_get_data_reassembly_mode(body_info->content_type); } } } @@ -2126,21 +2190,36 @@ get_body_uncompression_info(packet_info *pinfo) return BODY_UNCOMPRESSION_NONE; } +static guint32 +create_streaming_reassembly_id(void) +{ + static guint32 global_streaming_reassembly_id = 0; + return ++global_streaming_reassembly_id; +} + +static http2_streaming_reassembly_info_t* +get_streaming_reassembly_info(packet_info *pinfo) +{ + return &(get_oneway_stream_info(pinfo, FALSE)->data_stream_reassembly_info.streaming_reassembly_info); +} + /* Try to dissect reassembled http2.data.data according to content_type. */ static void dissect_body_data(proto_tree *tree, packet_info *pinfo, tvbuff_t *tvb, - const gint start, gint length, const guint encoding) + const gint start, gint length, const guint encoding, gboolean streaming_mode) { http2_data_stream_body_info_t *body_info = get_data_stream_body_info(pinfo); gchar *content_type = body_info->content_type; http_message_info_t metadata_used_for_media_type_handle = { HTTP_OTHERS, body_info->content_type_parameters, NULL, NULL }; - proto_tree_add_item(tree, hf_http2_data_data, tvb, start, length, encoding); + if (!streaming_mode) + proto_tree_add_item(tree, hf_http2_data_data, tvb, start, length, encoding); if (content_type != NULL) { /* add it to STREAM level */ proto_tree *ptree = proto_tree_get_parent_tree(tree); - dissector_try_string(media_type_dissector_table, content_type, tvb_new_subset_length(tvb, start, length), pinfo, + dissector_try_string((streaming_mode ? streaming_content_type_dissector_table : media_type_dissector_table), + content_type, tvb_new_subset_length(tvb, start, length), pinfo, ptree, &metadata_used_for_media_type_handle); } } @@ -2177,14 +2256,14 @@ dissect_http2_data_full_body(tvbuff_t *tvb, packet_info *pinfo, proto_tree *http guint uncompressed_length = tvb_captured_length(uncompressed_tvb); add_new_data_source(pinfo, uncompressed_tvb, "Uncompressed entity body"); proto_item_append_text(compressed_proto_item, " -> %u bytes", uncompressed_length); - dissect_body_data(compressed_entity_tree, pinfo, uncompressed_tvb, 0, uncompressed_length, ENC_NA); + dissect_body_data(compressed_entity_tree, pinfo, uncompressed_tvb, 0, uncompressed_length, ENC_NA, FALSE); } else { proto_tree_add_expert(compressed_entity_tree, pinfo, &ei_http2_body_decompression_failed, tvb, 0, datalen); - dissect_body_data(compressed_entity_tree, pinfo, tvb, 0, datalen, ENC_NA); + dissect_body_data(compressed_entity_tree, pinfo, tvb, 0, datalen, ENC_NA, FALSE); } } else { - dissect_body_data(http2_tree, pinfo, tvb, 0, datalen, ENC_NA); + dissect_body_data(http2_tree, pinfo, tvb, 0, datalen, ENC_NA, FALSE); } } @@ -2294,9 +2373,318 @@ dissect_http2_data_partial_body(tvbuff_t *tvb, packet_info *pinfo, proto_tree *h proto_tree_add_item(http2_tree, hf_http2_data_data, tvb, offset, length, ENC_NA); } +/* Streaming reassembly of HTTP2 DATA will handle many cases. The most complicated one is: + * + * +--------------------------------------------- HTTP2 DATA payload ---------------------------------------------+ + * | Part1: end of a multisegment PDU | Part2: one or more non-fragment PDUs | Part3: begin of a multisegment PDU | + * +----------------------------------+--------------------------------------+------------------------------------+ + * Note: we use short name 'MSP' for 'Multisegment PDU', and 'NFP' for 'Non-fragment PDU'. + * + * In this case, one HTTP2 DATA payload contains: + * Part1: At the begin of the DATA, there is the last part of a multisegment PDU. + * Part2: The middle part of DATA payload contains one or more non-fragment PDUs. + * Part3: At the tail of the DATA, there is the begin of a new multisegment PDU. + * All of three parts are optional. For example, one DATA could contain only Part1, Part2 or Part3; or contain + * Part1 and Part2 without Part3; or contain Part1 and Part3 without Part2; or contain Part2 and Part3 without + * Part1. + * + * And another case is the entire DATA is one of middle parts of a multisegment PDU. We call it MoMSP. Following + * case shows a multisegment PDU composed of Part3 + MoMSP + MoMSP + MoMSP + Part3: + * + * +-------------------------------- One Multisegment PDU ---------------------------------+ + * | | + * +----- HTTP2 DATA1 -----+ +------------- HTTP2 DATA2 -----------+ +- DATA3 -+ +- DATA4 -+ +- HTTP2 DATA5 -+ + * | Part1 | Part2 | Part3 | | MoMSP: middle of a multisegment PDU | | MoMSP | | MoMSP | | Part1 | Part3 | + * +-------+-------+-------+ +-------------------------------------+ +---------+ +---------+ +---------------+ + * | | + * +---------------------------------------------------------------------------------------+ + * + * For a DATA composed of Part1 + Part2 + Part3 will call fragment_add() twice on Part1 and Part3; and call + * process_reassembled_data() once for generating tvb of a MSP to which Part1 belongs; and call subdissector twice on + * reassembled MSP of Part1 and Part2 + Part3. After that finds Part3 is a beginning of a MSP at first scan. + * + * The rules are: + * - If a DATA contains Part1, we will need call fragment_add(), process_reassembled_data() and + * subdissector once on it to end a MSP. (May run twice at first scan, because subdissector may only return the + * head length of message by pinfo->desegment_len. We need run second time for subdissector to determine the length + * of entire message). + * - If a DATA contains Part2, we will need only call subdissector once on it. The subdissector need dissect + * all non-fragment PDUs in it. (no desegment_len should output) + * - If a DATA contains Part3, we will need call subdissector once on Part3 or Parts2+3 (because unknown + * during first scan). The subdissector will output desegment_len (!= 0). Then we will call fragment_add() + * with a new reassembly id on Part3 for starting a new MSP. + * - If a DATA only contains MoMSP (entire DATA is part of a MSP), we will only call fragment_add() once or twice + * (at first scan) on it. The subdissector will not be called. + * + * In this implementation, only multisegment PDUs are recorded in multisegment_pdus tree with first frame + * number (guint64) as key. Each MSP in the tree has a pointer referred to previous MSP, because we may need + * two MSPs to dissect a DATA contains Part1 + Part3 at the same time. The multisegment_pdus tree is built during + * first scan (pinfo->visited == FALSE) with help of prev_deseg_len and last_msp fields of http2_streaming_reassembly_info_t + * for each direction of a HTTP2 STREAM. The prev_deseg_len record how many bytes of next DATAs belong to previous + * PDU during first scan. The last_msp member of http2_streaming_reassembly_info_t is always point to last MSP which + * is created during scan previous or early DATAs. Since subdissector might return only the head length of entire + * message (by pinfo->desegment_len) when there is not enough data to determine the message length, we need reopen + * reassembly fragments for adding more bytes during scanning next DATA. We have to use fragment_add() instead of + * fragment_add_check() or fragment_add_seq_next(). + * + * Subdissector behave: The pinfo->desegment_len should contain the estimated number of additional bytes required + * for completing the PDU. and set pinfo->desegment_offset to the offset in the tvbuff at which the dissector will + * continue processing when next called. Next time the subdissector is called, it will be passed a tvbuff composed + * of the end of the data from the previous tvbuff together with desegment_len more bytes. If the dissector cannot + * tell how many more bytes it will need, it should set pinfo->desegment_len to additional bytes required for parsing + * message head. It will then be called again as soon as more data becomes available. Dissectors MUST NOT set the + * pinfo->desegment_len to DESEGMENT_ONE_MORE_SEGMENT. + */ +static void +reassemble_http2_data_according_to_subdissector(tvbuff_t *tvb, packet_info *pinfo, proto_tree *http2_tree, + guint offset, guint8 flags _U_, gint length) +{ + guint orig_offset = offset; + gint orig_length = length; + wmem_tree_key_t key[2]; + gint datalen = 0; + gint bytes_belong_to_prev_msp = 0; /* bytes belongs to previous MSP */ + guint32 num32[2], reassembly_id = 0, frag_offset = 0; + fragment_head *head = NULL; + gboolean need_more = FALSE; + http2_multisegment_pdu_t *cur_msp = NULL, *prev_msp = NULL; + http2_streaming_reassembly_info_t* reassembly_info = get_streaming_reassembly_info(pinfo); + http2_frame_num_t cur_frame_num = get_http2_frame_num(tvb, pinfo); + guint16 save_can_desegment; + int save_desegment_offset; + guint32 save_desegment_len; + + proto_tree_add_bytes_format(http2_tree, hf_http2_data_data, tvb, offset, + length, NULL, "DATA payload (%u byte%s)", length, plurality(length, "", "s")); + + num32[0] = (guint32) (cur_frame_num >> 32); + num32[1] = (guint32) cur_frame_num; + key[0].length = 2; + key[0].key = num32; + key[1].length = 0; + key[1].key = NULL; + + save_can_desegment = pinfo->can_desegment; + save_desegment_offset = pinfo->desegment_offset; + save_desegment_len = pinfo->desegment_len; + + /* calculate how many bytes of this DATA belongs to previous MSP (Part1) */ + if (!PINFO_FD_VISITED(pinfo)) { + /* This is first scan. Use information of per-direction stream session */ + if (reassembly_info->prev_deseg_len > 0) { + /* part or all of current DATA belong to previous MSP */ + bytes_belong_to_prev_msp = MIN(reassembly_info->prev_deseg_len, length); + need_more = (reassembly_info->prev_deseg_len > length); + reassembly_info->prev_deseg_len -= bytes_belong_to_prev_msp; + } /* else { beginning of a new PDU (might be a NFP or MSP) } */ + + if (bytes_belong_to_prev_msp > 0) { + DISSECTOR_ASSERT(reassembly_info->last_msp != NULL); + reassembly_id = reassembly_info->last_msp->streaming_reassembly_id; + frag_offset = reassembly_info->last_msp->length; + if (reassembly_info->http2fn_frag_offset == NULL) { + reassembly_info->http2fn_frag_offset = wmem_tree_new(wmem_file_scope()); + } + wmem_tree_insert32_array(reassembly_info->http2fn_frag_offset, key, GUINT_TO_POINTER(frag_offset)); + } + } else { + /* not first scan, use information of multisegment_pdus built during first scan */ + cur_msp = (http2_multisegment_pdu_t *) wmem_tree_lookup32_array_le(reassembly_info->multisegment_pdus, key); + if (cur_msp) { + if (cur_msp->first_frame == cur_frame_num) { + /* Current DATA contains a beginning of a MSP. (Part3) + * The cur_msp contains information about the beginning MSP. + * If prev_msp is not null, that means this DATA also contains + * the last part of previous MSP. (Part1) */ + prev_msp = cur_msp->prev_msp; + } else { + /* Current DATA is not a first frame of a MSP (not include Part3). */ + prev_msp = cur_msp; + cur_msp = NULL; + } + } + + if (prev_msp && prev_msp->last_frame >= cur_frame_num) { + if (prev_msp->last_frame == cur_frame_num) { + /* this DATA contains part of previous MSP (contains Part1) */ + bytes_belong_to_prev_msp = prev_msp->end_offset_at_last_frame - offset; + } else { /* if (prev_msp->last_frame > cur_frame_num) */ + /* this DATA all belongs to previous MSP */ + bytes_belong_to_prev_msp = length; + need_more = TRUE; + } + reassembly_id = prev_msp->streaming_reassembly_id; + } + frag_offset = GPOINTER_TO_UINT(wmem_tree_lookup32_array(reassembly_info->http2fn_frag_offset, key)); + } + + /* handling Part1 or MoMSP (entire DATA being middle part of a MSP) */ + while (bytes_belong_to_prev_msp > 0) { + tvbuff_t *reassembled_tvb = NULL; + DISSECTOR_ASSERT(reassembly_id > 0); + pinfo->can_desegment = 2; /* this will be decreased one while passing to subdissector */ + pinfo->desegment_offset = 0; + pinfo->desegment_len = 0; + + head = fragment_add(&http2_streaming_reassembly_table, tvb, offset, pinfo, reassembly_id, NULL, + frag_offset, bytes_belong_to_prev_msp, need_more); + + if (head) { + proto_item_set_generated( + proto_tree_add_uint(http2_tree, hf_http2_body_reassembled_in, tvb, offset, bytes_belong_to_prev_msp, head->reassembled_in) + ); + + if (!need_more) { + reassembled_tvb = process_reassembled_data(tvb, offset, pinfo, "Reassembled HTTP2 streaming body", + head, &http2_body_fragment_items, NULL, + proto_tree_get_parent_tree(http2_tree)); + } + } + + proto_tree_add_bytes_format(http2_tree, hf_http2_data_segment, tvb, offset, + bytes_belong_to_prev_msp, NULL, "DATA segment data (%u byte%s)", bytes_belong_to_prev_msp, + plurality(bytes_belong_to_prev_msp, "", "s")); + + if (reassembled_tvb) { + datalen = tvb_reported_length(reassembled_tvb); + + /* normally, this stage will dissect one or more completed pdus */ + dissect_body_data(http2_tree, pinfo, reassembled_tvb, 0, datalen, ENC_NA, TRUE); + } + + offset += bytes_belong_to_prev_msp; + length -= bytes_belong_to_prev_msp; + DISSECTOR_ASSERT(length >= 0); + if (!PINFO_FD_VISITED(pinfo)) { + reassembly_info->last_msp->length += bytes_belong_to_prev_msp; + frag_offset = reassembly_info->last_msp->length; + } + + if (pinfo->desegment_len) { + /* that must only happen during first scan the reassembly_info->prev_deseg_len might be only the + * head length of entire message. */ + DISSECTOR_ASSERT(!PINFO_FD_VISITED(pinfo)); + DISSECTOR_ASSERT_HINT(pinfo->desegment_len != DESEGMENT_ONE_MORE_SEGMENT, "Subdissector MUST NOT " + "set pinfo->desegment_len to DESEGMENT_ONE_MORE_SEGMENT. Instead, it can set pinfo->desegment_len " + "to the length of head if the length of entire message is not able to be determined."); + DISSECTOR_ASSERT_HINT(pinfo->desegment_offset == 0, "Subdissector MUST NOT set pinfo->desegment_len greater than " + "the length of one message if the length message is undetermined."); + + /* reopen fragments for add more segment */ + fragment_set_partial_reassembly(&http2_streaming_reassembly_table, pinfo, reassembly_id, NULL); + + bytes_belong_to_prev_msp = MIN(pinfo->desegment_len, (guint) length); + reassembly_info->prev_deseg_len = pinfo->desegment_len - bytes_belong_to_prev_msp; + need_more = (reassembly_info->prev_deseg_len > 0); + } else { + if (!PINFO_FD_VISITED(pinfo) && reassembled_tvb) { + /* completed current msp */ + reassembly_info->last_msp->last_frame = cur_frame_num; + reassembly_info->last_msp->end_offset_at_last_frame = offset; + reassembly_info->prev_deseg_len = 0; + } + bytes_belong_to_prev_msp = 0; /* break */ + } + } + + /* to find and handle Part2, and find Part3 at first scan. */ + if (length > 0) { + if (!PINFO_FD_VISITED(pinfo)) { + /* It is first scan, dissect remaining bytes to find there is Part2 only, or Part3 only or Part2 + Part3. */ + datalen = length; + DISSECTOR_ASSERT(cur_msp == NULL); + } else { + /* Not first scan */ + if (cur_msp) { + /* There's Part3. Let's calculate the length of Part2 between Part1 and Part3 */ + datalen = cur_msp->start_offset_at_first_frame - offset; /* if result is zero that means no Part2 */ + } else { + /* This DATA is not a beginning of MSP. The remaining bytes all belong to Part2 without Part3 */ + datalen = length; + } + } + DISSECTOR_ASSERT(datalen >= 0); + + /* Dissect the remaining of this DATA. If (datalen == 0) means remaining only have Part3 without Part2. */ + if (datalen > 0) { + /* we dissect if it is not dissected before or it is a non-fragment pdu (between two multisegment pdus) */ + pinfo->can_desegment = 2; + pinfo->desegment_offset = 0; + pinfo->desegment_len = 0; + + dissect_body_data(http2_tree, pinfo, tvb, offset, datalen, ENC_NA, TRUE); + + if (pinfo->desegment_len) { + /* only happen during first scan */ + DISSECTOR_ASSERT(!PINFO_FD_VISITED(pinfo) && datalen == length); + offset += pinfo->desegment_offset; + length -= pinfo->desegment_offset; + } else { + /* all remaining bytes are consumed by subdissector */ + offset += datalen; + length -= datalen; + } + if (!PINFO_FD_VISITED(pinfo)) { + reassembly_info->prev_deseg_len = pinfo->desegment_len; + } + } /* else all remaining bytes (Part3) belong to a new MSP */ + DISSECTOR_ASSERT(length >= 0); + } + + /* handling Part3 */ + if (length > 0) { + col_append_sep_str(pinfo->cinfo, COL_INFO, " ", "[HTTP2 segment of a reassembled PDU]"); + if (!PINFO_FD_VISITED(pinfo)) { + /* create a msp for current frame during first scan */ + cur_msp = wmem_new0(wmem_file_scope(), http2_multisegment_pdu_t); + cur_msp->first_frame = cur_frame_num; + cur_msp->start_offset_at_first_frame = offset; + cur_msp->length = length; + cur_msp->streaming_reassembly_id = reassembly_id = create_streaming_reassembly_id(); + cur_msp->prev_msp = reassembly_info->last_msp; + reassembly_info->last_msp = cur_msp; + if (reassembly_info->multisegment_pdus == NULL) { + reassembly_info->multisegment_pdus = wmem_tree_new(wmem_file_scope()); + } + wmem_tree_insert32_array(reassembly_info->multisegment_pdus, key, cur_msp); + } else { + DISSECTOR_ASSERT(cur_msp && cur_msp->start_offset_at_first_frame == offset); + reassembly_id = cur_msp->streaming_reassembly_id; + } + /* add first fragment of the new MSP to reassembly table */ + head = fragment_add(&http2_streaming_reassembly_table, tvb, offset, pinfo, reassembly_id, + NULL, 0, length, TRUE); + + if (head) { + proto_item_set_generated( + proto_tree_add_uint(http2_tree, hf_http2_body_reassembled_in, tvb, offset, length, + head->reassembled_in) + ); + } + proto_tree_add_bytes_format(http2_tree, hf_http2_data_segment, tvb, offset, + length, NULL, "DATA segment data (%u byte%s)", length, plurality(length, "", "s")); + } + + if (IS_HTTP2_END_STREAM(flags) && reassembly_info->prev_deseg_len) { + proto_tree_add_expert_format(http2_tree, pinfo, &ei_http2_reassembly_error, tvb, orig_offset, orig_length, + "Reassembling HTTP2 body for subdissector failed, %d bytes are missing before END_STREAM.", + reassembly_info->prev_deseg_len); + } + + pinfo->can_desegment = save_can_desegment; + pinfo->desegment_offset = save_desegment_offset; + pinfo->desegment_len = save_desegment_len; +} + static void dissect_http2_data_body(tvbuff_t *tvb, packet_info *pinfo, proto_tree *http2_tree, guint offset, guint8 flags, gint length) { + http2_stream_info_t *stream_info = get_stream_info(get_http2_session(pinfo)); + if (stream_info->reassembly_mode == HTTP2_DATA_REASSEMBLY_MODE_STREAMING) { + reassemble_http2_data_according_to_subdissector(tvb, pinfo, http2_tree, offset, flags, length); + return; + } + tvbuff_t *data_tvb = reassemble_http2_data_into_full_frame(tvb, pinfo, http2_tree, offset, flags, length); if (data_tvb != NULL) { @@ -2408,17 +2796,38 @@ dissect_http2_headers(tvbuff_t *tvb, packet_info *pinfo, proto_tree *http2_tree, gint headlen; #ifdef HAVE_NGHTTP2 http2_session_t *h2session; + fragment_head *head; + http2_stream_info_t *info; + http2_streaming_reassembly_info_t *reassembly_info; h2session = get_http2_session(pinfo); /* Trailing headers coming after a DATA stream should have END_STREAM set. DATA should be complete * so try to reassemble DATA fragments if that is the case */ - if(IS_HTTP2_END_STREAM(flags) ) { - fragment_head *head = fragment_end_seq_next(&http2_body_reassembly_table, pinfo, get_reassembly_id_from_stream(pinfo), NULL); - if(head) { - tvbuff_t *reassembled_data = process_reassembled_data(tvb, 0, pinfo, "Reassembled body", head, - &http2_body_fragment_items, NULL, http2_tree); - dissect_http2_data_full_body(reassembled_data, pinfo, http2_tree); + if(IS_HTTP2_END_STREAM(flags)) { + info = get_stream_info(h2session); + switch (info->reassembly_mode) { + case HTTP2_DATA_REASSEMBLY_MODE_END_STREAM: + head = fragment_end_seq_next(&http2_body_reassembly_table, pinfo, get_reassembly_id_from_stream(pinfo), NULL); + if(head) { + tvbuff_t *reassembled_data = process_reassembled_data(tvb, 0, pinfo, "Reassembled body", head, + &http2_body_fragment_items, NULL, http2_tree); + dissect_http2_data_full_body(reassembled_data, pinfo, http2_tree); + } + break; + + case HTTP2_DATA_REASSEMBLY_MODE_STREAMING: + reassembly_info = get_streaming_reassembly_info(pinfo); + if (reassembly_info->prev_deseg_len) { + proto_tree_add_expert_format(http2_tree, pinfo, &ei_http2_reassembly_error, tvb, offset, tvb_reported_length(tvb), + "Reassembling HTTP2 body for subdissector failed, %d bytes are missing before END_STREAM.", + reassembly_info->prev_deseg_len); + } + break; + + default: + /* do nothing */ + break; } } @@ -3128,6 +3537,11 @@ proto_register_http2(void) }, /* Data */ + { &hf_http2_data_segment, + { "DATA segment", "http2.data.segment", + FT_BYTES, BASE_NONE, NULL, 0x0, + "A data segment used in reassembly", HFILL} + }, { &hf_http2_data_data, { "Data", "http2.data.data", FT_BYTES, BASE_NONE, NULL, 0x0, @@ -3189,6 +3603,11 @@ proto_register_http2(void) FT_UINT32, BASE_DEC, NULL, 0x0, "Reassembled body in frame number", HFILL } }, + { &hf_http2_body_reassembled_data, + { "Reassembled body data", "http2.body.reassembled.data", + FT_BYTES, BASE_NONE, NULL, 0x0, + "Reassembled body data for multisegment PDU spanning across DATAs", HFILL } + }, /* Headers */ { &hf_http2_headers, @@ -3444,6 +3863,10 @@ proto_register_http2(void) { &ei_http2_body_decompression_failed, { "http2.body_decompression_failed", PI_UNDECODED, PI_WARN, "Body decompression failed", EXPFILL } + }, + { &ei_http2_reassembly_error, + { "http2.reassembly_error", PI_UNDECODED, PI_WARN, + "Reassembly failed", EXPFILL } } }; @@ -3511,6 +3934,12 @@ proto_register_http2(void) reassembly_table_register(&http2_body_reassembly_table, &addresses_ports_reassembly_table_functions); + reassembly_table_register(&http2_streaming_reassembly_table, + &addresses_ports_reassembly_table_functions); + + streaming_content_type_dissector_table = + register_dissector_table("streaming_content_type", + "Data Transmitted over HTTP2 in Streaming Mode", proto_http2, FT_STRING, BASE_NONE); http2_tap = register_tap("http2"); http2_follow_tap = register_tap("http2_follow");