Protobuf/gRPC: add test cases for Protobuf and gRPC

Add case_dissect_protobuf and case_dissect_grpc in test/suite_dissection.py.
Add *.proto into the sub directories of test/protobuf_lang_files/.
Run command like 'pytest --program-path .\run\Debug\ -k "grpc or protobuf"'
in build directory (in windows) to test these cases only.
This commit is contained in:
Huang Qiangxiong 2020-11-22 21:55:47 +08:00 committed by huangqiangxiong
parent 6a0feb8d0a
commit dcf6bdda84
17 changed files with 382 additions and 0 deletions

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -195,6 +195,7 @@ def dirs():
config_dir=os.path.join(this_dir, 'config'),
key_dir=os.path.join(this_dir, 'keys'),
lua_dir=os.path.join(this_dir, 'lua'),
protobuf_lang_files_dir=os.path.join(this_dir, 'protobuf_lang_files'),
tools_dir=os.path.join(this_dir, '..', 'tools'),
)

View File

@ -0,0 +1,68 @@
do
local protobuf_dissector = Dissector.get("protobuf")
-- Create protobuf dissector based on UDP or TCP.
-- The UDP dissector will take the whole tvb as a message.
-- The TCP dissector will parse tvb as format:
-- [4bytes length][a message][4bytes length][a message]...
-- @param name The name of the new dissector.
-- @param desc The description of the new dissector.
-- @param for_udp Register the new dissector to UDP table.(Enable 'Decode as')
-- @param for_tcp Register the new dissector to TCP table.(Enable 'Decode as')
-- @param msgtype Message type. This must be the root message defined in your .proto file.
local function create_protobuf_dissector(name, desc, for_udp, for_tcp, msgtype)
local proto = Proto(name, desc)
local f_length = ProtoField.uint32(name .. ".length", "Length", base.DEC)
proto.fields = { f_length }
proto.dissector = function(tvb, pinfo, tree)
local subtree = tree:add(proto, tvb())
if for_udp and pinfo.port_type == 3 then -- UDP
if msgtype ~= nil then
pinfo.private["pb_msg_type"] = "message," .. msgtype
end
pcall(Dissector.call, protobuf_dissector, tvb, pinfo, subtree)
elseif for_tcp and pinfo.port_type == 2 then -- TCP
local offset = 0
local remaining_len = tvb:len()
while remaining_len > 0 do
if remaining_len < 4 then -- head not enough
pinfo.desegment_offset = offset
pinfo.desegment_len = DESEGMENT_ONE_MORE_SEGMENT
return -1
end
local data_len = tvb(offset, 4):uint()
if remaining_len - 4 < data_len then -- data not enough
pinfo.desegment_offset = offset
pinfo.desegment_len = data_len - (remaining_len - 4)
return -1
end
subtree:add(f_length, tvb(offset, 4))
if msgtype ~= nil then
pinfo.private["pb_msg_type"] = "message," .. msgtype
end
pcall(Dissector.call, protobuf_dissector,
tvb(offset + 4, data_len):tvb(), pinfo, subtree)
offset = offset + 4 + data_len
remaining_len = remaining_len - 4 - data_len
end
end
pinfo.columns.protocol:set(name)
end
if for_udp then DissectorTable.get("udp.port"):add(0, proto) end
if for_tcp then DissectorTable.get("tcp.port"):add(0, proto) end
return proto
end
-- default pure protobuf udp and tcp dissector without message type
create_protobuf_dissector("protobuf_udp", "Protobuf UDP")
create_protobuf_dissector("protobuf_tcp", "Protobuf TCP")
-- add more protobuf dissectors with message types
create_protobuf_dissector("AddrBook", "Tutorial AddressBook",
true, true, "tutorial.AddressBook")
end

View File

@ -0,0 +1,6 @@
-- Test protobuf_field dissector table
do
local protobuf_field_table = DissectorTable.get("protobuf_field")
local png_dissector = Dissector.get("png")
protobuf_field_table:add("tutorial.Person.portrait_image", png_dissector)
end

View File

@ -0,0 +1,30 @@
// This file comes from the official Protobuf example with a little modification.
syntax = "proto3";
package tutorial;
import "google/protobuf/timestamp.proto";
message Person {
string name = 1;
int32 id = 2; // Unique ID number for this person.
string email = 3;
enum PhoneType {
MOBILE = 0;
HOME = 1;
WORK = 2;
}
message PhoneNumber {
string number = 1;
PhoneType type = 2;
}
repeated PhoneNumber phone = 4;
google.protobuf.Timestamp last_updated = 5;
bytes portrait_image = 6;
}
message AddressBook {
repeated Person people = 1;
}

View File

@ -0,0 +1,15 @@
// A gRPC service that searches for persons based on certain attributes.
syntax = "proto3";
package tutorial;
import "addressbook.proto";
message PersonSearchRequest {
repeated string name = 1;
repeated int32 id = 2;
repeated string phoneNumber = 3;
}
service PersonSearchService {
rpc Search (PersonSearchRequest) returns (stream Person) {}
}

View File

@ -0,0 +1,59 @@
// Test default values of Protobuf fields
syntax = "proto2";
package wireshark.protobuf.test;
message TestDefaultValueMessage {
enum EnumFoo {
ENUM_FOO_V_FIRST = 1;
ENUM_FOO_V_SECOND = 0x2;
ENUM_FOO_V_THIRD = 3;
ENUM_FOO_V_FOURTH = - 4;
}
// The format of field name is:
// <type> "With" ( "Value" | "DefaultValue" | "NoValue" ) [ "_" <correct_value_in_wireshark> ]
// The "DefaultValue" fields should be wrapped with generated mark ("[" and "]") of Wireshark tree item.
// The "NoValue" fields should not appear in Wireshark.
// The default value is overridden to 8 at running time.
required int32 int32WithValue_8 = 1 [ default = 2 ];
// The default value is overridden to ENUM_FOO_V_THIRD at running time.
optional EnumFoo enumFooWithValue_Third = 2 [ default = ENUM_FOO_V_SECOND ];
// default values of bool
optional bool boolWithDefaultValue_False = 11;
optional bool boolWithDefaultValue_True = 12 [ default = true ];
// default values of enum
optional EnumFoo enumFooWithDefaultValue_First = 21;
optional EnumFoo enumFooWithDefaultValue_Second = 22 [ default = ENUM_FOO_V_SECOND ];
optional EnumFoo enumFooWithDefaultValue_Fouth = 23 [ default = ENUM_FOO_V_FOURTH ];
// default values of integer number
optional int32 int32WithDefaultValue_0 = 31;
optional int64 int64WithDefaultValue_Negative1152921504606846976 = 32 [ default = - 1152921504606846976 ];
optional uint32 uint32WithDefaultValue_11 = 33 [ default = 11 ];
optional uint64 uint64WithDefaultValue_1152921504606846976 = 34 [ default = 1152921504606846976 ]; // equals to 2^60
optional sint32 sint32WithDefaultValue_Negative12 = 35 [ default = -12 ];
optional sint64 sint64WithDefaultValue_0 = 36; // default value is zero
optional fixed64 fixed64WithDefaultValue_1152921504606846976 = 37 [ default = 1152921504606846976 ];
optional sfixed32 sfixed32WithDefaultValue_Negative31 = 38 [ default = -0X1f ]; // -21
// default values of float and double
optional float floatWithDefaultValue_0point23 = 41 [ default = 0.23 ];
optional double doubleWithDefaultValue_Negative0point12345678 = 42 [ default = -0.12345678 ];
// default values of string and bytes
optional string stringWithNoValue = 51; // default value must not appear because not declared
optional string stringWithDefaultValue_SymbolPi = 52 [ default = "The symbol \'\xF0\x9D\x9B\x91\' is mathematical bold small Pi."];
optional bytes bytesWithNoValue = 53; // default value must not appear because not declared
// '\'nnn is octal value of a byte, '\x'nn is hex value of a byte
optional bytes bytesWithDefaultValue_1F2F890D0A00004B = 54 [ default = "\x1F\x2F\211\r\n\000\x0\x4B" ];
// others
repeated int32 repeatedFieldWithNoValue = 81; // should not appear
required int32 missingRequiredField = 82; // for testing required field. (comment this line if you regenerated stub code)
// test taking keyword as identification feature
optional int32 message = 83;
optional int32 optional = 84;
}

View File

@ -0,0 +1,24 @@
syntax="proto3";
package a.b;
message a {
string param1 = 1;
message b {
string param2 = 2;
message c {
string param3 = 3;
}
}
}
message msg {
a.b.c param4 = 4; /* the full name of the type is a.b.a.b.c */
.a.b.c param5 = 5; /* the full name of the type is a.b.c */
}
message c {
string param6 = 6;
}

View File

@ -0,0 +1,17 @@
syntax="proto3";
package test.map;
message Foo {
int32 param1 = 1;
}
message MapMaster {
oneof Abc {
string param2 = 2;
string param3 = 3;
}
map<string, int64> param4 = 4;
map<sint32, Foo> param5 = 5;
}

View File

@ -0,0 +1,20 @@
// This file is from https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto
// To reduce the file size, some comments have been removed.
syntax = "proto3";
package google.protobuf;
message Timestamp {
// Represents seconds of UTC time since Unix epoch
// 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to
// 9999-12-31T23:59:59Z inclusive.
int64 seconds = 1;
// Non-negative fractions of a second at nanosecond resolution. Negative
// second values with fractions must still have non-negative nanos values
// that count forward in time. Must be from 0 to 999,999,999
// inclusive.
int32 nanos = 2;
}

View File

@ -14,6 +14,38 @@ import unittest
import fixtures
import sys
@fixtures.mark_usefixtures('test_env')
@fixtures.uses_fixtures
class case_dissect_grpc(subprocesstest.SubprocessTestCase):
def test_grpc_with_json(self, cmd_tshark, features, dirs, capture_file):
'''gRPC with JSON payload'''
if not features.have_nghttp2:
self.skipTest('Requires nghttp2.')
self.assertRun((cmd_tshark,
'-r', capture_file('grpc_person_search_json_with_image.pcapng.gz'),
'-d', 'tcp.port==50052,http2',
'-Y', 'grpc.message_length == 208 && json.value.string == "87561234"',
))
self.assertTrue(self.grepOutput('GRPC/JSON'))
def test_grpc_with_protobuf(self, cmd_tshark, features, dirs, capture_file):
'''gRPC with Protobuf payload'''
if not features.have_nghttp2:
self.skipTest('Requires nghttp2.')
well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
self.assertRun((cmd_tshark,
'-r', capture_file('grpc_person_search_protobuf_with_image.pcapng.gz'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
'-d', 'tcp.port==50051,http2',
'-Y', 'protobuf.message.name == "tutorial.PersonSearchRequest"'
' || (grpc.message_length == 66 && protobuf.field.value.string == "Jason"'
' && protobuf.field.value.int64 == 1602601886)',
))
self.assertTrue(self.grepOutput('tutorial.PersonSearchService/Search')) # grpc request
self.assertTrue(self.grepOutput('tutorial.Person')) # grpc response
@fixtures.mark_usefixtures('test_env')
@fixtures.uses_fixtures
class case_dissect_http(subprocesstest.SubprocessTestCase):
@ -82,6 +114,116 @@ class case_dissect_http2(subprocesstest.SubprocessTestCase):
self.assertFalse(self.grepOutput('00000000 00 00 12 04 00 00 00 00'))
self.assertTrue(self.grepOutput('00000000 00 00 2c 01 05 00 00 00'))
@fixtures.mark_usefixtures('test_env')
@fixtures.uses_fixtures
class case_dissect_protobuf(subprocesstest.SubprocessTestCase):
def test_protobuf_udp_message_mapping(self, cmd_tshark, features, dirs, capture_file):
'''Test Protobuf UDP Message Mapping and parsing google.protobuf.Timestamp features'''
well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
self.assertRun((cmd_tshark,
'-r', capture_file('protobuf_udp_addressbook_with_image_ts.pcapng'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
'-o', 'uat:protobuf_udp_message_types: "8127","tutorial.AddressBook"',
'-o', 'protobuf.preload_protos: TRUE',
'-o', 'protobuf.pbf_as_hf: TRUE',
'-Y', 'pbf.tutorial.Person.name == "Jason"'
' && pbf.tutorial.Person.last_updated > "2020-10-15"'
' && pbf.tutorial.Person.last_updated < "2020-10-19"',
))
self.assertTrue(self.grepOutput('tutorial.AddressBook'))
def test_protobuf_message_type_leading_with_dot(self, cmd_tshark, features, dirs, capture_file):
'''Test Protobuf Message type is leading with dot'''
well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
self.assertRun((cmd_tshark,
'-r', capture_file('protobuf_test_leading_dot.pcapng'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
'-o', 'uat:protobuf_udp_message_types: "8123","a.b.msg"',
'-o', 'protobuf.preload_protos: TRUE',
'-o', 'protobuf.pbf_as_hf: TRUE',
'-Y', 'pbf.a.b.a.b.c.param3 contains "in a.b.a.b.c" && pbf.a.b.c.param6 contains "in a.b.c"',
))
self.assertTrue(self.grepOutput('PB[(]a.b.msg[)]'))
def test_protobuf_map_and_oneof_types(self, cmd_tshark, features, dirs, capture_file):
'''Test Protobuf map and oneof types, and taking keyword as identification'''
well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
self.assertRun((cmd_tshark,
'-r', capture_file('protobuf_test_map_and_oneof_types.pcapng'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
'-o', 'uat:protobuf_udp_message_types: "8124","test.map.MapMaster"',
'-o', 'protobuf.preload_protos: TRUE',
'-o', 'protobuf.pbf_as_hf: TRUE',
'-Y', 'pbf.test.map.MapMaster.param3 == "I\'m param3 for oneof test."' # test oneof type
' && pbf.test.map.MapMaster.param4MapEntry.value == 1234' # test map type
' && pbf.test.map.Foo.param1 == 88 && pbf.test.map.MapMaster.param5MapEntry.key == 88'
))
self.assertTrue(self.grepOutput('PB[(]test.map.MapMaster[)]'))
def test_protobuf_default_value(self, cmd_tshark, features, dirs, capture_file):
'''Test Protobuf feature adding missing fields with default values'''
well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
self.assertRun((cmd_tshark,
'-r', capture_file('protobuf_test_default_value.pcapng'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
'-o', 'uat:protobuf_udp_message_types: "8128","wireshark.protobuf.test.TestDefaultValueMessage"',
'-o', 'protobuf.preload_protos: TRUE',
'-o', 'protobuf.pbf_as_hf: TRUE',
'-o', 'protobuf.add_default_value: all',
'-O', 'protobuf',
'-Y', 'pbf.wireshark.protobuf.test.TestDefaultValueMessage.enumFooWithDefaultValue_Fouth == -4'
' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.boolWithDefaultValue_False == false'
' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.int32WithDefaultValue_0 == 0'
' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.doubleWithDefaultValue_Negative0point12345678 == -0.12345678'
' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.stringWithDefaultValue_SymbolPi contains "Pi."'
' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.bytesWithDefaultValue_1F2F890D0A00004B == 1f:2f:89:0d:0a:00:00:4b'
' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.optional' # test taking keyword 'optional' as identification
' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.message' # test taking keyword 'message' as identification
))
self.assertTrue(self.grepOutput('floatWithDefaultValue_0point23: 0.23')) # another default value will be displayed
self.assertTrue(self.grepOutput('missing required field \'missingRequiredField\'')) # check the missing required field export warn
def test_protobuf_field_subdissector(self, cmd_tshark, features, dirs, capture_file):
'''Test "protobuf_field" subdissector table'''
well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
lua_file = os.path.join(dirs.lua_dir, 'protobuf_test_field_subdissector_table.lua')
self.assertRun((cmd_tshark,
'-r', capture_file('protobuf_udp_addressbook_with_image_ts.pcapng'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
'-o', 'uat:protobuf_udp_message_types: "8127","tutorial.AddressBook"',
'-o', 'protobuf.preload_protos: TRUE',
'-o', 'protobuf.pbf_as_hf: TRUE',
'-X', 'lua_script:{}'.format(lua_file),
'-Y', 'pbf.tutorial.Person.name == "Jason" && pbf.tutorial.Person.last_updated && png',
))
self.assertTrue(self.grepOutput('PB[(]tutorial.AddressBook[)]'))
def test_protobuf_called_by_custom_dissector(self, cmd_tshark, features, dirs, capture_file):
'''Test Protobuf invoked by other dissector (passing type by pinfo.private)'''
well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
lua_file = os.path.join(dirs.lua_dir, 'protobuf_test_called_by_custom_dissector.lua')
self.assertRun((cmd_tshark,
'-r', capture_file('protobuf_tcp_addressbook.pcapng.gz'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
'-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
'-o', 'protobuf.preload_protos: TRUE',
'-o', 'protobuf.pbf_as_hf: TRUE',
'-X', 'lua_script:{}'.format(lua_file),
'-d', 'tcp.port==18127,addrbook',
'-Y', 'pbf.tutorial.Person.name == "Jason" && pbf.tutorial.Person.last_updated',
))
self.assertTrue(self.grepOutput('tutorial.AddressBook'))
@fixtures.mark_usefixtures('test_env')
@fixtures.uses_fixtures