Protobuf/gRPC: add test cases for Protobuf and gRPC

Add case_dissect_protobuf and case_dissect_grpc in test/suite_dissection.py.
Add *.proto into the sub directories of test/protobuf_lang_files/.
Run command like 'pytest --program-path .\run\Debug\ -k "grpc or protobuf"'
in build directory (in windows) to test these cases only.

test/fixtures_ws.py

@@ -195,6 +195,7 @@ def dirs():
         config_dir=os.path.join(this_dir, 'config'),
         key_dir=os.path.join(this_dir, 'keys'),
         lua_dir=os.path.join(this_dir, 'lua'),
+        protobuf_lang_files_dir=os.path.join(this_dir, 'protobuf_lang_files'),
         tools_dir=os.path.join(this_dir, '..', 'tools'),
     )
 

test/lua/protobuf_test_called_by_custom_dissector.lua

@@ -0,0 +1,68 @@
+do
+    local protobuf_dissector = Dissector.get("protobuf")
+
+    -- Create protobuf dissector based on UDP or TCP.
+    -- The UDP dissector will take the whole tvb as a message.
+    -- The TCP dissector will parse tvb as format:
+    --         [4bytes length][a message][4bytes length][a message]...
+    -- @param name  The name of the new dissector.
+    -- @param desc  The description of the new dissector.
+    -- @param for_udp  Register the new dissector to UDP table.(Enable 'Decode as')
+    -- @param for_tcp  Register the new dissector to TCP table.(Enable 'Decode as')
+    -- @param msgtype  Message type. This must be the root message defined in your .proto file.
+    local function create_protobuf_dissector(name, desc, for_udp, for_tcp, msgtype)
+        local proto = Proto(name, desc)
+        local f_length = ProtoField.uint32(name .. ".length", "Length", base.DEC)
+        proto.fields = { f_length }
+
+        proto.dissector = function(tvb, pinfo, tree)
+            local subtree = tree:add(proto, tvb())
+            if for_udp and pinfo.port_type == 3 then -- UDP
+                if msgtype ~= nil then
+                    pinfo.private["pb_msg_type"] = "message," .. msgtype
+                end
+                pcall(Dissector.call, protobuf_dissector, tvb, pinfo, subtree)
+            elseif for_tcp and pinfo.port_type == 2 then -- TCP
+                local offset = 0
+                local remaining_len = tvb:len()
+                while remaining_len > 0 do
+                    if remaining_len < 4 then -- head not enough
+                        pinfo.desegment_offset = offset
+                        pinfo.desegment_len = DESEGMENT_ONE_MORE_SEGMENT
+                        return -1
+                    end
+
+                    local data_len = tvb(offset, 4):uint()
+
+                    if remaining_len - 4 < data_len then -- data not enough
+                        pinfo.desegment_offset = offset
+                        pinfo.desegment_len = data_len - (remaining_len - 4)
+                        return -1
+                    end
+                    subtree:add(f_length, tvb(offset, 4))
+
+                    if msgtype ~= nil then
+                        pinfo.private["pb_msg_type"] = "message," .. msgtype
+                    end
+                    pcall(Dissector.call, protobuf_dissector, 
+                           tvb(offset + 4, data_len):tvb(), pinfo, subtree)
+
+                    offset = offset + 4 + data_len
+                    remaining_len = remaining_len - 4 - data_len
+                end
+            end
+            pinfo.columns.protocol:set(name)
+        end
+
+        if for_udp then DissectorTable.get("udp.port"):add(0, proto) end
+        if for_tcp then DissectorTable.get("tcp.port"):add(0, proto) end
+        return proto
+    end
+
+    -- default pure protobuf udp and tcp dissector without message type
+    create_protobuf_dissector("protobuf_udp", "Protobuf UDP")
+    create_protobuf_dissector("protobuf_tcp", "Protobuf TCP")
+    -- add more protobuf dissectors with message types
+    create_protobuf_dissector("AddrBook", "Tutorial AddressBook",
+                              true, true, "tutorial.AddressBook")
+end

test/lua/protobuf_test_field_subdissector_table.lua

@@ -0,0 +1,6 @@
+-- Test protobuf_field dissector table
+do
+    local protobuf_field_table = DissectorTable.get("protobuf_field")
+    local png_dissector = Dissector.get("png")
+    protobuf_field_table:add("tutorial.Person.portrait_image", png_dissector)
+end

test/protobuf_lang_files/user_defined_types/addressbook.proto

@@ -0,0 +1,30 @@
+// This file comes from the official Protobuf example with a little modification.
+syntax = "proto3";
+package tutorial;
+import "google/protobuf/timestamp.proto";
+
+message Person {
+  string name = 1;
+  int32 id = 2;  // Unique ID number for this person.
+  string email = 3;
+
+  enum PhoneType {
+    MOBILE = 0;
+    HOME = 1;
+    WORK = 2;
+  }
+
+  message PhoneNumber {
+    string number = 1;
+    PhoneType type = 2;
+  }
+
+  repeated PhoneNumber phone = 4;
+  google.protobuf.Timestamp last_updated = 5;
+  bytes portrait_image = 6;
+}
+
+message AddressBook {
+  repeated Person people = 1;
+}
+

test/protobuf_lang_files/user_defined_types/person_search_service.proto

@@ -0,0 +1,15 @@
+// A gRPC service that searches for persons based on certain attributes.
+syntax = "proto3";
+package tutorial;
+import "addressbook.proto";
+
+message PersonSearchRequest {
+  repeated string name = 1;
+  repeated int32 id = 2;
+  repeated string phoneNumber = 3;
+}
+
+service PersonSearchService {
+  rpc Search (PersonSearchRequest) returns (stream Person) {}
+}
+

test/protobuf_lang_files/user_defined_types/test_default_value.proto

@@ -0,0 +1,59 @@
+// Test default values of Protobuf fields
+syntax = "proto2";
+package wireshark.protobuf.test;
+
+message TestDefaultValueMessage {
+  enum EnumFoo {
+    ENUM_FOO_V_FIRST = 1;
+    ENUM_FOO_V_SECOND = 0x2;
+    ENUM_FOO_V_THIRD = 3;
+    ENUM_FOO_V_FOURTH = - 4;
+  }
+
+  // The format of field name is:
+  //    <type> "With" ( "Value" | "DefaultValue" | "NoValue" ) [ "_" <correct_value_in_wireshark> ]
+  // The "DefaultValue" fields should be wrapped with generated mark ("[" and "]") of Wireshark tree item.
+  // The "NoValue" fields should not appear in Wireshark.
+
+  // The default value is overridden to 8 at running time.
+  required int32 int32WithValue_8 = 1 [ default = 2 ];
+  // The default value is overridden to ENUM_FOO_V_THIRD at running time.
+  optional EnumFoo enumFooWithValue_Third = 2 [ default = ENUM_FOO_V_SECOND ];
+  
+  // default values of bool
+  optional bool boolWithDefaultValue_False = 11;
+  optional bool boolWithDefaultValue_True = 12 [ default = true ];
+  
+  // default values of enum
+  optional EnumFoo enumFooWithDefaultValue_First = 21;
+  optional EnumFoo enumFooWithDefaultValue_Second = 22 [ default = ENUM_FOO_V_SECOND ];
+  optional EnumFoo enumFooWithDefaultValue_Fouth = 23 [ default = ENUM_FOO_V_FOURTH ];
+
+  // default values of integer number
+  optional int32 int32WithDefaultValue_0 = 31;
+  optional int64 int64WithDefaultValue_Negative1152921504606846976 = 32 [ default = - 1152921504606846976 ];
+  optional uint32 uint32WithDefaultValue_11 = 33 [ default = 11 ];
+  optional uint64 uint64WithDefaultValue_1152921504606846976 = 34 [ default = 1152921504606846976 ]; // equals to 2^60
+  optional sint32 sint32WithDefaultValue_Negative12 = 35 [ default = -12 ];
+  optional sint64 sint64WithDefaultValue_0 = 36; // default value is zero
+  optional fixed64 fixed64WithDefaultValue_1152921504606846976 = 37 [ default = 1152921504606846976 ];
+  optional sfixed32 sfixed32WithDefaultValue_Negative31 = 38 [ default = -0X1f ]; // -21
+
+  // default values of float and double
+  optional float floatWithDefaultValue_0point23 = 41 [ default = 0.23 ];
+  optional double doubleWithDefaultValue_Negative0point12345678 = 42 [ default = -0.12345678 ];
+
+  // default values of string and bytes
+  optional string stringWithNoValue = 51; // default value must not appear because not declared
+  optional string stringWithDefaultValue_SymbolPi = 52 [ default = "The symbol \'\xF0\x9D\x9B\x91\' is mathematical bold small Pi."]; 
+  optional bytes bytesWithNoValue = 53; // default value must not appear because not declared
+  // '\'nnn is octal value of a byte, '\x'nn is hex value of a byte
+  optional bytes bytesWithDefaultValue_1F2F890D0A00004B = 54 [ default = "\x1F\x2F\211\r\n\000\x0\x4B" ];
+
+  // others
+  repeated int32 repeatedFieldWithNoValue = 81; // should not appear
+  required int32 missingRequiredField = 82; // for testing required field. (comment this line if you regenerated stub code)
+  // test taking keyword as identification feature
+  optional int32 message = 83;
+  optional int32 optional = 84;
+}

test/protobuf_lang_files/user_defined_types/test_leading_dot.proto

@@ -0,0 +1,24 @@
+syntax="proto3";
+
+package a.b;
+
+message a {
+  string param1 = 1;
+
+  message b {
+    string param2 = 2;
+
+    message c {
+      string param3 = 3;
+    }
+  }
+}
+
+message msg {
+  a.b.c param4 = 4; /* the full name of the type is a.b.a.b.c */
+  .a.b.c param5 = 5; /* the full name of the type is a.b.c */
+}
+
+message c {
+  string param6 = 6;
+}

test/protobuf_lang_files/user_defined_types/test_map.proto

@@ -0,0 +1,17 @@
+syntax="proto3";
+
+package test.map;
+
+message Foo {
+  int32 param1 = 1;
+}
+
+message MapMaster {
+  oneof Abc {
+    string param2 = 2;
+    string param3 = 3;
+  }
+
+  map<string, int64> param4 = 4;
+  map<sint32, Foo> param5 = 5;
+}

test/protobuf_lang_files/well_know_types/google/protobuf/timestamp.proto

@@ -0,0 +1,20 @@
+// This file is from https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto
+// To reduce the file size, some comments have been removed.
+
+syntax = "proto3";
+
+package google.protobuf;
+
+message Timestamp {
+
+  // Represents seconds of UTC time since Unix epoch
+  // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to
+  // 9999-12-31T23:59:59Z inclusive.
+  int64 seconds = 1;
+
+  // Non-negative fractions of a second at nanosecond resolution. Negative
+  // second values with fractions must still have non-negative nanos values
+  // that count forward in time. Must be from 0 to 999,999,999
+  // inclusive.
+  int32 nanos = 2;
+}

test/suite_dissection.py

@@ -14,6 +14,38 @@ import unittest
 import fixtures
 import sys
 
+@fixtures.mark_usefixtures('test_env')
+@fixtures.uses_fixtures
+class case_dissect_grpc(subprocesstest.SubprocessTestCase):
+    def test_grpc_with_json(self, cmd_tshark, features, dirs, capture_file):
+        '''gRPC with JSON payload'''
+        if not features.have_nghttp2:
+            self.skipTest('Requires nghttp2.')
+        self.assertRun((cmd_tshark,
+                '-r', capture_file('grpc_person_search_json_with_image.pcapng.gz'),
+                '-d', 'tcp.port==50052,http2',
+                '-Y', 'grpc.message_length == 208 && json.value.string == "87561234"',
+            ))
+        self.assertTrue(self.grepOutput('GRPC/JSON'))
+
+    def test_grpc_with_protobuf(self, cmd_tshark, features, dirs, capture_file):
+        '''gRPC with Protobuf payload'''
+        if not features.have_nghttp2:
+            self.skipTest('Requires nghttp2.')
+        well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
+        user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
+        self.assertRun((cmd_tshark,
+                '-r', capture_file('grpc_person_search_protobuf_with_image.pcapng.gz'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
+                '-d', 'tcp.port==50051,http2',
+                '-Y', 'protobuf.message.name == "tutorial.PersonSearchRequest"'
+                      ' || (grpc.message_length == 66 && protobuf.field.value.string == "Jason"'
+                      '     && protobuf.field.value.int64 == 1602601886)',
+            ))
+        self.assertTrue(self.grepOutput('tutorial.PersonSearchService/Search')) # grpc request
+        self.assertTrue(self.grepOutput('tutorial.Person')) # grpc response
+
 @fixtures.mark_usefixtures('test_env')
 @fixtures.uses_fixtures
 class case_dissect_http(subprocesstest.SubprocessTestCase):
@@ -82,6 +114,116 @@ class case_dissect_http2(subprocesstest.SubprocessTestCase):
         self.assertFalse(self.grepOutput('00000000  00 00 12 04 00 00 00 00'))
         self.assertTrue(self.grepOutput('00000000  00 00 2c 01 05 00 00 00'))
 
+@fixtures.mark_usefixtures('test_env')
+@fixtures.uses_fixtures
+class case_dissect_protobuf(subprocesstest.SubprocessTestCase):
+    def test_protobuf_udp_message_mapping(self, cmd_tshark, features, dirs, capture_file):
+        '''Test Protobuf UDP Message Mapping and parsing google.protobuf.Timestamp features'''
+        well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
+        user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
+        self.assertRun((cmd_tshark,
+                '-r', capture_file('protobuf_udp_addressbook_with_image_ts.pcapng'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
+                '-o', 'uat:protobuf_udp_message_types: "8127","tutorial.AddressBook"',
+                '-o', 'protobuf.preload_protos: TRUE',
+                '-o', 'protobuf.pbf_as_hf: TRUE',
+                '-Y', 'pbf.tutorial.Person.name == "Jason"'
+                      ' && pbf.tutorial.Person.last_updated > "2020-10-15"'
+                      ' && pbf.tutorial.Person.last_updated < "2020-10-19"',
+            ))
+        self.assertTrue(self.grepOutput('tutorial.AddressBook'))
+
+    def test_protobuf_message_type_leading_with_dot(self, cmd_tshark, features, dirs, capture_file):
+        '''Test Protobuf Message type is leading with dot'''
+        well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
+        user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
+        self.assertRun((cmd_tshark,
+                '-r', capture_file('protobuf_test_leading_dot.pcapng'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
+                '-o', 'uat:protobuf_udp_message_types: "8123","a.b.msg"',
+                '-o', 'protobuf.preload_protos: TRUE',
+                '-o', 'protobuf.pbf_as_hf: TRUE',
+                '-Y', 'pbf.a.b.a.b.c.param3 contains "in a.b.a.b.c" && pbf.a.b.c.param6 contains "in a.b.c"',
+            ))
+        self.assertTrue(self.grepOutput('PB[(]a.b.msg[)]'))
+
+    def test_protobuf_map_and_oneof_types(self, cmd_tshark, features, dirs, capture_file):
+        '''Test Protobuf map and oneof types, and taking keyword as identification'''
+        well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
+        user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
+        self.assertRun((cmd_tshark,
+                '-r', capture_file('protobuf_test_map_and_oneof_types.pcapng'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
+                '-o', 'uat:protobuf_udp_message_types: "8124","test.map.MapMaster"',
+                '-o', 'protobuf.preload_protos: TRUE',
+                '-o', 'protobuf.pbf_as_hf: TRUE',
+                '-Y', 'pbf.test.map.MapMaster.param3 == "I\'m param3 for oneof test."'  # test oneof type
+                      ' && pbf.test.map.MapMaster.param4MapEntry.value == 1234'        # test map type
+                      ' && pbf.test.map.Foo.param1 == 88 && pbf.test.map.MapMaster.param5MapEntry.key == 88'
+            ))
+        self.assertTrue(self.grepOutput('PB[(]test.map.MapMaster[)]'))
+
+    def test_protobuf_default_value(self, cmd_tshark, features, dirs, capture_file):
+        '''Test Protobuf feature adding missing fields with default values'''
+        well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
+        user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
+        self.assertRun((cmd_tshark,
+                '-r', capture_file('protobuf_test_default_value.pcapng'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
+                '-o', 'uat:protobuf_udp_message_types: "8128","wireshark.protobuf.test.TestDefaultValueMessage"',
+                '-o', 'protobuf.preload_protos: TRUE',
+                '-o', 'protobuf.pbf_as_hf: TRUE',
+                '-o', 'protobuf.add_default_value: all',
+                '-O', 'protobuf',
+                '-Y', 'pbf.wireshark.protobuf.test.TestDefaultValueMessage.enumFooWithDefaultValue_Fouth == -4'
+                      ' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.boolWithDefaultValue_False == false'
+                      ' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.int32WithDefaultValue_0 == 0'
+                      ' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.doubleWithDefaultValue_Negative0point12345678 == -0.12345678'
+                      ' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.stringWithDefaultValue_SymbolPi contains "Pi."'
+                      ' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.bytesWithDefaultValue_1F2F890D0A00004B == 1f:2f:89:0d:0a:00:00:4b'
+                      ' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.optional' # test taking keyword 'optional' as identification
+                      ' && pbf.wireshark.protobuf.test.TestDefaultValueMessage.message' # test taking keyword 'message' as identification
+            ))
+        self.assertTrue(self.grepOutput('floatWithDefaultValue_0point23: 0.23')) # another default value will be displayed
+        self.assertTrue(self.grepOutput('missing required field \'missingRequiredField\'')) # check the missing required field export warn
+
+    def test_protobuf_field_subdissector(self, cmd_tshark, features, dirs, capture_file):
+        '''Test "protobuf_field" subdissector table'''
+        well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
+        user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
+        lua_file = os.path.join(dirs.lua_dir, 'protobuf_test_field_subdissector_table.lua')
+        self.assertRun((cmd_tshark,
+                '-r', capture_file('protobuf_udp_addressbook_with_image_ts.pcapng'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
+                '-o', 'uat:protobuf_udp_message_types: "8127","tutorial.AddressBook"',
+                '-o', 'protobuf.preload_protos: TRUE',
+                '-o', 'protobuf.pbf_as_hf: TRUE',
+                '-X', 'lua_script:{}'.format(lua_file),
+                '-Y', 'pbf.tutorial.Person.name == "Jason" && pbf.tutorial.Person.last_updated && png',
+            ))
+        self.assertTrue(self.grepOutput('PB[(]tutorial.AddressBook[)]'))
+
+    def test_protobuf_called_by_custom_dissector(self, cmd_tshark, features, dirs, capture_file):
+        '''Test Protobuf invoked by other dissector (passing type by pinfo.private)'''
+        well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/')
+        user_defined_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'user_defined_types').replace('\\', '/')
+        lua_file = os.path.join(dirs.lua_dir, 'protobuf_test_called_by_custom_dissector.lua')
+        self.assertRun((cmd_tshark,
+                '-r', capture_file('protobuf_tcp_addressbook.pcapng.gz'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'),
+                '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(user_defined_types_dir, 'TRUE'),
+                '-o', 'protobuf.preload_protos: TRUE',
+                '-o', 'protobuf.pbf_as_hf: TRUE',
+                '-X', 'lua_script:{}'.format(lua_file),
+                '-d', 'tcp.port==18127,addrbook',
+                '-Y', 'pbf.tutorial.Person.name == "Jason" && pbf.tutorial.Person.last_updated',
+            ))
+        self.assertTrue(self.grepOutput('tutorial.AddressBook'))
 
 @fixtures.mark_usefixtures('test_env')
 @fixtures.uses_fixtures