Test: Make sure we run our display filter tests.

Change the test suite list in CMakeLists.txt to a static list. Add a
CTest coverage unit test.

Change-Id: I8459f320a2d0707618d6d56abdfce80274fddd2d
Reviewed-on: https://code.wireshark.org/review/27377
Petri-Dish: Gerald Combs <gerald@wireshark.org>
Tested-by: Petri Dish Buildbot
Reviewed-by: Gerald Combs <gerald@wireshark.org>

CMakeLists.txt

@@ -2957,7 +2957,7 @@ if(RPMBUILD_EXECUTABLE)
 
 	_SET_GITVERSION_CMAKE_VARIABLE(_git_description)
 
-	if (NOT _git_description)	
+	if (NOT _git_description)
 		# We're building the rpm outside the source. Guess the version from the dirname.
 		get_filename_component(CMAKE_SOURCE_DIR_NAME ${CMAKE_SOURCE_DIR} NAME)
 		# XXX this assumes the directory to start with "wireshark-"
@@ -3160,26 +3160,37 @@ set_target_properties(test-programs PROPERTIES
 
 # Test suites
 enable_testing()
-file(GLOB _test_suite_py_list test/suite_*.py)
+# We could try to build this list dynamically, but given that we tend to
+# go years between adding suites just run
+#     test/test.py --list-suites | sort
+# and paste the output here.
+set(_test_suite_list
+	suite_capture
+	suite_clopts
+	suite_decryption
+	suite_dfilter
+	suite_dissection
+	suite_fileformats
+	suite_io
+	suite_mergecap
+	suite_nameres
+	suite_text2pcap
+	suite_unittests
+	suite_wslua
+)
 if(WIN32)
 	set(_test_suite_program_path ./run/$<CONFIG>)
 else()
 	set(_test_suite_program_path ./run)
 endif()
 
-# We currently don't handle spaces in arguments. On Windows this
+# We don't currently handle spaces in arguments. On Windows this
 # means that you will probably have to pass in an interface index
 # instead of a name.
 set(TEST_EXTRA_ARGS "" CACHE STRING "Extra arguments to pass to test/test.py")
 separate_arguments(TEST_EXTRA_ARGS)
 
-# We can enumerate suites two ways: by probing the filesystem and by
-# running `test.py --list-suites`. Probe the filesystem for now, which
-# should hopefully give us enough parallelization. If we want to split
-# our tests by cases or individual tests we'll have to run and parse
-# `test.py --list-cases` or `test.py --list` respectively.
-foreach(_suite_py ${_test_suite_py_list})
-	get_filename_component(_suite_name ${_suite_py} NAME_WE)
+foreach(_suite_name ${_test_suite_list})
 	add_test(
 		NAME ${_suite_name}
 		COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/test/test.py

test/config.py

@@ -63,6 +63,8 @@ key_dir = os.path.join(this_dir, 'keys')
 lua_dir = os.path.join(this_dir, 'lua')
 tools_dir = os.path.join(this_dir, '..', 'tools')
 
+all_suites = []
+
 def canCapture():
     # XXX This appears to be evaluated at the wrong time when called
     # from a unittest.skipXXX decorator.

test/suite_dfilter/group_bytes_ether.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_bytes_ether(dfiltertest.DFTestCase):
     trace_file = "ipx_rip.pcap"

test/suite_dfilter/group_bytes_ipv6.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_bytes_ipv6(dfiltertest.DFTestCase):
     trace_file = "ipv6.pcap"

test/suite_dfilter/group_bytes_type.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_bytes_type(dfiltertest.DFTestCase):
     trace_file = "arp.pcap"

test/suite_dfilter/group_double.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_double(dfiltertest.DFTestCase):
 

test/suite_dfilter/group_integer.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_integer(dfiltertest.DFTestCase):
     trace_file = "ntp.pcap"

test/suite_dfilter/group_integer_1byte.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_integer_1_byte(dfiltertest.DFTestCase):
 

test/suite_dfilter/group_ipv4.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_ipv4(dfiltertest.DFTestCase):
     trace_file = "nfs.pcap"

test/suite_dfilter/group_membership.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_membership(dfiltertest.DFTestCase):
     trace_file = "http.pcap"

test/suite_dfilter/group_range_method.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_range(dfiltertest.DFTestCase):
     trace_file = "ipx_rip.pcap"

test/suite_dfilter/group_scanner.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_scanner(dfiltertest.DFTestCase):
     trace_file = "http.pcap"

test/suite_dfilter/group_string_type.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_string(dfiltertest.DFTestCase):
     trace_file = "http.pcap"

test/suite_dfilter/group_stringz.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_stringz(dfiltertest.DFTestCase):
     trace_file = "tftp.pcap"

test/suite_dfilter/group_time_relative.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_time_relative(dfiltertest.DFTestCase):
     trace_file = "nfs.pcap"

test/suite_dfilter/group_time_type.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_time(dfiltertest.DFTestCase):
     trace_file = "http.pcap"

test/suite_dfilter/group_tvb.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 import unittest
 
 class case_tvb(dfiltertest.DFTestCase):

test/suite_dfilter/group_uint64.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_uint64(dfiltertest.DFTestCase):
     trace_file = "nfs.pcap"

test/suite_unittests.py

@@ -12,7 +12,10 @@
 '''EPAN unit tests'''
 
 import config
+import difflib
 import os.path
+import pprint
+import re
 import subprocesstest
 import unittest
 
@@ -49,6 +52,26 @@ class case_unittests(subprocesstest.SubprocessTestCase):
         '''fieldcount'''
         self.assertRun((config.cmd_tshark, '-G', 'fieldcount'))
 
+    def test_unit_ctest_coverage(self):
+        '''Make sure CTest runs all of our tests.'''
+        with open(os.path.join(config.this_dir, '..', 'CMakeLists.txt')) as cml_fd:
+            suite_re = re.compile('set *\( *_test_suite_list')
+            in_list = False
+            cml_suites = []
+            for cml_line in cml_fd:
+                if suite_re.search(cml_line):
+                    in_list = True
+                    continue
+                if in_list:
+                    if ')' in cml_line:
+                        break
+                    cml_suites.append(cml_line.strip())
+        cml_suites.sort()
+        if not config.all_suites == cml_suites:
+            diff = '\n'.join(list(difflib.unified_diff(config.all_suites, cml_suites, 'all test suites', 'CMakeLists.txt test suites')))
+            self.fail("CMakeLists.txt doesn't test all available suites:\n" + diff)
+
+
 class Proto:
     """Data for a protocol."""
     def __init__(self, line):

test/test.py

@@ -87,12 +87,15 @@ def main():
         print('\n'.join(run_ids))
         sys.exit(0)
 
+    all_suites = set()
+    for aid in all_ids:
+        aparts = aid.split('.')
+        all_suites |= {aparts[0]}
+    config.all_suites = list(all_suites)
+    config.all_suites.sort()
+
     if args.list_suites:
-        suites = set()
-        for rid in run_ids:
-            rparts = rid.split('.')
-            suites |= {rparts[0]}
-        print('\n'.join(list(suites)))
+        print('\n'.join(config.all_suites))
         sys.exit(0)
 
     if args.list_cases: