From ce22d958a86de5a75239e3cb480ff50e37f92553 Mon Sep 17 00:00:00 2001 From: Gerald Combs Date: Tue, 27 Apr 2021 11:06:55 -0700 Subject: [PATCH] Test: Add external tests. Add test/suite_external.py, which can dynamically generate tests from a configuration file. This is intended to make happy-shark useful, but it should make it easy to add simple TShark tests elsewhere. The configuration file format must currently be JSON as described in the Developer's Guide. --- docbook/wsdg_src/WSDG_chapter_tests.adoc | 81 ++++++++++- test/suite_external.py | 168 +++++++++++++++++++++++ test/test.py | 5 + 3 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 test/suite_external.py diff --git a/docbook/wsdg_src/WSDG_chapter_tests.adoc b/docbook/wsdg_src/WSDG_chapter_tests.adoc index 4d0542f827..b6bee042be 100644 --- a/docbook/wsdg_src/WSDG_chapter_tests.adoc +++ b/docbook/wsdg_src/WSDG_chapter_tests.adoc @@ -234,7 +234,7 @@ test failures since the `SubprocessTestCase.tearDown` method is not executed. This limitation might be addressed in the future. [[ChTestsDevelop]] -=== Adding Or Modifying Tests +=== Adding Or Modifying Built-In Tests Tests must be in a Python module whose name matches “suite_*.py”. The module must contain one or more subclasses of “SubprocessTestCase” or @@ -301,3 +301,82 @@ Tests can be run in parallel. This means that any files you create must be unique for each test. “subprocesstest.filename_from_id” can be used to generate a filename based on the current test name. It also ensures that the file will be automatically removed after the test has run. + +[[ChTestsExternal]] +=== Adding Or Modifying External Tests + +You can test the dissection of files outside the Wireshark source code repository by using the external test generator, which creates tests using a JSON configuration file. +The file must have the following format: + +[source] +---- +{ + "case_name": "", + "tests": [ + { + "test_name": "", + "tshark_args": [ ], + "requirements": [ ] + } + ] +} +---- + +`tshark_args` elements can use `${case_dir}` to specify the path to the JSON configuration file. +`requirements` can be one or more of + +`[ "count", "", ]`:: +Require `count` occurrences of `pattern` in the dissection output. +Equivalent to the built-in Python `assertEqual(countOutput(')` + +`[ "grep", "" ]`:: +Dissection output must contain `pattern`. +Equivalent to `assertTrue(grepOutput(''))`. + +`[ "!grep", "" ]`:: +Dissection output must _not_ contain `pattern`. +Equivalent to `assertFalse(grepOutput(''))`. + +`[ "in", "", ]`:: +Zero-indexed line `line` of the dissection output must contain `string`. +Equivalent to `assertIn('', lines[])`. + +`[ "!in", "", ]`:: +Zero-indexed line `line` of the dissection output must _not_ contain `string`. +Equivalent to `assertNotIn('', lines[])`. + +Patterns can be any valid Python regular expression. + +The example below defines a single test case, named “external_example”. +The case has a single test named “dns”, which runs TShark on `tests/dns-1/dns.pcapng`, relative to the JSON configuration file. + +[source,json] +---- +{ + "case_name": "external_example", + "tests": [ + { + "test_name": "dns", + "tshark_args": [ "-r", "${case_dir}/tests/dns-1/dns.pcapng", + "-Y", "dns", "-T", "fields", "-e", "dns.qry.name" + ], + "requirements": [ + [ "count", "in.m.yahoo.com", 1 ], + [ "grep", "in.m.yahoo.com" ], + [ "!grep", "in.m.notyahoo.com" ], + [ "in", "in.m.yahoo.com", 0 ], + [ "!in", "in.m.notyahoo.com", 0 ] + ] + } + ] +} +---- + +You can specify external tests using the `test.py --add-external-test`. +For example, if the JSON file above is named `wireshark-tests.json` you can list its test by running the following: + +[source,sh] +---- +$ ./test/test.py -p ./build/run --add-external-test /path/to/wireshark-tests.json --list external +suite_external.case_external_example.test_dns +---- diff --git a/test/suite_external.py b/test/suite_external.py new file mode 100644 index 0000000000..849b9e7e7d --- /dev/null +++ b/test/suite_external.py @@ -0,0 +1,168 @@ +# +# Externally configured Wireshark tests +# By Gerald Combs +# +# SPDX-License-Identifier: GPL-2.0-or-later +# +''' +External tests + +This module reads one or more test case configuration files specified using `add_external_configs()`. +It creates a test case for each file and one or more tests as specfied. + +Configuration files are JSON-formatted and must have the following structure: + + { + "case_name": "happy_shark", + "tests": [ + { + "test_name": "dns", + "tshark_args": [ "-r", "${case_dir}/tests/dns-1/dns.pcapng", + "-Y", "dns", "-T", "fields", "-e", "dns.qry.name" + ], + "requirements": [ + [ "count", "in.m.yahoo.com", 1 ], + [ "grep", "in.m.yahoo.com" ], + [ "!grep", "in.m.notyahoo.com" ], + [ "in", "in.m.yahoo.com", 0 ], + [ "!in", "in.m.notyahoo.com", 0 ] + ] + } + ] + } + +`${case_dir}` will be replaced by the path to the configuration file. + +"requirements" is a list of search or count requirements. + +Search requirements can have one of the following formats: + + Requirement Python test API equivalent + [ "count", "", ] assertEqual(countOutput(') + [ "grep", "" ] assertTrue(grepOutput('')) + [ "!grep", "" ] assertFalse(grepOutput('')) + [ "in", "", ] assertIn('', lines[]) + [ "!in", "", ] assertNotIn('', lines[]) +''' + +# To do: +# - Add JSON matching so that we can migrate group_asterisk to happy_shark. + +import fixtures +import json +import os.path +import subprocesstest +import unittest +import traceback +import sys + +external_configs = [] +debug = True + + +def add_external_configs(configs): + if configs: + external_configs.extend(configs) + + +def make_tshark_test(tshark_args, requirements): + '''TShark test function generator''' + def tshark_test(self, cmd_tshark, features): + + proc = self.assertRun((cmd_tshark, *tshark_args)) + + for requirement in requirements: + negated = False + try: + if requirement[0].startswith('!'): + negated = True + except IndexError: + self.fail('Test type missing.') + + try: + pattern = requirement[1] + except IndexError: + self.fail('Search pattern missing.') + + if requirement[0] == 'count': + try: + required_count = requirement[2] + except IndexError: + self.fail('"count" requires a count argument.') + self.assertEqual(self.countOutput(pattern), required_count) + elif requirement[0].endswith('grep'): + if negated: + self.assertFalse(self.grepOutput(pattern)) + else: + self.assertTrue(self.grepOutput(pattern)) + elif requirement[0].endswith('in'): + try: + stdout_line = proc.stdout_str.splitlines()[requirement[2]] + except IndexError: + self.fail('"in" requires a line number (starting from zero).') + if negated: + self.assertNotIn(pattern, stdout_line) + else: + self.assertIn(pattern, stdout_line) + else: + self.fail('Unrecognized operation "{}"'.format(requirement[0])) + + return tshark_test + + +def load_tests(loader, standard_tests, pattern): + '''Create our cases and suites. Run by unittest.defaultTestLoader.discover''' + for config_file in external_configs: + try: + with open(config_file, 'r') as cf: + config_str = cf.read() + config_str = config_str.replace('${case_dir}', os.path.dirname(config_file)) + config = json.loads(config_str) + except Error as e: + print('Error reading {}: {}'.format(config_file, e)) + continue + + try: + case_name = 'case_{}'.format(config['case_name']) + except KeyError: + print('Error reading {}: case_name not present'.format(config_file)) + continue + + case_tests = dict() + try: + # Create 'test_...' functions to match our configuration. + test_num = 1 + for test_attrs in config['tests']: + try: + test_name = 'test_{}'.format(test_attrs['test_name']) + except KeyError: + print('{}: No test name for test {} '.format(config_file, test_num)) + continue + + try: + requirements = test_attrs['requirements'] + if not isinstance(requirements, list): + raise TypeError + except: + print('{}: Missing or malformed requirements for test {} '.format(config_file, test_num)) + continue + + tshark_test = make_tshark_test(test_attrs['tshark_args'], requirements) + setattr(tshark_test, '__name__', test_name) + case_tests[test_name] = tshark_test + test_num += 1 + # Create a SubprocessTestCase name 'case_...' and add our 'test_...' functions. + case_class = type(case_name, (subprocesstest.SubprocessTestCase,), case_tests) + # Apply @fixtures.mark_usefixtures('test_env') and @fixtures.uses_fixtures + case_class = fixtures.mark_usefixtures('test_env')(case_class) + case_class = fixtures.uses_fixtures(case_class) + globals()[case_name] = case_class + # Hand our generated class over to unittest.defaultTestLoader. + return loader.loadTestsFromTestCase(case_class) + except KeyError: + print('{}: Missing or malformed tests'.format(config_file)) + except: + if debug: + print(traceback.format_exc()) + raise + return unittest.TestSuite() diff --git a/test/test.py b/test/test.py index a21ab615a9..b04eda71d1 100755 --- a/test/test.py +++ b/test/test.py @@ -17,6 +17,7 @@ import argparse import codecs import os.path +import suite_external import sys import unittest import fixtures @@ -43,6 +44,7 @@ def main(): release_group = parser.add_mutually_exclusive_group() release_group.add_argument('--enable-release', action='store_true', help='Enable release tests') parser.add_argument('-p', '--program-path', default=os.path.curdir, help='Path to Wireshark executables.') + parser.add_argument('-x', '--add-external-tests', action='append', help='Path to an external test definition (.json) file.') parser.add_argument('--skip-missing-programs', help='Skip tests that lack programs from this list instead of failing' ' them. Use "all" to ignore all missing programs.') @@ -55,6 +57,9 @@ def main(): parser.add_argument('tests_to_run', nargs='*', metavar='test', default=['all'], help='Tests to run. One of "all" or a full or partial test name. Default is "all".') args = parser.parse_args() + # XXX This should be a fixture. + suite_external.add_external_configs(args.add_external_tests) + all_tests = unittest.defaultTestLoader.discover(os.path.dirname(__file__), pattern='suite_*') all_ids = []