229 lines
9.5 KiB
Python
Executable File
229 lines
9.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# osmo-gsm-tester.py: main program file
|
|
#
|
|
# Copyright (C) 2016-2020 sysmocom - s.f.m.c. GmbH <info@sysmocom.de>
|
|
#
|
|
# Author: Neels Hofmeyr <neels@hofmeyr.de>
|
|
# Author: Pau Espin Pedrol <pespin@sysmocom.de>
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
# License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
'''osmo-gsm-tester.py: invoke a single test run
|
|
|
|
Examples:
|
|
|
|
./osmo-gsm-tester.py -c doc/examples/2g_osmocom/main.conf ~/my_trial_dir/ -s osmo_trx
|
|
./osmo-gsm-tester.py -c doc/examples/2g_osmocom/main.conf ~/my_trial_dir/ -s sms_tests:dyn_ts+eu_band+bts_sysmo
|
|
./osmo-gsm-tester.py -c sysmocom/main.conf ~/my_trial_dir/ -s sms_tests/mo_mt_sms:bts_trx
|
|
|
|
(The names for test suites and scenarios used in these examples must be defined
|
|
by the osmo-gsm-tester configuration.)
|
|
|
|
A trial package contains binaries (usually built by a jenkins job) of software
|
|
to be run by Osmo-Gsm-Tester, like core network programs as well as binaries for
|
|
the various BTS models on a 2G network.
|
|
|
|
A test suite defines specific actions to be taken and verifies their outcome.
|
|
Such a test suite may leave certain aspects of a setup undefined, e.g. it may
|
|
be BTS model agnostic or does not care which voice codecs are chosen.
|
|
|
|
A test scenario completes the picture in that it defines which specific choices
|
|
shall be made to run a test suite. Any one test suite may thus run on any
|
|
number of different scenarios, e.g. to test various voice codecs.
|
|
|
|
Test scenarios may be combined. For example, one scenario may define a timeslot
|
|
configuration to use, while another scenario may define the voice codec
|
|
configuration.
|
|
|
|
There may still be aspects that are neither required by a test suite nor
|
|
strictly defined by a scenario, which will be resolved automatically, e.g. by
|
|
choosing the first available item that matches the other constraints.
|
|
|
|
A test run thus needs to define:
|
|
* A trial package containing built binaries
|
|
* A set of test suites, each with its combinations of scenarios
|
|
* A main configuration file specifying paths to other files containing sets of
|
|
resources, default configurations and paths on where to find suites,
|
|
scenarios, etc.
|
|
|
|
If no combination of suites and scenarios is provided, the default list of
|
|
suites will be run as defined in the osmo-gsm-tester configuration.
|
|
|
|
The scenarios and suites run for a given trial will be recorded in a trial
|
|
package's directory: Upon launch, a '$trial_dir/run.<date>' directory will be
|
|
created, which will collect logs and reports.
|
|
'''
|
|
|
|
import sys
|
|
import argparse
|
|
from signal import *
|
|
from osmo_gsm_tester import __version__
|
|
from osmo_gsm_tester.core import log
|
|
from osmo_gsm_tester.core import trial
|
|
from osmo_gsm_tester.core import suite
|
|
from osmo_gsm_tester.core import config
|
|
from osmo_gsm_tester.core.schema import generate_schemas
|
|
|
|
def sig_handler_cleanup(signum, frame):
|
|
print("killed by signal %d" % signum)
|
|
# This sys.exit() will raise a SystemExit base exception at the current
|
|
# point of execution. Code must be prepared to clean system-wide resources
|
|
# by using the "finally" section. This allows at the end 'atexit' hooks to
|
|
# be called before exiting.
|
|
sys.exit(1)
|
|
|
|
def main():
|
|
|
|
for sig in (SIGINT, SIGTERM, SIGQUIT, SIGPIPE, SIGHUP):
|
|
signal(sig, sig_handler_cleanup)
|
|
|
|
parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter)
|
|
# Note: since we're using RawTextHelpFormatter to keep nicely separate
|
|
# paragraphs in the long help text, we unfortunately also need to take care
|
|
# of line wraps in the shorter cmdline options help.
|
|
# The line width here is what remains of screen width after the list of
|
|
# options placed by ArgumentParser. That's unfortunately subject to change
|
|
# and undefined, so when things change, just run a local
|
|
# ./osmo-gsm-tester.py --help and try to keep everything in 80 chars width.
|
|
# The help text is indented automatically, but line width is manual.
|
|
# Using multi-line strings here -- doesn't look nice in the python flow but
|
|
# is easiest to maintain.
|
|
parser.add_argument('-V', '--version', action='store_true',
|
|
help='Show version')
|
|
parser.add_argument('-c', '--conf-path', dest='conf_path',
|
|
help='''Specify main configuration file path''')
|
|
parser.add_argument('trial_dir', nargs='?', default=None,
|
|
help='Directory containing binaries to test')
|
|
parser.add_argument('-s', '--suite-scenario', dest='suite_scenario', action='append',
|
|
help='''A suite-scenarios combination
|
|
like suite:scenario+scenario''')
|
|
parser.add_argument('-S', '--suites-file', dest='suites_file', action='append',
|
|
default=[],
|
|
help='''Read suites to run from a yml listing,
|
|
like default-suites.conf. The path is relative to
|
|
--conf-path.''')
|
|
parser.add_argument('-t', '--test', dest='test', action='append',
|
|
help='''Run only tests matching this name.
|
|
Any test name that contains the given string is run.
|
|
To get an exact match, prepend a "=" like
|
|
"-t =my_exact_name". The ".py" suffix is always
|
|
optional.''')
|
|
parser.add_argument('-l', '--log-level', dest='log_level', choices=log.LEVEL_STRS.keys(),
|
|
default=None,
|
|
help='Set logging level for all categories (on stdout)')
|
|
parser.add_argument('-T', '--traceback', dest='trace', action='store_true',
|
|
help='Enable stdout logging of tracebacks')
|
|
parser.add_argument('-R', '--source', dest='source', action='store_true',
|
|
help='Enable stdout logging of source file')
|
|
args = parser.parse_args()
|
|
|
|
if args.version:
|
|
print(__version__)
|
|
exit(0)
|
|
|
|
print('combinations:', repr(args.suite_scenario))
|
|
print('trial:', repr(args.trial_dir))
|
|
print('tests:', repr(args.test))
|
|
|
|
# create a default log to stdout
|
|
log.LogTarget().style(all_origins_on_levels=(log.L_ERR, log.L_TRACEBACK), src=False)
|
|
|
|
if args.log_level:
|
|
log.set_all_levels(log.LEVEL_STRS.get(args.log_level))
|
|
if args.trace:
|
|
log.style_change(trace=True)
|
|
if args.source:
|
|
log.style_change(src=True)
|
|
if args.conf_path:
|
|
config.override_conf = args.conf_path
|
|
|
|
if args.trial_dir is not None:
|
|
trial_dir = args.trial_dir
|
|
else:
|
|
trial_dir = config.get_main_config_value(config.CFG_TRIAL_DIR)
|
|
|
|
combination_strs = list(args.suite_scenario or [])
|
|
|
|
for suites_file in args.suites_file:
|
|
suites_file = config.main_config_path_to_abspath(suites_file)
|
|
from_this_file = config.read(suites_file)
|
|
print(('Running suites from %r:\n ' % suites_file) + ('\n '.join(from_this_file)))
|
|
combination_strs.extend(from_this_file)
|
|
|
|
if not combination_strs:
|
|
combination_strs = config.read_config_file(config.CFG_DEFAULT_SUITES_CONF, if_missing_return=[])
|
|
|
|
if combination_strs:
|
|
print('Running default suites:\n ' + ('\n '.join(combination_strs)))
|
|
else:
|
|
print('Failed to load default suites (%r)' % config.get_main_config_value(config.DEFAULT_SUITES_CONF, fail_if_missing=False))
|
|
|
|
|
|
if not combination_strs:
|
|
raise RuntimeError('Need at least one suite:scenario to run')
|
|
|
|
# Generate supported schemas dynamically from objects:
|
|
generate_schemas()
|
|
|
|
# make sure all suite:scenarios exist
|
|
suite_scenarios = []
|
|
for combination_str in combination_strs:
|
|
suite_scenarios.append(suite.load_suite_scenario_str(combination_str))
|
|
|
|
# pick tests and make sure they exist
|
|
test_names = []
|
|
for test_name in (args.test or []):
|
|
found = False
|
|
if test_name.startswith('=') and not test_name.endswith('.py'):
|
|
test_name = test_name + '.py'
|
|
for suite_scenario_str, suite_def, scenarios in suite_scenarios:
|
|
for def_test_name in suite_def.test_basenames:
|
|
if test_name.startswith('='):
|
|
match = test_name[1:] == def_test_name
|
|
else:
|
|
match = test_name in def_test_name
|
|
if match:
|
|
found = True
|
|
test_names.append(def_test_name)
|
|
if not found:
|
|
raise RuntimeError('No test found for %r' % test_name)
|
|
if test_names:
|
|
test_names = sorted(set(test_names))
|
|
print(repr(test_names))
|
|
|
|
with trial.Trial(trial_dir) as current_trial:
|
|
current_trial.verify()
|
|
for suite_scenario_str, suite_def, scenarios in suite_scenarios:
|
|
current_trial.add_suite_run(suite_scenario_str, suite_def, scenarios)
|
|
current_trial.run_suites(test_names)
|
|
|
|
if current_trial.status != trial.Trial.PASS:
|
|
return 1
|
|
return 0
|
|
|
|
if __name__ == '__main__':
|
|
rc = 2
|
|
try:
|
|
rc = main()
|
|
except:
|
|
# Tell the log to show the exception, then terminate the program with the exception anyway.
|
|
# Since exceptions within test runs should be caught and evaluated, this is basically about
|
|
# exceptions during command line parsing and such, so it's appropriate to abort immediately.
|
|
log.log_exn()
|
|
raise
|
|
exit(rc)
|
|
|
|
# vim: expandtab tabstop=4 shiftwidth=4
|