Add JUnit XML reports; refactor test reporting
* Add Junit output file support * Differentiate between an expected failure test and an error in the test, as described in JUnit. * In case of an error/exception during test, record and attach it to the Test object and continue running the tests, and show it at the end during the trial report. Change-Id: Iedf6d912b3cce3333a187a4ac6d5c6b70fe9d5c5
This commit is contained in:
parent
023fd2c748
commit
0ffb414406
|
@ -8,3 +8,4 @@ set_pythonpath
|
||||||
test_work
|
test_work
|
||||||
state
|
state
|
||||||
*.pyc
|
*.pyc
|
||||||
|
selftest/trial_test/
|
||||||
|
|
|
@ -59,15 +59,30 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]]
|
||||||
tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]]
|
tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]]
|
||||||
tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]]
|
tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]]
|
||||||
tst hello_world.py:[LINENR] PASS [test_suite↪hello_world.py]
|
tst hello_world.py:[LINENR] PASS [test_suite↪hello_world.py]
|
||||||
pass: all 1 tests passed.
|
tst test_suite: PASS
|
||||||
|
pass: all 6 tests passed (5 skipped).
|
||||||
|
|
||||||
- a test with an error
|
- a test with an error
|
||||||
tst test_suite: Suite run start [suite.py:[LINENR]]
|
tst test_suite: Suite run start [suite.py:[LINENR]]
|
||||||
tst test_error.py:[LINENR] START [test_suite↪test_error.py] [suite.py:[LINENR]]
|
tst test_error.py:[LINENR] START [test_suite↪test_error.py] [suite.py:[LINENR]]
|
||||||
tst test_error.py:[LINENR]: I am 'test_suite' / 'test_error.py:[LINENR]' [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]]
|
tst test_error.py:[LINENR]: I am 'test_suite' / 'test_error.py:[LINENR]' [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]]
|
||||||
tst test_error.py:[LINENR]: FAIL [test_suite↪test_error.py:[LINENR]] [suite.py:[LINENR]]
|
tst test_error.py:[LINENR]: ERR: AssertionError: [test_error.py:[LINENR]: assert False]
|
||||||
tst test_error.py:[LINENR]: ERR: AssertionError: [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]: assert False]
|
tst test_error.py:[LINENR] FAIL (AssertionError) [test_suite↪test_error.py] [suite.py:[LINENR]]
|
||||||
FAIL: 1 of 1 tests failed:
|
tst test_suite: FAIL [suite.py:[LINENR]]
|
||||||
test_error.py
|
|
||||||
|
- a test with a failure
|
||||||
|
tst test_suite: Suite run start [suite.py:[LINENR]]
|
||||||
|
tst test_fail.py:[LINENR] START [test_suite↪test_fail.py] [suite.py:[LINENR]]
|
||||||
|
tst test_fail.py:[LINENR]: I am 'test_suite' / 'test_fail.py:[LINENR]' [test_suite↪test_fail.py:[LINENR]] [test_fail.py:[LINENR]]
|
||||||
|
tst test_fail.py:[LINENR] FAIL (EpicFail) [test_suite↪test_fail.py] [suite.py:[LINENR]]
|
||||||
|
tst test_suite: FAIL [suite.py:[LINENR]]
|
||||||
|
|
||||||
|
- a test with a raised failure
|
||||||
|
tst test_suite: Suite run start [suite.py:[LINENR]]
|
||||||
|
tst test_fail_raise.py:[LINENR] START [test_suite↪test_fail_raise.py] [suite.py:[LINENR]]
|
||||||
|
tst test_fail_raise.py:[LINENR]: I am 'test_suite' / 'test_fail_raise.py:[LINENR]' [test_suite↪test_fail_raise.py:[LINENR]] [test_fail_raise.py:[LINENR]]
|
||||||
|
tst test_fail_raise.py:[LINENR]: ERR: Failure: ('EpicFail', 'This failure is expected') [test_fail_raise.py:[LINENR]: raise Failure('EpicFail', 'This failure is expected')]
|
||||||
|
tst test_fail_raise.py:[LINENR] FAIL (EpicFail) [test_suite↪test_fail_raise.py] [suite.py:[LINENR]]
|
||||||
|
tst test_suite: FAIL [suite.py:[LINENR]]
|
||||||
|
|
||||||
- graceful exit.
|
- graceful exit.
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import os
|
import os
|
||||||
import _prep
|
import _prep
|
||||||
from osmo_gsm_tester import log, suite, config
|
from osmo_gsm_tester import log, suite, config, report
|
||||||
|
|
||||||
config.ENV_CONF = './suite_test'
|
config.ENV_CONF = './suite_test'
|
||||||
|
|
||||||
|
@ -22,13 +22,33 @@ print(config.tostr(s_def.conf))
|
||||||
print('- run hello world test')
|
print('- run hello world test')
|
||||||
s = suite.SuiteRun(None, 'test_suite', s_def)
|
s = suite.SuiteRun(None, 'test_suite', s_def)
|
||||||
results = s.run_tests('hello_world.py')
|
results = s.run_tests('hello_world.py')
|
||||||
print(str(results))
|
print(report.suite_to_text(s))
|
||||||
|
|
||||||
log.style_change(src=True)
|
log.style_change(src=True)
|
||||||
#log.style_change(trace=True)
|
#log.style_change(trace=True)
|
||||||
print('\n- a test with an error')
|
print('\n- a test with an error')
|
||||||
results = s.run_tests('test_error.py')
|
results = s.run_tests('test_error.py')
|
||||||
print(str(results))
|
output = report.suite_to_text(s)
|
||||||
|
assert 'FAIL: [test_suite] 1 failed ' in output
|
||||||
|
assert 'FAIL: [test_error.py]' in output
|
||||||
|
assert "type:'AssertionError' message: AssertionError()" in output
|
||||||
|
assert 'assert False' in output
|
||||||
|
|
||||||
|
print('\n- a test with a failure')
|
||||||
|
results = s.run_tests('test_fail.py')
|
||||||
|
output = report.suite_to_text(s)
|
||||||
|
assert 'FAIL: [test_suite] 1 failed ' in output
|
||||||
|
assert 'FAIL: [test_fail.py]' in output
|
||||||
|
assert "type:'EpicFail' message: This failure is expected" in output
|
||||||
|
assert "test.set_fail('EpicFail', 'This failure is expected')" in output
|
||||||
|
|
||||||
|
print('\n- a test with a raised failure')
|
||||||
|
results = s.run_tests('test_fail_raise.py')
|
||||||
|
output = report.suite_to_text(s)
|
||||||
|
assert 'FAIL: [test_suite] 1 failed ' in output
|
||||||
|
assert 'FAIL: [test_fail_raise.py]' in output
|
||||||
|
assert "type:'EpicFail' message: This failure is expected" in output
|
||||||
|
assert "raise Failure('EpicFail', 'This failure is expected')" in output
|
||||||
|
|
||||||
print('\n- graceful exit.')
|
print('\n- graceful exit.')
|
||||||
# vim: expandtab tabstop=4 shiftwidth=4
|
# vim: expandtab tabstop=4 shiftwidth=4
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from osmo_gsm_tester.test import *
|
||||||
|
|
||||||
|
print('I am %r / %r' % (suite.name(), test.name()))
|
||||||
|
|
||||||
|
test.set_fail('EpicFail', 'This failure is expected')
|
|
@ -0,0 +1,6 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from osmo_gsm_tester.test import *
|
||||||
|
|
||||||
|
print('I am %r / %r' % (suite.name(), test.name()))
|
||||||
|
|
||||||
|
raise Failure('EpicFail', 'This failure is expected')
|
|
@ -4,7 +4,7 @@
|
||||||
[TMP]/third
|
[TMP]/third
|
||||||
- fetch trial dirs in order
|
- fetch trial dirs in order
|
||||||
first
|
first
|
||||||
['taken']
|
['last_run', 'run.[TIMESTAMP]', 'taken']
|
||||||
second
|
second
|
||||||
third
|
third
|
||||||
- no more trial dirs left
|
- no more trial dirs left
|
||||||
|
|
|
@ -169,44 +169,32 @@ optional.''')
|
||||||
t.verify()
|
t.verify()
|
||||||
trials.append(t)
|
trials.append(t)
|
||||||
|
|
||||||
trials_passed = []
|
trials_run = []
|
||||||
trials_failed = []
|
any_failed = False
|
||||||
|
|
||||||
for current_trial in trials:
|
for current_trial in trials:
|
||||||
try:
|
try:
|
||||||
with current_trial:
|
with current_trial:
|
||||||
suites_passed = []
|
|
||||||
suites_failed = []
|
|
||||||
for suite_scenario_str, suite_def, scenarios in suite_scenarios:
|
for suite_scenario_str, suite_def, scenarios in suite_scenarios:
|
||||||
log.large_separator(current_trial.name(), suite_scenario_str)
|
log.large_separator(current_trial.name(), suite_scenario_str)
|
||||||
suite_run = suite.SuiteRun(current_trial, suite_scenario_str, suite_def, scenarios)
|
suite_run = suite.SuiteRun(current_trial, suite_scenario_str, suite_def, scenarios)
|
||||||
result = suite_run.run_tests(test_names)
|
current_trial.add_suite(suite_run)
|
||||||
if result.all_passed:
|
|
||||||
suites_passed.append(suite_scenario_str)
|
status = current_trial.run_suites(test_names)
|
||||||
suite_run.log('PASS')
|
if status == trial.Trial.FAIL:
|
||||||
else:
|
any_failed = True
|
||||||
suites_failed.append(suite_scenario_str)
|
trials_run.append(current_trial)
|
||||||
suite_run.err('FAIL')
|
|
||||||
if not suites_failed:
|
|
||||||
current_trial.log('PASS')
|
|
||||||
trials_passed.append(current_trial.name())
|
|
||||||
else:
|
|
||||||
current_trial.err('FAIL')
|
|
||||||
trials_failed.append((current_trial.name(), suites_passed, suites_failed))
|
|
||||||
except:
|
except:
|
||||||
current_trial.log_exn()
|
current_trial.log_exn()
|
||||||
|
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
log.large_separator()
|
if not any_failed:
|
||||||
if trials_passed:
|
log.large_separator('All trials passed:\n ' + ('\n '.join(mytrial.name() for mytrial in trials_run)))
|
||||||
print('Trials passed:\n ' + ('\n '.join(trials_passed)))
|
else:
|
||||||
if trials_failed:
|
for mytrial in trials_run:
|
||||||
print('Trials failed:')
|
log.large_separator('Trial Report for %s' % mytrial.name())
|
||||||
for trial_name, suites_passed, suites_failed in trials_failed:
|
mytrial.log_report()
|
||||||
print(' %s (%d of %d suite runs failed)' % (trial_name, len(suites_failed), len(suites_failed) + len(suites_passed)))
|
|
||||||
for suite_failed in suites_failed:
|
|
||||||
print(' FAIL:', suite_failed)
|
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
@ -0,0 +1,84 @@
|
||||||
|
# osmo_gsm_tester: report: directory of binaries to be tested
|
||||||
|
#
|
||||||
|
# Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH
|
||||||
|
#
|
||||||
|
# Author: Pau Espin Pedrol <pespin@sysmocom.de>
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as
|
||||||
|
# published by the Free Software Foundation, either version 3 of the
|
||||||
|
# License, or (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
import os
|
||||||
|
import math
|
||||||
|
from datetime import datetime
|
||||||
|
import xml.etree.ElementTree as et
|
||||||
|
from . import log, suite
|
||||||
|
|
||||||
|
def trial_to_junit_write(trial, junit_path):
|
||||||
|
elements = et.ElementTree(element=trial_to_junit(trial))
|
||||||
|
elements.write(junit_path)
|
||||||
|
|
||||||
|
def trial_to_junit(trial):
|
||||||
|
testsuites = et.Element('testsuites')
|
||||||
|
for suite in trial.suites:
|
||||||
|
testsuite = suite_to_junit(suite)
|
||||||
|
testsuites.append(testsuite)
|
||||||
|
return testsuites
|
||||||
|
|
||||||
|
def suite_to_junit(suite):
|
||||||
|
testsuite = et.Element('testsuite')
|
||||||
|
testsuite.set('name', suite.name())
|
||||||
|
testsuite.set('hostname', 'localhost')
|
||||||
|
testsuite.set('timestamp', datetime.fromtimestamp(round(suite.start_timestamp)).isoformat())
|
||||||
|
testsuite.set('time', str(math.ceil(suite.duration)))
|
||||||
|
testsuite.set('tests', str(len(suite.tests)))
|
||||||
|
testsuite.set('failures', str(suite.test_failed_ctr))
|
||||||
|
for test in suite.tests:
|
||||||
|
testcase = test_to_junit(test)
|
||||||
|
testsuite.append(testcase)
|
||||||
|
return testsuite
|
||||||
|
|
||||||
|
def test_to_junit(test):
|
||||||
|
testcase = et.Element('testcase')
|
||||||
|
testcase.set('name', test.name())
|
||||||
|
testcase.set('time', str(math.ceil(test.duration)))
|
||||||
|
if test.status == suite.Test.SKIP:
|
||||||
|
skip = et.SubElement(testcase, 'skipped')
|
||||||
|
elif test.status == suite.Test.FAIL:
|
||||||
|
failure = et.SubElement(testcase, 'failure')
|
||||||
|
failure.set('type', test.fail_type)
|
||||||
|
failure.text = test.fail_message
|
||||||
|
return testcase
|
||||||
|
|
||||||
|
def trial_to_text(trial):
|
||||||
|
msg = '\n%s [%s]\n ' % (trial.status, trial.name())
|
||||||
|
msg += '\n '.join(suite_to_text(result) for result in trial.suites)
|
||||||
|
return msg
|
||||||
|
|
||||||
|
def suite_to_text(suite):
|
||||||
|
if suite.test_failed_ctr:
|
||||||
|
return 'FAIL: [%s] %d failed out of %d tests run (%d skipped):\n %s' % (
|
||||||
|
suite.name(), suite.test_failed_ctr, len(suite.tests), suite.test_skipped_ctr,
|
||||||
|
'\n '.join([test_to_text(t) for t in suite.tests]))
|
||||||
|
if not suite.tests:
|
||||||
|
return 'no tests were run.'
|
||||||
|
return 'pass: all %d tests passed (%d skipped).' % (len(suite.tests), suite.test_skipped_ctr)
|
||||||
|
|
||||||
|
def test_to_text(test):
|
||||||
|
ret = "%s: [%s]" % (test.status, test.name())
|
||||||
|
if test.status != suite.Test.SKIP:
|
||||||
|
ret += " (%s, %d sec)" % (datetime.fromtimestamp(round(test.start_timestamp)).isoformat(), test.duration)
|
||||||
|
if test.status == suite.Test.FAIL:
|
||||||
|
ret += " type:'%s' message: %s" % (test.fail_type, test.fail_message.replace('\n', '\n '))
|
||||||
|
return ret
|
||||||
|
|
||||||
|
# vim: expandtab tabstop=4 shiftwidth=4
|
|
@ -21,12 +21,23 @@ import os
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import copy
|
import copy
|
||||||
|
import traceback
|
||||||
from . import config, log, template, util, resource, schema, ofono_client, osmo_nitb
|
from . import config, log, template, util, resource, schema, ofono_client, osmo_nitb
|
||||||
from . import test
|
from . import test
|
||||||
|
|
||||||
class Timeout(Exception):
|
class Timeout(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
class Failure(Exception):
|
||||||
|
'''Test failure exception, provided to be raised by tests. fail_type is
|
||||||
|
usually a keyword used to quickly identify the type of failure that
|
||||||
|
occurred. fail_msg is a more extensive text containing information about
|
||||||
|
the issue.'''
|
||||||
|
|
||||||
|
def __init__(self, fail_type, fail_msg):
|
||||||
|
self.fail_type = fail_type
|
||||||
|
self.fail_msg = fail_msg
|
||||||
|
|
||||||
class SuiteDefinition(log.Origin):
|
class SuiteDefinition(log.Origin):
|
||||||
'''A test suite reserves resources for a number of tests.
|
'''A test suite reserves resources for a number of tests.
|
||||||
Each test requires a specific number of modems, BTSs etc., which are
|
Each test requires a specific number of modems, BTSs etc., which are
|
||||||
|
@ -78,9 +89,11 @@ class SuiteDefinition(log.Origin):
|
||||||
raise ValueError('add_test(): test already belongs to another suite')
|
raise ValueError('add_test(): test already belongs to another suite')
|
||||||
self.tests.append(test)
|
self.tests.append(test)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class Test(log.Origin):
|
class Test(log.Origin):
|
||||||
|
UNKNOWN = 'UNKNOWN'
|
||||||
|
SKIP = 'SKIP'
|
||||||
|
PASS = 'PASS'
|
||||||
|
FAIL = 'FAIL'
|
||||||
|
|
||||||
def __init__(self, suite, test_basename):
|
def __init__(self, suite, test_basename):
|
||||||
self.suite = suite
|
self.suite = suite
|
||||||
|
@ -89,26 +102,43 @@ class Test(log.Origin):
|
||||||
super().__init__(self.path)
|
super().__init__(self.path)
|
||||||
self.set_name(self.basename)
|
self.set_name(self.basename)
|
||||||
self.set_log_category(log.C_TST)
|
self.set_log_category(log.C_TST)
|
||||||
|
self.status = Test.UNKNOWN
|
||||||
|
self.start_timestamp = 0
|
||||||
|
self.duration = 0
|
||||||
|
self.fail_type = None
|
||||||
|
self.fail_message = None
|
||||||
|
|
||||||
def run(self, suite_run):
|
def run(self, suite_run):
|
||||||
assert self.suite is suite_run.definition
|
assert self.suite is suite_run.definition
|
||||||
with self:
|
try:
|
||||||
test.setup(suite_run, self, ofono_client, sys.modules[__name__])
|
with self:
|
||||||
success = False
|
self.status = Test.UNKNOWN
|
||||||
try:
|
self.start_timestamp = time.time()
|
||||||
|
test.setup(suite_run, self, ofono_client, sys.modules[__name__])
|
||||||
self.log('START')
|
self.log('START')
|
||||||
with self.redirect_stdout():
|
with self.redirect_stdout():
|
||||||
util.run_python_file('%s.%s' % (self.suite.name(), self.name()),
|
util.run_python_file('%s.%s' % (self.suite.name(), self.name()),
|
||||||
self.path)
|
self.path)
|
||||||
success = True
|
if self.status == Test.UNKNOWN:
|
||||||
except resource.NoResourceExn:
|
self.set_pass()
|
||||||
self.err('Current resource state:\n', repr(suite_run.reserved_resources))
|
except Exception as e:
|
||||||
raise
|
self.log_exn()
|
||||||
finally:
|
if isinstance(e, Failure):
|
||||||
if success:
|
ftype = e.fail_type
|
||||||
self.log('PASS')
|
fmsg = e.fail_msg + '\n' + traceback.format_exc().rstrip()
|
||||||
else:
|
else:
|
||||||
self.log('FAIL')
|
ftype = type(e).__name__
|
||||||
|
fmsg = repr(e) + '\n' + traceback.format_exc().rstrip()
|
||||||
|
if isinstance(e, resource.NoResourceExn):
|
||||||
|
msg += '\n' + 'Current resource state:\n' + repr(suite_run.reserved_resources)
|
||||||
|
self.set_fail(ftype, fmsg, False)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
if self.status == Test.PASS or self.status == Test.SKIP:
|
||||||
|
self.log(self.status)
|
||||||
|
else:
|
||||||
|
self.log('%s (%s)' % (self.status, self.fail_type))
|
||||||
|
return self.status
|
||||||
|
|
||||||
def name(self):
|
def name(self):
|
||||||
l = log.get_line_for_src(self.path)
|
l = log.get_line_for_src(self.path)
|
||||||
|
@ -116,7 +146,26 @@ class Test(log.Origin):
|
||||||
return '%s:%s' % (self._name, l)
|
return '%s:%s' % (self._name, l)
|
||||||
return super().name()
|
return super().name()
|
||||||
|
|
||||||
|
def set_fail(self, fail_type, fail_message, tb=True):
|
||||||
|
self.status = Test.FAIL
|
||||||
|
self.duration = time.time() - self.start_timestamp
|
||||||
|
self.fail_type = fail_type
|
||||||
|
self.fail_message = fail_message
|
||||||
|
if tb:
|
||||||
|
self.fail_message += '\n' + ''.join(traceback.format_stack()[:-1]).rstrip()
|
||||||
|
|
||||||
|
def set_pass(self):
|
||||||
|
self.status = Test.PASS
|
||||||
|
self.duration = time.time() - self.start_timestamp
|
||||||
|
|
||||||
|
def set_skip(self):
|
||||||
|
self.status = Test.SKIP
|
||||||
|
self.duration = 0
|
||||||
|
|
||||||
class SuiteRun(log.Origin):
|
class SuiteRun(log.Origin):
|
||||||
|
UNKNOWN = 'UNKNOWN'
|
||||||
|
PASS = 'PASS'
|
||||||
|
FAIL = 'FAIL'
|
||||||
|
|
||||||
trial = None
|
trial = None
|
||||||
resources_pool = None
|
resources_pool = None
|
||||||
|
@ -133,6 +182,14 @@ class SuiteRun(log.Origin):
|
||||||
self.set_log_category(log.C_TST)
|
self.set_log_category(log.C_TST)
|
||||||
self.resources_pool = resource.ResourcesPool()
|
self.resources_pool = resource.ResourcesPool()
|
||||||
|
|
||||||
|
def mark_start(self):
|
||||||
|
self.tests = []
|
||||||
|
self.start_timestamp = time.time()
|
||||||
|
self.duration = 0
|
||||||
|
self.test_failed_ctr = 0
|
||||||
|
self.test_skipped_ctr = 0
|
||||||
|
self.status = SuiteRun.UNKNOWN
|
||||||
|
|
||||||
def combined(self, conf_name):
|
def combined(self, conf_name):
|
||||||
self.dbg(combining=conf_name)
|
self.dbg(combining=conf_name)
|
||||||
with log.Origin(combining_scenarios=conf_name):
|
with log.Origin(combining_scenarios=conf_name):
|
||||||
|
@ -157,32 +214,6 @@ class SuiteRun(log.Origin):
|
||||||
self._config = self.combined('config')
|
self._config = self.combined('config')
|
||||||
return self._config
|
return self._config
|
||||||
|
|
||||||
class Results:
|
|
||||||
def __init__(self):
|
|
||||||
self.passed = []
|
|
||||||
self.failed = []
|
|
||||||
self.all_passed = None
|
|
||||||
|
|
||||||
def add_pass(self, test):
|
|
||||||
self.passed.append(test)
|
|
||||||
|
|
||||||
def add_fail(self, test):
|
|
||||||
self.failed.append(test)
|
|
||||||
|
|
||||||
def conclude(self):
|
|
||||||
self.all_passed = bool(self.passed) and not bool(self.failed)
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __str__(self):
|
|
||||||
if self.failed:
|
|
||||||
return 'FAIL: %d of %d tests failed:\n %s' % (
|
|
||||||
len(self.failed),
|
|
||||||
len(self.failed) + len(self.passed),
|
|
||||||
'\n '.join([t.name() for t in self.failed]))
|
|
||||||
if not self.passed:
|
|
||||||
return 'no tests were run.'
|
|
||||||
return 'pass: all %d tests passed.' % len(self.passed)
|
|
||||||
|
|
||||||
def reserve_resources(self):
|
def reserve_resources(self):
|
||||||
if self.reserved_resources:
|
if self.reserved_resources:
|
||||||
raise RuntimeError('Attempt to reserve resources twice for a SuiteRun')
|
raise RuntimeError('Attempt to reserve resources twice for a SuiteRun')
|
||||||
|
@ -192,24 +223,28 @@ class SuiteRun(log.Origin):
|
||||||
|
|
||||||
def run_tests(self, names=None):
|
def run_tests(self, names=None):
|
||||||
self.log('Suite run start')
|
self.log('Suite run start')
|
||||||
|
self.mark_start()
|
||||||
if not self.reserved_resources:
|
if not self.reserved_resources:
|
||||||
self.reserve_resources()
|
self.reserve_resources()
|
||||||
results = SuiteRun.Results()
|
|
||||||
for test in self.definition.tests:
|
for test in self.definition.tests:
|
||||||
if names and not test.name() in names:
|
if names and not test.name() in names:
|
||||||
|
test.set_skip()
|
||||||
|
self.test_skipped_ctr += 1
|
||||||
|
self.tests.append(test)
|
||||||
continue
|
continue
|
||||||
self._run_test(test, results)
|
|
||||||
self.stop_processes()
|
|
||||||
return results.conclude()
|
|
||||||
|
|
||||||
def _run_test(self, test, results):
|
|
||||||
try:
|
|
||||||
with self:
|
with self:
|
||||||
test.run(self)
|
st = test.run(self)
|
||||||
results.add_pass(test)
|
if st == Test.FAIL:
|
||||||
except:
|
self.test_failed_ctr += 1
|
||||||
results.add_fail(test)
|
self.tests.append(test)
|
||||||
self.log_exn()
|
self.stop_processes()
|
||||||
|
self.duration = time.time() - self.start_timestamp
|
||||||
|
if self.test_failed_ctr:
|
||||||
|
self.status = SuiteRun.FAIL
|
||||||
|
else:
|
||||||
|
self.status = SuiteRun.PASS
|
||||||
|
self.log(self.status)
|
||||||
|
return self.status
|
||||||
|
|
||||||
def remember_to_stop(self, process):
|
def remember_to_stop(self, process):
|
||||||
if self._processes is None:
|
if self._processes is None:
|
||||||
|
|
|
@ -32,9 +32,10 @@ sleep = None
|
||||||
poll = None
|
poll = None
|
||||||
prompt = None
|
prompt = None
|
||||||
Timeout = None
|
Timeout = None
|
||||||
|
Failure = None
|
||||||
|
|
||||||
def setup(suite_run, _test, ofono_client, suite_module):
|
def setup(suite_run, _test, ofono_client, suite_module):
|
||||||
global trial, suite, test, resources, log, dbg, err, wait, sleep, poll, prompt, Timeout
|
global trial, suite, test, resources, log, dbg, err, wait, sleep, poll, prompt, Failure, Timeout
|
||||||
trial = suite_run.trial
|
trial = suite_run.trial
|
||||||
suite = suite_run
|
suite = suite_run
|
||||||
test = _test
|
test = _test
|
||||||
|
@ -46,6 +47,7 @@ def setup(suite_run, _test, ofono_client, suite_module):
|
||||||
sleep = suite_run.sleep
|
sleep = suite_run.sleep
|
||||||
poll = suite_run.poll
|
poll = suite_run.poll
|
||||||
prompt = suite_run.prompt
|
prompt = suite_run.prompt
|
||||||
|
Failure = suite_module.Failure
|
||||||
Timeout = suite_module.Timeout
|
Timeout = suite_module.Timeout
|
||||||
|
|
||||||
# vim: expandtab tabstop=4 shiftwidth=4
|
# vim: expandtab tabstop=4 shiftwidth=4
|
||||||
|
|
|
@ -22,7 +22,7 @@ import time
|
||||||
import shutil
|
import shutil
|
||||||
import tarfile
|
import tarfile
|
||||||
|
|
||||||
from . import log, util
|
from . import log, util, suite, report
|
||||||
|
|
||||||
FILE_MARK_TAKEN = 'taken'
|
FILE_MARK_TAKEN = 'taken'
|
||||||
FILE_CHECKSUMS = 'checksums.md5'
|
FILE_CHECKSUMS = 'checksums.md5'
|
||||||
|
@ -32,6 +32,10 @@ FILE_LOG = 'log'
|
||||||
FILE_LOG_BRIEF = 'log_brief'
|
FILE_LOG_BRIEF = 'log_brief'
|
||||||
|
|
||||||
class Trial(log.Origin):
|
class Trial(log.Origin):
|
||||||
|
UNKNOWN = 'UNKNOWN'
|
||||||
|
PASS = 'PASS'
|
||||||
|
FAIL = 'FAIL'
|
||||||
|
|
||||||
path = None
|
path = None
|
||||||
dir = None
|
dir = None
|
||||||
_run_dir = None
|
_run_dir = None
|
||||||
|
@ -58,6 +62,9 @@ class Trial(log.Origin):
|
||||||
self.dir = util.Dir(self.path)
|
self.dir = util.Dir(self.path)
|
||||||
self.inst_dir = util.Dir(self.dir.child('inst'))
|
self.inst_dir = util.Dir(self.dir.child('inst'))
|
||||||
self.bin_tars = []
|
self.bin_tars = []
|
||||||
|
self.suites = []
|
||||||
|
self.junit_path = self.get_run_dir().new_file(self.name()+'.xml')
|
||||||
|
self.status = Trial.UNKNOWN
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
return self.name()
|
return self.name()
|
||||||
|
@ -176,4 +183,24 @@ class Trial(log.Origin):
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def add_suite(self, suite_run):
|
||||||
|
self.suites.append(suite_run)
|
||||||
|
|
||||||
|
def run_suites(self, names=None):
|
||||||
|
self.status = Trial.UNKNOWN
|
||||||
|
for suite_run in self.suites:
|
||||||
|
st = suite_run.run_tests(names)
|
||||||
|
if st == suite.SuiteRun.FAIL:
|
||||||
|
self.status = Trial.FAIL
|
||||||
|
elif self.status == Trial.UNKNOWN:
|
||||||
|
self.status = Trial.PASS
|
||||||
|
self.log(self.status)
|
||||||
|
junit_path = self.get_run_dir().new_file(self.name()+'.xml')
|
||||||
|
self.log('Storing JUnit report in', junit_path)
|
||||||
|
report.trial_to_junit_write(self, junit_path)
|
||||||
|
return self.status
|
||||||
|
|
||||||
|
def log_report(self):
|
||||||
|
self.log(report.trial_to_text(self))
|
||||||
|
|
||||||
# vim: expandtab tabstop=4 shiftwidth=4
|
# vim: expandtab tabstop=4 shiftwidth=4
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from osmo_gsm_tester.test import *
|
||||||
|
|
||||||
|
# This can be used to verify that a test error is reported properly.
|
||||||
|
assert False
|
|
@ -2,4 +2,4 @@
|
||||||
from osmo_gsm_tester.test import *
|
from osmo_gsm_tester.test import *
|
||||||
|
|
||||||
# This can be used to verify that a test failure is reported properly.
|
# This can be used to verify that a test failure is reported properly.
|
||||||
assert False
|
test.set_fail('EpicFail', 'This failure is expected')
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from osmo_gsm_tester.test import *
|
||||||
|
|
||||||
|
# This can be used to verify that a test failure is reported properly.
|
||||||
|
raise Failure('EpicFail', 'This failure is expected')
|
|
@ -0,0 +1,5 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
from osmo_gsm_tester.test import *
|
||||||
|
|
||||||
|
# This can be used to verify that a test passes correctly.
|
||||||
|
pass
|
Loading…
Reference in New Issue