2017-11-09 13:26:35 +00:00
|
|
|
# osmo_gsm_tester: test class
|
|
|
|
#
|
|
|
|
# Copyright (C) 2017 by sysmocom - s.f.m.c. GmbH
|
|
|
|
#
|
|
|
|
# Author: Pau Espin Pedrol <pespin@sysmocom.de>
|
|
|
|
#
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License as
|
|
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
|
|
# License, or (at your option) any later version.
|
|
|
|
#
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
# GNU General Public License for more details.
|
|
|
|
#
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
import os
|
|
|
|
import sys
|
|
|
|
import time
|
|
|
|
import traceback
|
|
|
|
|
2020-05-04 18:21:31 +00:00
|
|
|
from . import log
|
|
|
|
from . import util
|
|
|
|
from . import resource
|
|
|
|
from .event_loop import MainLoop
|
|
|
|
|
|
|
|
from .. import testenv
|
2017-11-09 13:26:35 +00:00
|
|
|
|
|
|
|
class Test(log.Origin):
|
2020-03-05 16:22:40 +00:00
|
|
|
UNKNOWN = 'UNKNOWN' # matches junit 'error'
|
2017-11-09 13:26:35 +00:00
|
|
|
SKIP = 'skip'
|
|
|
|
PASS = 'pass'
|
|
|
|
FAIL = 'FAIL'
|
|
|
|
|
2020-06-12 15:54:55 +00:00
|
|
|
def __init__(self, suite_run, test_basename, config_test_specific):
|
2017-11-09 13:26:35 +00:00
|
|
|
self.basename = test_basename
|
|
|
|
super().__init__(log.C_TST, self.basename)
|
2018-08-09 11:45:55 +00:00
|
|
|
self._run_dir = None
|
2017-11-09 13:26:35 +00:00
|
|
|
self.suite_run = suite_run
|
2020-06-12 15:54:55 +00:00
|
|
|
self._config_test_specific = config_test_specific
|
2017-11-09 13:26:35 +00:00
|
|
|
self.path = os.path.join(self.suite_run.definition.suite_dir, self.basename)
|
|
|
|
self.status = Test.UNKNOWN
|
|
|
|
self.start_timestamp = 0
|
|
|
|
self.duration = 0
|
|
|
|
self.fail_type = None
|
|
|
|
self.fail_message = None
|
2020-06-11 15:57:43 +00:00
|
|
|
self.log_targets = []
|
2020-03-04 15:14:31 +00:00
|
|
|
self._report_stdout = None
|
2020-06-15 12:27:50 +00:00
|
|
|
self._kpis = None
|
2020-06-12 15:54:55 +00:00
|
|
|
self.timeout = int(config_test_specific['timeout']) if 'timeout' in config_test_specific else None
|
2017-11-09 13:26:35 +00:00
|
|
|
|
2020-05-06 19:11:02 +00:00
|
|
|
def module_name(self):
|
|
|
|
'Return test name without trailing .py'
|
|
|
|
assert self.basename.endswith('.py')
|
|
|
|
return self.basename[:-3]
|
|
|
|
|
2017-11-09 13:26:35 +00:00
|
|
|
def get_run_dir(self):
|
|
|
|
if self._run_dir is None:
|
|
|
|
self._run_dir = util.Dir(self.suite_run.get_run_dir().new_dir(self._name))
|
|
|
|
return self._run_dir
|
|
|
|
|
|
|
|
def run(self):
|
2020-05-04 18:21:31 +00:00
|
|
|
testenv_obj = None
|
2017-11-09 13:26:35 +00:00
|
|
|
try:
|
2020-06-11 15:57:43 +00:00
|
|
|
self.log_targets = [log.FileLogTarget(self.get_run_dir().new_child(log.FILE_LOG)).set_all_levels(log.L_DBG).style_change(trace=True),
|
|
|
|
log.FileLogTarget(self.get_run_dir().new_child(log.FILE_LOG_BRIEF)).style_change(src=False, all_origins_on_levels=(log.L_ERR, log.L_TRACEBACK))]
|
2020-05-05 10:54:37 +00:00
|
|
|
log.large_separator(self.suite_run.trial().name(), self.suite_run.name(), self.name(), sublevel=3)
|
2017-11-09 13:26:35 +00:00
|
|
|
self.status = Test.UNKNOWN
|
|
|
|
self.start_timestamp = time.time()
|
2020-05-04 18:21:31 +00:00
|
|
|
testenv_obj = testenv.setup(self.suite_run, self)
|
2017-11-09 13:26:35 +00:00
|
|
|
with self.redirect_stdout():
|
|
|
|
util.run_python_file('%s.%s' % (self.suite_run.definition.name(), self.basename),
|
|
|
|
self.path)
|
|
|
|
if self.status == Test.UNKNOWN:
|
|
|
|
self.set_pass()
|
|
|
|
except Exception as e:
|
|
|
|
if hasattr(e, 'msg'):
|
|
|
|
msg = e.msg
|
|
|
|
else:
|
|
|
|
msg = str(e)
|
|
|
|
if isinstance(e, AssertionError):
|
|
|
|
# AssertionError lacks further information on what was
|
|
|
|
# asserted. Find the line where the code asserted:
|
|
|
|
msg += log.get_src_from_exc_info(sys.exc_info())
|
|
|
|
# add source file information to failure report
|
|
|
|
if hasattr(e, 'origins'):
|
|
|
|
msg += ' [%s]' % e.origins
|
|
|
|
tb_str = traceback.format_exc()
|
|
|
|
if isinstance(e, resource.NoResourceExn):
|
|
|
|
tb_str += self.suite_run.resource_status_str()
|
|
|
|
self.set_fail(type(e).__name__, msg, tb_str, log.get_src_from_exc_info())
|
|
|
|
except BaseException as e:
|
|
|
|
# when the program is aborted by a signal (like Ctrl-C), escalate to abort all.
|
|
|
|
self.err('TEST RUN ABORTED: %s' % type(e).__name__)
|
|
|
|
raise
|
2019-09-18 14:50:38 +00:00
|
|
|
finally:
|
2020-05-04 18:21:31 +00:00
|
|
|
if testenv_obj:
|
|
|
|
testenv_obj.stop()
|
2020-06-11 15:57:43 +00:00
|
|
|
for log_tgt in self.log_targets:
|
|
|
|
log_tgt.remove()
|
2017-11-09 13:26:35 +00:00
|
|
|
|
fix: line nr in test name in wrong places
test.Test() overrides name() in order to provide source line number
information. However, overriding name() is the wrong place for that, as
name() is also often used for identifying an object - when listing the
tests of a suite, the line number should not appear in the test name.
For example, the line number sometimes ends up in the test results in
jenkins, making 'foo.py' and 'foo.py:23' two distinct report items.
Instead, add a separate function Origin.src() that defaults to name(),
but specific classes can override src() if they wish to provide more
detailed information with the object name.
Override src() in Test, not name().
Use src() in backtraces.
The suite_test.ok shows that the backtracing in the log remains
unchanged, but the place where the test name is printed is corrected:
I am 'test_suite' / 'hello_world.py:23'
becomes
I am 'test_suite' / 'hello_world.py'
(Notice that "[LINENR]" in suite_test.ok is a masking of an actual
number, done within the selftest suite)
Change-Id: I0c4698fa2b3db3de777d8b6dcdcee84e433c62b7
2020-12-04 16:25:23 +00:00
|
|
|
def src(self):
|
2017-11-09 13:26:35 +00:00
|
|
|
l = log.get_line_for_src(self.path)
|
|
|
|
if l is not None:
|
fix: line nr in test name in wrong places
test.Test() overrides name() in order to provide source line number
information. However, overriding name() is the wrong place for that, as
name() is also often used for identifying an object - when listing the
tests of a suite, the line number should not appear in the test name.
For example, the line number sometimes ends up in the test results in
jenkins, making 'foo.py' and 'foo.py:23' two distinct report items.
Instead, add a separate function Origin.src() that defaults to name(),
but specific classes can override src() if they wish to provide more
detailed information with the object name.
Override src() in Test, not name().
Use src() in backtraces.
The suite_test.ok shows that the backtracing in the log remains
unchanged, but the place where the test name is printed is corrected:
I am 'test_suite' / 'hello_world.py:23'
becomes
I am 'test_suite' / 'hello_world.py'
(Notice that "[LINENR]" in suite_test.ok is a masking of an actual
number, done within the selftest suite)
Change-Id: I0c4698fa2b3db3de777d8b6dcdcee84e433c62b7
2020-12-04 16:25:23 +00:00
|
|
|
return '%s:%s' % (self.name(), l)
|
|
|
|
return self.name()
|
2017-11-09 13:26:35 +00:00
|
|
|
|
2020-06-12 14:38:37 +00:00
|
|
|
def elapsed_time(self):
|
|
|
|
'time elapsed since test was started'
|
|
|
|
return time.time() - self.start_timestamp
|
|
|
|
|
2017-11-09 13:26:35 +00:00
|
|
|
def set_fail(self, fail_type, fail_message, tb_str=None, src=4):
|
|
|
|
self.status = Test.FAIL
|
2020-06-12 14:38:37 +00:00
|
|
|
self.duration = self.elapsed_time()
|
2017-11-09 13:26:35 +00:00
|
|
|
self.fail_type = fail_type
|
|
|
|
self.fail_message = fail_message
|
|
|
|
|
|
|
|
if tb_str is None:
|
|
|
|
# populate an exception-less call to set_fail() with traceback info
|
|
|
|
tb_str = ''.join(traceback.format_stack()[:-1])
|
|
|
|
|
|
|
|
self.fail_tb = tb_str
|
|
|
|
self.err('%s: %s' % (self.fail_type, self.fail_message), _src=src)
|
|
|
|
if self.fail_tb:
|
|
|
|
self.log(self.fail_tb, _level=log.L_TRACEBACK)
|
|
|
|
self.log('Test FAILED (%.1f sec)' % self.duration)
|
|
|
|
|
|
|
|
def set_pass(self):
|
|
|
|
self.status = Test.PASS
|
2020-06-12 14:38:37 +00:00
|
|
|
self.duration = self.elapsed_time()
|
2017-11-09 13:26:35 +00:00
|
|
|
self.log('Test passed (%.1f sec)' % self.duration)
|
|
|
|
|
|
|
|
def set_skip(self):
|
|
|
|
self.status = Test.SKIP
|
|
|
|
self.duration = 0
|
|
|
|
|
2020-06-12 15:13:26 +00:00
|
|
|
def config_test_specific(self):
|
|
|
|
return self._config_test_specific
|
|
|
|
|
2020-06-15 12:27:50 +00:00
|
|
|
def set_kpis(self, kpis):
|
|
|
|
if not isinstance(kpis, dict):
|
|
|
|
raise log.Error('Expected dictionary in toplevel kpis')
|
2020-06-22 20:49:24 +00:00
|
|
|
if isinstance(self._kpis, dict):
|
|
|
|
self._kpis.update(kpis)
|
|
|
|
else:
|
|
|
|
self._kpis = kpis
|
2020-06-15 12:27:50 +00:00
|
|
|
|
|
|
|
def kpis(self):
|
|
|
|
return self._kpis
|
|
|
|
|
2020-03-04 15:14:31 +00:00
|
|
|
def set_report_stdout(self, text):
|
|
|
|
'Overwrite stdout text stored in report from inside a test'
|
|
|
|
self._report_stdout = text
|
|
|
|
|
|
|
|
def report_stdout(self):
|
|
|
|
# If test overwrote the text, provide it:
|
|
|
|
if self._report_stdout is not None:
|
|
|
|
return self._report_stdout
|
2020-06-11 15:57:43 +00:00
|
|
|
# Otherwise vy default provide the entire test brief log:
|
|
|
|
if len(self.log_targets) == 2 and self.log_targets[1].log_file_path() is not None:
|
|
|
|
with open(self.log_targets[1].log_file_path(), 'r') as myfile:
|
2020-03-04 15:14:31 +00:00
|
|
|
return myfile.read()
|
|
|
|
else:
|
|
|
|
return 'test log file not available'
|
2020-02-24 17:19:10 +00:00
|
|
|
|
2017-11-09 13:26:35 +00:00
|
|
|
# vim: expandtab tabstop=4 shiftwidth=4
|