2017-03-28 10:16:58 +00:00
|
|
|
# osmo_gsm_tester: test suite
|
|
|
|
#
|
|
|
|
# Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH
|
|
|
|
#
|
|
|
|
# Author: Neels Hofmeyr <neels@hofmeyr.de>
|
|
|
|
#
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
2017-06-03 07:51:45 +00:00
|
|
|
# it under the terms of the GNU General Public License as
|
2017-03-28 10:16:58 +00:00
|
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
|
|
# License, or (at your option) any later version.
|
|
|
|
#
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
2017-06-03 07:51:45 +00:00
|
|
|
# GNU General Public License for more details.
|
2017-03-28 10:16:58 +00:00
|
|
|
#
|
2017-06-03 07:51:45 +00:00
|
|
|
# You should have received a copy of the GNU General Public License
|
2017-03-28 10:16:58 +00:00
|
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
import os
|
2017-03-28 12:30:28 +00:00
|
|
|
import sys
|
|
|
|
import time
|
2017-05-22 18:02:41 +00:00
|
|
|
import pprint
|
2020-05-04 10:05:05 +00:00
|
|
|
from .core import config, log, util, process, schema
|
2020-04-10 17:51:31 +00:00
|
|
|
from .core.event_loop import MainLoop
|
2020-04-10 18:46:07 +00:00
|
|
|
from .obj import nitb_osmo, hlr_osmo, mgcpgw_osmo, mgw_osmo, msc_osmo, bsc_osmo, stp_osmo, ggsn_osmo, sgsn_osmo, esme, osmocon, ms_driver, iperf3
|
|
|
|
from .obj import run_node
|
2020-04-10 17:51:31 +00:00
|
|
|
from . import resource, test
|
2017-03-28 10:16:58 +00:00
|
|
|
|
2017-05-07 00:15:21 +00:00
|
|
|
class Timeout(Exception):
|
|
|
|
pass
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
class SuiteDefinition(log.Origin):
|
2017-03-28 10:16:58 +00:00
|
|
|
'''A test suite reserves resources for a number of tests.
|
|
|
|
Each test requires a specific number of modems, BTSs etc., which are
|
|
|
|
reserved beforehand by a test suite. This way several test suites can be
|
|
|
|
scheduled dynamically without resource conflicts arising halfway through
|
|
|
|
the tests.'''
|
|
|
|
|
|
|
|
CONF_FILENAME = 'suite.conf'
|
|
|
|
|
|
|
|
def __init__(self, suite_dir):
|
|
|
|
self.suite_dir = suite_dir
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
super().__init__(log.C_CNF, os.path.basename(self.suite_dir))
|
2017-03-28 10:16:58 +00:00
|
|
|
self.read_conf()
|
|
|
|
|
|
|
|
def read_conf(self):
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
self.dbg('reading %s' % SuiteDefinition.CONF_FILENAME)
|
|
|
|
if not os.path.isdir(self.suite_dir):
|
|
|
|
raise RuntimeError('No such directory: %r' % self.suite_dir)
|
|
|
|
self.conf = config.read(os.path.join(self.suite_dir,
|
|
|
|
SuiteDefinition.CONF_FILENAME),
|
2020-05-04 10:05:05 +00:00
|
|
|
schema.get_all_schema())
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
self.load_test_basenames()
|
|
|
|
|
|
|
|
def load_test_basenames(self):
|
|
|
|
self.test_basenames = []
|
|
|
|
for basename in sorted(os.listdir(self.suite_dir)):
|
|
|
|
if not basename.endswith('.py'):
|
|
|
|
continue
|
|
|
|
self.test_basenames.append(basename)
|
2017-03-28 10:16:58 +00:00
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
class SuiteRun(log.Origin):
|
2017-05-15 16:24:35 +00:00
|
|
|
UNKNOWN = 'UNKNOWN'
|
|
|
|
PASS = 'PASS'
|
|
|
|
FAIL = 'FAIL'
|
2017-03-28 12:30:28 +00:00
|
|
|
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
def __init__(self, trial, suite_scenario_str, suite_definition, scenarios=[]):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
super().__init__(log.C_TST, suite_scenario_str)
|
2018-08-09 11:45:55 +00:00
|
|
|
self.start_timestamp = None
|
|
|
|
self.duration = None
|
|
|
|
self.reserved_resources = None
|
|
|
|
self.objects_to_clean_up = None
|
|
|
|
self.test_import_modules_to_clean_up = []
|
|
|
|
self._resource_requirements = None
|
2018-08-21 12:58:29 +00:00
|
|
|
self._resource_modifiers = None
|
2018-08-09 11:45:55 +00:00
|
|
|
self._config = None
|
2019-04-04 15:44:33 +00:00
|
|
|
self._processes = []
|
2018-08-09 11:45:55 +00:00
|
|
|
self._run_dir = None
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
self.trial = trial
|
2017-03-28 12:30:28 +00:00
|
|
|
self.definition = suite_definition
|
|
|
|
self.scenarios = scenarios
|
|
|
|
self.resources_pool = resource.ResourcesPool()
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
self.status = SuiteRun.UNKNOWN
|
|
|
|
self.load_tests()
|
|
|
|
|
|
|
|
def load_tests(self):
|
|
|
|
self.tests = []
|
|
|
|
for test_basename in self.definition.test_basenames:
|
2017-11-09 13:26:35 +00:00
|
|
|
self.tests.append(test.Test(self, test_basename))
|
2017-03-28 12:30:28 +00:00
|
|
|
|
2017-05-29 02:13:58 +00:00
|
|
|
def register_for_cleanup(self, *obj):
|
|
|
|
assert all([hasattr(o, 'cleanup') for o in obj])
|
|
|
|
self.objects_to_clean_up = self.objects_to_clean_up or []
|
|
|
|
self.objects_to_clean_up.extend(obj)
|
|
|
|
|
|
|
|
def objects_cleanup(self):
|
|
|
|
while self.objects_to_clean_up:
|
|
|
|
obj = self.objects_to_clean_up.pop()
|
suite: Don't stop cleanup of objects on cleanup exception
Let's accept that during cleanup time some stuff may not be in a good
state, specially in case of a failure, but let's keep harder to clean
up everything as much as possible.
This should fix an issue in which after a test failure using an esme object,
an smpp connection is kept in a buggy state and never removed/cleaned up, and
makes all tests after it fail too because the conn is triggered at
poll() time:
File "/home/jenkins/workspace/osmo-gsm-tester_run/osmo-gsm-tester/src/osmo_gsm_tester/suite.py", line 260, in run_tests
self.objects_cleanup()
File "/home/jenkins/workspace/osmo-gsm-tester_run/osmo-gsm-tester/src/osmo_gsm_tester/suite.py", line 199, in objects_cleanup
obj.cleanup()
File "/home/jenkins/workspace/osmo-gsm-tester_run/osmo-gsm-tester/src/osmo_gsm_tester/ofono_client.py", line 345, in cleanup
self.dbus.cleanup()
File "/home/jenkins/workspace/osmo-gsm-tester_run/osmo-gsm-tester/src/osmo_gsm_tester/ofono_client.py", line 170, in cleanup
self.set_powered(False)
File "/home/jenkins/workspace/osmo-gsm-tester_run/osmo-gsm-tester/src/osmo_gsm_tester/ofono_client.py", line 304, in set_powered
self.set_bool('Powered', powered)
File "/home/jenkins/workspace/osmo-gsm-tester_run/osmo-gsm-tester/src/osmo_gsm_tester/ofono_client.py", line 295, in set_bool
event_loop.poll()
File "/home/jenkins/workspace/osmo-gsm-tester_run/osmo-gsm-tester/src/osmo_gsm_tester/event_loop.py", line 39, in poll
func()
File "/home/jenkins/workspace/osmo-gsm-tester_run/osmo-gsm-tester/src/osmo_gsm_tester/esme.py", line 78, in poll
self.client.poll()
File "/usr/local/lib/python3.4/dist-packages/smpplib/client.py", line 321, in poll
self.read_once(ignore_error_codes)
File "/usr/local/lib/python3.4/dist-packages/smpplib/client.py", line 279, in read_once
p = self.read_pdu()
File "/usr/local/lib/python3.4/dist-packages/smpplib/client.py", line 206, in read_pdu
raise exceptions.ConnectionError()
smpplib.exceptions.ConnectionError
Change-Id: Ie7ef9284490f12f5cfd76c35b33b57eefab20eb6
2017-07-31 16:19:06 +00:00
|
|
|
try:
|
|
|
|
obj.cleanup()
|
|
|
|
except Exception:
|
|
|
|
log.log_exn()
|
2017-05-29 02:13:58 +00:00
|
|
|
|
2018-05-09 09:24:23 +00:00
|
|
|
def test_import_modules_register_for_cleanup(self, mod):
|
|
|
|
'''
|
|
|
|
Tests are required to call this API for any module loaded from its own
|
|
|
|
lib subdir, because they are loaded in the global namespace. Otherwise
|
|
|
|
later tests importing modules with the same name will re-use an already
|
|
|
|
loaded module.
|
|
|
|
'''
|
|
|
|
if mod not in self.test_import_modules_to_clean_up:
|
|
|
|
self.dbg('registering module %r for cleanup' % mod)
|
|
|
|
self.test_import_modules_to_clean_up.append(mod)
|
|
|
|
|
|
|
|
def test_import_modules_cleanup(self):
|
|
|
|
while self.test_import_modules_to_clean_up:
|
|
|
|
mod = self.test_import_modules_to_clean_up.pop()
|
|
|
|
try:
|
|
|
|
self.dbg('Cleaning up module %r' % mod)
|
|
|
|
del sys.modules[mod.__name__]
|
|
|
|
del mod
|
|
|
|
except Exception:
|
|
|
|
log.log_exn()
|
|
|
|
|
2017-05-15 16:24:35 +00:00
|
|
|
def mark_start(self):
|
|
|
|
self.start_timestamp = time.time()
|
|
|
|
self.duration = 0
|
|
|
|
self.status = SuiteRun.UNKNOWN
|
|
|
|
|
2018-08-27 10:49:35 +00:00
|
|
|
def combined(self, conf_name, replicate_times=True):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg(combining=conf_name)
|
|
|
|
log.ctx(combining_scenarios=conf_name)
|
2018-08-27 10:49:35 +00:00
|
|
|
combination = self.definition.conf.get(conf_name, {})
|
|
|
|
if replicate_times:
|
|
|
|
combination = config.replicate_times(combination)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg(definition_conf=combination)
|
|
|
|
for scenario in self.scenarios:
|
|
|
|
log.ctx(combining_scenarios=conf_name, scenario=scenario.name())
|
2018-08-27 10:49:35 +00:00
|
|
|
c = scenario.get(conf_name, {})
|
|
|
|
if replicate_times:
|
|
|
|
c = config.replicate_times(c)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg(scenario=scenario.name(), conf=c)
|
|
|
|
if c is None:
|
|
|
|
continue
|
2020-05-04 10:05:05 +00:00
|
|
|
schema.combine(combination, c)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
return combination
|
2017-03-28 12:30:28 +00:00
|
|
|
|
2017-06-14 11:27:08 +00:00
|
|
|
def get_run_dir(self):
|
|
|
|
if self._run_dir is None:
|
|
|
|
self._run_dir = util.Dir(self.trial.get_run_dir().new_dir(self.name()))
|
|
|
|
return self._run_dir
|
|
|
|
|
|
|
|
def get_test_run_dir(self):
|
|
|
|
if self.current_test:
|
|
|
|
return self.current_test.get_run_dir()
|
|
|
|
return self.get_run_dir()
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def resource_requirements(self):
|
|
|
|
if self._resource_requirements is None:
|
|
|
|
self._resource_requirements = self.combined('resources')
|
|
|
|
return self._resource_requirements
|
|
|
|
|
2018-08-21 12:58:29 +00:00
|
|
|
def resource_modifiers(self):
|
|
|
|
if self._resource_modifiers is None:
|
|
|
|
self._resource_modifiers = self.combined('modifiers')
|
|
|
|
return self._resource_modifiers
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def config(self):
|
|
|
|
if self._config is None:
|
2018-08-27 10:49:35 +00:00
|
|
|
self._config = self.combined('config', False)
|
2017-03-28 12:30:28 +00:00
|
|
|
return self._config
|
|
|
|
|
|
|
|
def reserve_resources(self):
|
|
|
|
if self.reserved_resources:
|
|
|
|
raise RuntimeError('Attempt to reserve resources twice for a SuiteRun')
|
2017-05-14 01:37:13 +00:00
|
|
|
self.log('reserving resources in', self.resources_pool.state_dir, '...')
|
2018-08-21 12:58:29 +00:00
|
|
|
self.reserved_resources = self.resources_pool.reserve(self, self.resource_requirements(), self.resource_modifiers())
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
def run_tests(self, names=None):
|
2018-05-08 13:28:48 +00:00
|
|
|
suite_libdir = os.path.join(self.definition.suite_dir, 'lib')
|
2017-05-17 12:51:31 +00:00
|
|
|
try:
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.large_separator(self.trial.name(), self.name(), sublevel=2)
|
|
|
|
self.mark_start()
|
2018-05-08 13:28:48 +00:00
|
|
|
util.import_path_prepend(suite_libdir)
|
2018-03-28 17:17:34 +00:00
|
|
|
MainLoop.register_poll_func(self.poll)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
if not self.reserved_resources:
|
|
|
|
self.reserve_resources()
|
2017-11-09 13:26:35 +00:00
|
|
|
for t in self.tests:
|
|
|
|
if names and not t.name() in names:
|
|
|
|
t.set_skip()
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
continue
|
2017-11-09 13:26:35 +00:00
|
|
|
self.current_test = t
|
|
|
|
t.run()
|
2017-06-13 16:07:57 +00:00
|
|
|
self.stop_processes()
|
|
|
|
self.objects_cleanup()
|
|
|
|
self.reserved_resources.put_all()
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
except Exception:
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.log_exn()
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
except BaseException as e:
|
|
|
|
# when the program is aborted by a signal (like Ctrl-C), escalate to abort all.
|
|
|
|
self.err('SUITE RUN ABORTED: %s' % type(e).__name__)
|
|
|
|
raise
|
2017-05-17 12:51:31 +00:00
|
|
|
finally:
|
|
|
|
# if sys.exit() called from signal handler (e.g. SIGINT), SystemExit
|
|
|
|
# base exception is raised. Make sure to stop processes in this
|
|
|
|
# finally section. Resources are automatically freed with 'atexit'.
|
|
|
|
self.stop_processes()
|
2017-05-29 02:13:58 +00:00
|
|
|
self.objects_cleanup()
|
2017-05-29 00:53:54 +00:00
|
|
|
self.free_resources()
|
2018-03-28 17:17:34 +00:00
|
|
|
MainLoop.unregister_poll_func(self.poll)
|
2018-05-09 09:24:23 +00:00
|
|
|
self.test_import_modules_cleanup()
|
2018-05-08 13:28:48 +00:00
|
|
|
util.import_path_remove(suite_libdir)
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
self.duration = time.time() - self.start_timestamp
|
|
|
|
|
2020-03-05 16:22:40 +00:00
|
|
|
passed, skipped, failed, errors = self.count_test_results()
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
# if no tests ran, count it as failure
|
2020-03-05 16:22:40 +00:00
|
|
|
if passed and not failed and not errors:
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
self.status = SuiteRun.PASS
|
|
|
|
else:
|
|
|
|
self.status = SuiteRun.FAIL
|
|
|
|
|
|
|
|
log.large_separator(self.trial.name(), self.name(), self.status, sublevel=2, space_above=False)
|
|
|
|
|
|
|
|
def passed(self):
|
|
|
|
return self.status == SuiteRun.PASS
|
|
|
|
|
|
|
|
def count_test_results(self):
|
|
|
|
passed = 0
|
|
|
|
skipped = 0
|
|
|
|
failed = 0
|
2020-03-05 16:22:40 +00:00
|
|
|
errors = 0
|
2017-11-09 13:26:35 +00:00
|
|
|
for t in self.tests:
|
2020-03-05 16:22:40 +00:00
|
|
|
if t.status == test.Test.SKIP:
|
|
|
|
skipped += 1
|
|
|
|
elif t.status == test.Test.PASS:
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
passed += 1
|
2017-11-09 13:26:35 +00:00
|
|
|
elif t.status == test.Test.FAIL:
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
failed += 1
|
2020-03-05 16:22:40 +00:00
|
|
|
else: # error, could not run
|
|
|
|
errors += 1
|
|
|
|
return (passed, skipped, failed, errors)
|
2017-03-28 10:16:58 +00:00
|
|
|
|
2018-05-22 18:32:30 +00:00
|
|
|
def remember_to_stop(self, process, respawn=False):
|
|
|
|
'''Ask suite to monitor and manage lifecycle of the Process object. If a
|
|
|
|
process managed by suite finishes before cleanup time, the current test
|
|
|
|
will be marked as FAIL and end immediatelly. If respwan=True, then suite
|
|
|
|
will respawn() the process instead.'''
|
|
|
|
self._processes.insert(0, (process, respawn))
|
2017-03-28 10:16:58 +00:00
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def stop_processes(self):
|
2019-04-04 14:43:12 +00:00
|
|
|
if len(self._processes) == 0:
|
|
|
|
return
|
2019-02-27 02:31:50 +00:00
|
|
|
strategy = process.ParallelTerminationStrategy()
|
2017-06-13 16:07:57 +00:00
|
|
|
while self._processes:
|
2019-02-27 02:31:50 +00:00
|
|
|
proc, _ = self._processes.pop()
|
|
|
|
strategy.add_process(proc)
|
|
|
|
strategy.terminate_all()
|
2017-03-28 10:16:58 +00:00
|
|
|
|
2018-03-08 20:01:26 +00:00
|
|
|
def stop_process(self, process):
|
|
|
|
'Remove process from monitored list and stop it'
|
2018-05-22 18:32:30 +00:00
|
|
|
for proc_respawn in self._processes:
|
|
|
|
proc, respawn = proc_respawn
|
|
|
|
if proc == process:
|
|
|
|
self._processes.remove(proc_respawn)
|
|
|
|
proc.terminate()
|
2018-03-08 20:01:26 +00:00
|
|
|
|
2017-05-29 00:53:54 +00:00
|
|
|
def free_resources(self):
|
|
|
|
if self.reserved_resources is None:
|
|
|
|
return
|
|
|
|
self.reserved_resources.free()
|
|
|
|
|
2017-06-06 19:52:03 +00:00
|
|
|
def ip_address(self, specifics=None):
|
|
|
|
return self.reserved_resources.get(resource.R_IP_ADDRESS, specifics=specifics)
|
2017-03-28 12:30:28 +00:00
|
|
|
|
2017-05-18 16:35:32 +00:00
|
|
|
def nitb(self, ip_address=None):
|
|
|
|
if ip_address is None:
|
|
|
|
ip_address = self.ip_address()
|
2020-04-10 17:41:06 +00:00
|
|
|
return nitb_osmo.OsmoNitb(self, ip_address)
|
2017-03-28 12:30:28 +00:00
|
|
|
|
2017-05-18 13:24:02 +00:00
|
|
|
def hlr(self, ip_address=None):
|
|
|
|
if ip_address is None:
|
|
|
|
ip_address = self.ip_address()
|
2020-04-10 17:41:06 +00:00
|
|
|
return hlr_osmo.OsmoHlr(self, ip_address)
|
2017-05-18 13:24:02 +00:00
|
|
|
|
2017-08-31 16:30:11 +00:00
|
|
|
def ggsn(self, ip_address=None):
|
|
|
|
if ip_address is None:
|
|
|
|
ip_address = self.ip_address()
|
2020-04-10 17:41:06 +00:00
|
|
|
return ggsn_osmo.OsmoGgsn(self, ip_address)
|
2017-08-31 16:30:11 +00:00
|
|
|
|
2017-11-23 10:07:42 +00:00
|
|
|
def sgsn(self, hlr, ggsn, ip_address=None):
|
|
|
|
if ip_address is None:
|
|
|
|
ip_address = self.ip_address()
|
2020-04-10 17:41:06 +00:00
|
|
|
return sgsn_osmo.OsmoSgsn(self, hlr, ggsn, ip_address)
|
2017-11-23 10:07:42 +00:00
|
|
|
|
2017-05-18 13:24:02 +00:00
|
|
|
def mgcpgw(self, ip_address=None, bts_ip=None):
|
|
|
|
if ip_address is None:
|
|
|
|
ip_address = self.ip_address()
|
2020-04-10 17:41:06 +00:00
|
|
|
return mgcpgw_osmo.OsmoMgcpgw(self, ip_address, bts_ip)
|
2017-05-18 13:24:02 +00:00
|
|
|
|
2017-11-09 12:02:09 +00:00
|
|
|
def mgw(self, ip_address=None):
|
|
|
|
if ip_address is None:
|
|
|
|
ip_address = self.ip_address()
|
2020-04-10 17:41:06 +00:00
|
|
|
return mgw_osmo.OsmoMgw(self, ip_address)
|
2017-11-09 12:02:09 +00:00
|
|
|
|
2017-11-16 17:06:37 +00:00
|
|
|
def msc(self, hlr, mgcpgw, stp, ip_address=None):
|
2017-05-18 13:24:02 +00:00
|
|
|
if ip_address is None:
|
|
|
|
ip_address = self.ip_address()
|
2020-04-10 17:41:06 +00:00
|
|
|
return msc_osmo.OsmoMsc(self, hlr, mgcpgw, stp, ip_address)
|
2017-05-18 13:24:02 +00:00
|
|
|
|
2017-11-16 17:06:37 +00:00
|
|
|
def bsc(self, msc, mgw, stp, ip_address=None):
|
2017-05-18 13:24:02 +00:00
|
|
|
if ip_address is None:
|
|
|
|
ip_address = self.ip_address()
|
2020-04-10 17:41:06 +00:00
|
|
|
return bsc_osmo.OsmoBsc(self, msc, mgw, stp, ip_address)
|
2017-05-18 13:24:02 +00:00
|
|
|
|
2017-06-13 14:26:06 +00:00
|
|
|
def stp(self, ip_address=None):
|
|
|
|
if ip_address is None:
|
|
|
|
ip_address = self.ip_address()
|
2020-04-10 17:41:06 +00:00
|
|
|
return stp_osmo.OsmoStp(self, ip_address)
|
2017-06-13 14:26:06 +00:00
|
|
|
|
2018-08-29 03:28:33 +00:00
|
|
|
def ms_driver(self):
|
|
|
|
ms = ms_driver.MsDriver(self)
|
|
|
|
self.register_for_cleanup(ms)
|
|
|
|
return ms
|
|
|
|
|
2017-06-06 19:52:03 +00:00
|
|
|
def bts(self, specifics=None):
|
2017-09-08 11:55:54 +00:00
|
|
|
bts = bts_obj(self, self.reserved_resources.get(resource.R_BTS, specifics=specifics))
|
2017-11-06 17:40:23 +00:00
|
|
|
bts.set_lac(self.lac())
|
2017-11-28 14:50:02 +00:00
|
|
|
bts.set_rac(self.rac())
|
2017-11-07 10:13:20 +00:00
|
|
|
bts.set_cellid(self.cellid())
|
2017-11-28 14:50:02 +00:00
|
|
|
bts.set_bvci(self.bvci())
|
2017-09-08 11:55:54 +00:00
|
|
|
self.register_for_cleanup(bts)
|
|
|
|
return bts
|
2017-03-28 12:30:28 +00:00
|
|
|
|
2017-06-06 19:52:03 +00:00
|
|
|
def modem(self, specifics=None):
|
|
|
|
conf = self.reserved_resources.get(resource.R_MODEM, specifics=specifics)
|
2019-02-27 08:18:38 +00:00
|
|
|
ms_type = conf.get('type')
|
|
|
|
ms_class = resource.KNOWN_MS_TYPES.get(ms_type)
|
|
|
|
if ms_class is None:
|
|
|
|
raise RuntimeError('No such Modem type is defined: %r' % ms_type)
|
2017-05-29 02:13:58 +00:00
|
|
|
self.dbg('create Modem object', conf=conf)
|
2019-02-27 08:18:38 +00:00
|
|
|
ms = ms_class(self, conf)
|
2017-11-07 10:57:42 +00:00
|
|
|
self.register_for_cleanup(ms)
|
|
|
|
return ms
|
2017-03-28 12:30:28 +00:00
|
|
|
|
2017-05-06 13:05:02 +00:00
|
|
|
def modems(self, count):
|
|
|
|
l = []
|
|
|
|
for i in range(count):
|
|
|
|
l.append(self.modem())
|
|
|
|
return l
|
|
|
|
|
2019-02-25 09:48:50 +00:00
|
|
|
def all_resources(self, resource_func):
|
|
|
|
"""Returns all yielded resource."""
|
|
|
|
l = []
|
|
|
|
while True:
|
|
|
|
try:
|
|
|
|
l.append(resource_func())
|
|
|
|
except resource.NoResourceExn:
|
|
|
|
return l
|
|
|
|
|
2017-05-30 13:33:57 +00:00
|
|
|
def esme(self):
|
|
|
|
esme_obj = esme.Esme(self.msisdn())
|
2017-08-10 08:59:40 +00:00
|
|
|
self.register_for_cleanup(esme_obj)
|
2017-05-30 13:33:57 +00:00
|
|
|
return esme_obj
|
|
|
|
|
2020-02-11 16:41:13 +00:00
|
|
|
def run_node(self, specifics=None):
|
|
|
|
return run_node.RunNode.from_conf(self.reserved_resources.get(resource.R_RUN_NODE, specifics=specifics))
|
|
|
|
|
2020-02-11 16:45:26 +00:00
|
|
|
def enb(self, specifics=None):
|
|
|
|
enb = enb_obj(self, self.reserved_resources.get(resource.R_ENB, specifics=specifics))
|
|
|
|
self.register_for_cleanup(enb)
|
|
|
|
return enb
|
|
|
|
|
|
|
|
def epc(self, run_node=None):
|
|
|
|
if run_node is None:
|
|
|
|
run_node = self.run_node()
|
2020-03-31 11:45:01 +00:00
|
|
|
epc = epc_obj(self, run_node)
|
|
|
|
self.register_for_cleanup(epc)
|
|
|
|
return epc
|
2020-02-11 16:45:26 +00:00
|
|
|
|
2018-05-17 14:59:58 +00:00
|
|
|
def osmocon(self, specifics=None):
|
|
|
|
conf = self.reserved_resources.get(resource.R_OSMOCON, specifics=specifics)
|
|
|
|
osmocon_obj = osmocon.Osmocon(self, conf=conf)
|
|
|
|
self.register_for_cleanup(osmocon_obj)
|
|
|
|
return osmocon_obj
|
|
|
|
|
2018-10-26 13:54:28 +00:00
|
|
|
def iperf3srv(self, ip_address=None):
|
|
|
|
if ip_address is None:
|
|
|
|
ip_address = self.ip_address()
|
|
|
|
iperf3srv_obj = iperf3.IPerf3Server(self, ip_address)
|
|
|
|
return iperf3srv_obj
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def msisdn(self):
|
2017-05-30 13:33:57 +00:00
|
|
|
msisdn = self.resources_pool.next_msisdn(self)
|
2017-03-28 12:30:28 +00:00
|
|
|
self.log('using MSISDN', msisdn)
|
|
|
|
return msisdn
|
|
|
|
|
2017-11-06 17:40:23 +00:00
|
|
|
def lac(self):
|
|
|
|
lac = self.resources_pool.next_lac(self)
|
|
|
|
self.log('using LAC', lac)
|
|
|
|
return lac
|
|
|
|
|
2017-11-28 14:50:02 +00:00
|
|
|
def rac(self):
|
|
|
|
rac = self.resources_pool.next_rac(self)
|
|
|
|
self.log('using RAC', rac)
|
|
|
|
return rac
|
|
|
|
|
2017-11-07 10:13:20 +00:00
|
|
|
def cellid(self):
|
|
|
|
cellid = self.resources_pool.next_cellid(self)
|
|
|
|
self.log('using CellId', cellid)
|
|
|
|
return cellid
|
|
|
|
|
2017-11-28 14:50:02 +00:00
|
|
|
def bvci(self):
|
|
|
|
bvci = self.resources_pool.next_bvci(self)
|
|
|
|
self.log('using BVCI', bvci)
|
|
|
|
return bvci
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def poll(self):
|
2019-04-04 15:44:33 +00:00
|
|
|
for proc, respawn in self._processes:
|
|
|
|
if proc.terminated():
|
|
|
|
if respawn == True:
|
|
|
|
proc.respawn()
|
|
|
|
else:
|
|
|
|
proc.log_stdout_tail()
|
|
|
|
proc.log_stderr_tail()
|
|
|
|
log.ctx(proc)
|
|
|
|
raise log.Error('Process ended prematurely: %s' % proc.name())
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
def prompt(self, *msgs, **msg_details):
|
|
|
|
'ask for user interaction. Do not use in tests that should run automatically!'
|
|
|
|
if msg_details:
|
|
|
|
msgs = list(msgs)
|
|
|
|
msgs.append('{%s}' %
|
|
|
|
(', '.join(['%s=%r' % (k,v)
|
|
|
|
for k,v in sorted(msg_details.items())])))
|
|
|
|
msg = ' '.join(msgs) or 'Hit Enter to continue'
|
|
|
|
self.log('prompt:', msg)
|
2017-05-06 14:05:33 +00:00
|
|
|
sys.__stdout__.write('\n\n--- PROMPT ---\n')
|
2017-03-28 12:30:28 +00:00
|
|
|
sys.__stdout__.write(msg)
|
2017-05-06 14:05:33 +00:00
|
|
|
sys.__stdout__.write('\n')
|
2017-03-28 12:30:28 +00:00
|
|
|
sys.__stdout__.flush()
|
2018-03-28 17:17:34 +00:00
|
|
|
entered = util.input_polling('> ', MainLoop.poll)
|
2017-05-06 14:05:33 +00:00
|
|
|
self.log('prompt entered:', repr(entered))
|
2017-03-28 12:30:28 +00:00
|
|
|
return entered
|
|
|
|
|
2017-05-22 18:02:41 +00:00
|
|
|
def resource_status_str(self):
|
|
|
|
return '\n'.join(('',
|
|
|
|
'SUITE RUN: %s' % self.origin_id(),
|
|
|
|
'ASKED FOR:', pprint.pformat(self._resource_requirements),
|
|
|
|
'RESERVED COUNT:', pprint.pformat(self.reserved_resources.counts()),
|
|
|
|
'RESOURCES STATE:', repr(self.reserved_resources)))
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
loaded_suite_definitions = {}
|
|
|
|
|
|
|
|
def load(suite_name):
|
|
|
|
global loaded_suite_definitions
|
|
|
|
|
|
|
|
suite = loaded_suite_definitions.get(suite_name)
|
|
|
|
if suite is not None:
|
|
|
|
return suite
|
|
|
|
|
|
|
|
suites_dir = config.get_suites_dir()
|
|
|
|
suite_dir = suites_dir.child(suite_name)
|
|
|
|
if not suites_dir.exists(suite_name):
|
|
|
|
raise RuntimeError('Suite not found: %r in %r' % (suite_name, suites_dir))
|
|
|
|
if not suites_dir.isdir(suite_name):
|
|
|
|
raise RuntimeError('Suite name found, but not a directory: %r' % (suite_dir))
|
|
|
|
|
|
|
|
suite_def = SuiteDefinition(suite_dir)
|
|
|
|
loaded_suite_definitions[suite_name] = suite_def
|
|
|
|
return suite_def
|
|
|
|
|
|
|
|
def parse_suite_scenario_str(suite_scenario_str):
|
|
|
|
tokens = suite_scenario_str.split(':')
|
|
|
|
if len(tokens) > 2:
|
|
|
|
raise RuntimeError('invalid combination string: %r' % suite_scenario_str)
|
|
|
|
|
|
|
|
suite_name = tokens[0]
|
|
|
|
if len(tokens) <= 1:
|
|
|
|
scenario_names = []
|
|
|
|
else:
|
|
|
|
scenario_names = tokens[1].split('+')
|
|
|
|
|
|
|
|
return suite_name, scenario_names
|
|
|
|
|
|
|
|
def load_suite_scenario_str(suite_scenario_str):
|
|
|
|
suite_name, scenario_names = parse_suite_scenario_str(suite_scenario_str)
|
|
|
|
suite = load(suite_name)
|
2020-05-04 10:05:05 +00:00
|
|
|
scenarios = [config.get_scenario(scenario_name, schema.get_all_schema()) for scenario_name in scenario_names]
|
2017-04-13 01:11:59 +00:00
|
|
|
return (suite_scenario_str, suite, scenarios)
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
def bts_obj(suite_run, conf):
|
|
|
|
bts_type = conf.get('type')
|
2017-06-14 00:59:55 +00:00
|
|
|
log.dbg('create BTS object', type=bts_type)
|
2017-03-28 12:30:28 +00:00
|
|
|
bts_class = resource.KNOWN_BTS_TYPES.get(bts_type)
|
|
|
|
if bts_class is None:
|
|
|
|
raise RuntimeError('No such BTS type is defined: %r' % bts_type)
|
|
|
|
return bts_class(suite_run, conf)
|
|
|
|
|
2020-02-11 16:45:26 +00:00
|
|
|
def enb_obj(suite_run, conf):
|
|
|
|
enb_type = conf.get('type')
|
|
|
|
log.dbg('create ENB object', type=enb_type)
|
|
|
|
enb_class = resource.KNOWN_ENB_TYPES.get(enb_type)
|
|
|
|
if enb_class is None:
|
|
|
|
raise RuntimeError('No such ENB type is defined: %r' % enb_type)
|
|
|
|
return enb_class(suite_run, conf)
|
|
|
|
|
2020-03-31 11:45:01 +00:00
|
|
|
def epc_obj(suite_run, run_node):
|
|
|
|
values = dict(epc=config.get_defaults('epc'))
|
|
|
|
config.overlay(values, dict(epc=suite_run.config().get('epc', {})))
|
|
|
|
epc_type = values['epc'].get('type', None)
|
|
|
|
if epc_type is None:
|
|
|
|
raise RuntimeError('EPC type is not defined!')
|
|
|
|
log.dbg('create EPC object', type=epc_type)
|
|
|
|
epc_class = resource.KNOWN_EPC_TYPES.get(epc_type)
|
|
|
|
if epc_class is None:
|
|
|
|
raise RuntimeError('No such EPC type is defined: %r' % epc_type)
|
|
|
|
return epc_class(suite_run, run_node)
|
|
|
|
|
2017-03-28 10:16:58 +00:00
|
|
|
# vim: expandtab tabstop=4 shiftwidth=4
|