2017-03-28 12:30:28 +00:00
|
|
|
# osmo_gsm_tester: trial: directory of binaries to be tested
|
|
|
|
#
|
|
|
|
# Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH
|
|
|
|
#
|
|
|
|
# Author: Neels Hofmeyr <neels@hofmeyr.de>
|
|
|
|
#
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
2017-06-03 07:51:45 +00:00
|
|
|
# it under the terms of the GNU General Public License as
|
2017-03-28 12:30:28 +00:00
|
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
|
|
# License, or (at your option) any later version.
|
|
|
|
#
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
2017-06-03 07:51:45 +00:00
|
|
|
# GNU General Public License for more details.
|
2017-03-28 12:30:28 +00:00
|
|
|
#
|
2017-06-03 07:51:45 +00:00
|
|
|
# You should have received a copy of the GNU General Public License
|
2017-03-28 12:30:28 +00:00
|
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
import os
|
|
|
|
import time
|
|
|
|
import shutil
|
|
|
|
import tarfile
|
2020-06-05 11:19:02 +00:00
|
|
|
import pathlib
|
2017-03-28 12:30:28 +00:00
|
|
|
|
2020-05-05 10:18:35 +00:00
|
|
|
from . import log
|
|
|
|
from . import util
|
|
|
|
from . import report
|
2020-05-04 17:06:47 +00:00
|
|
|
from . import suite
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
FILE_MARK_TAKEN = 'taken'
|
|
|
|
FILE_CHECKSUMS = 'checksums.md5'
|
|
|
|
TIMESTAMP_FMT = '%Y-%m-%d_%H-%M-%S'
|
|
|
|
FILE_LAST_RUN = 'last_run'
|
2017-05-05 17:51:40 +00:00
|
|
|
FILE_LOG = 'log'
|
2017-05-06 22:01:09 +00:00
|
|
|
FILE_LOG_BRIEF = 'log_brief'
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
class Trial(log.Origin):
|
2017-05-15 16:24:35 +00:00
|
|
|
UNKNOWN = 'UNKNOWN'
|
|
|
|
PASS = 'PASS'
|
|
|
|
FAIL = 'FAIL'
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
@staticmethod
|
|
|
|
def next(trials_dir):
|
|
|
|
|
|
|
|
with trials_dir.lock('Trial.next'):
|
|
|
|
trials = [e for e in trials_dir.children()
|
|
|
|
if trials_dir.isdir(e) and not trials_dir.exists(e, FILE_MARK_TAKEN)]
|
|
|
|
if not trials:
|
|
|
|
return None
|
|
|
|
# sort by time to get the one that waited longest
|
|
|
|
trials.sort(key=lambda e: os.path.getmtime(trials_dir.child(e)))
|
|
|
|
next_trial = trials[0]
|
|
|
|
return Trial(trials_dir.child(next_trial)).take()
|
|
|
|
|
|
|
|
def __init__(self, trial_dir):
|
2017-05-14 01:25:45 +00:00
|
|
|
self.path = os.path.abspath(trial_dir)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
super().__init__(log.C_TST, os.path.basename(self.path))
|
2017-03-28 12:30:28 +00:00
|
|
|
self.dir = util.Dir(self.path)
|
|
|
|
self.inst_dir = util.Dir(self.dir.child('inst'))
|
2020-05-25 17:49:29 +00:00
|
|
|
self.bin_tars = {}
|
2017-05-15 16:24:35 +00:00
|
|
|
self.suites = []
|
|
|
|
self.status = Trial.UNKNOWN
|
2018-08-09 11:45:55 +00:00
|
|
|
self._run_dir = None
|
|
|
|
self.log_targets = None
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
def __repr__(self):
|
|
|
|
return self.name()
|
|
|
|
|
|
|
|
def __enter__(self):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
'''add a log target to log to the run dir, write taken marker, log a
|
|
|
|
starting separator.'''
|
2017-05-05 17:51:40 +00:00
|
|
|
run_dir = self.get_run_dir()
|
2017-05-14 14:16:31 +00:00
|
|
|
detailed_log = run_dir.new_child(FILE_LOG)
|
2017-05-05 17:51:40 +00:00
|
|
|
self.log_targets = [
|
2017-05-14 14:16:31 +00:00
|
|
|
log.FileLogTarget(detailed_log)
|
2017-05-05 17:51:40 +00:00
|
|
|
.set_all_levels(log.L_DBG)
|
|
|
|
.style_change(trace=True),
|
2017-05-06 22:01:09 +00:00
|
|
|
log.FileLogTarget(run_dir.new_child(FILE_LOG_BRIEF))
|
2017-05-24 16:31:01 +00:00
|
|
|
.style_change(src=False, all_origins_on_levels=(log.L_ERR, log.L_TRACEBACK))
|
2017-05-05 17:51:40 +00:00
|
|
|
]
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
log.large_separator(self.name(), sublevel=1)
|
2017-05-14 14:16:31 +00:00
|
|
|
self.log('Detailed log at', detailed_log)
|
2017-05-06 19:56:27 +00:00
|
|
|
self.take()
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
return self
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
def __exit__(self, *exc_info):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
'''log a report, then remove log file targets for this trial'''
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
self.log_report()
|
2017-05-05 17:51:40 +00:00
|
|
|
for lt in self.log_targets:
|
|
|
|
lt.remove()
|
|
|
|
self.log_targets = None
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def take(self):
|
|
|
|
self.dir.touch(FILE_MARK_TAKEN)
|
|
|
|
return self
|
|
|
|
|
|
|
|
def get_run_dir(self):
|
|
|
|
if self._run_dir is not None:
|
|
|
|
return self._run_dir
|
|
|
|
self._run_dir = util.Dir(self.dir.new_child('run.%s' % time.strftime(TIMESTAMP_FMT)))
|
|
|
|
self._run_dir.mkdir()
|
|
|
|
|
|
|
|
last_run = self.dir.child(FILE_LAST_RUN)
|
|
|
|
if os.path.islink(last_run):
|
|
|
|
os.remove(last_run)
|
|
|
|
if not os.path.exists(last_run):
|
|
|
|
os.symlink(self.dir.rel_path(self._run_dir.path), last_run)
|
|
|
|
return self._run_dir
|
|
|
|
|
|
|
|
def verify(self):
|
|
|
|
"verify checksums"
|
|
|
|
|
|
|
|
if not self.dir.exists():
|
|
|
|
raise RuntimeError('Trial dir does not exist: %r' % self.dir)
|
|
|
|
if not self.dir.isdir():
|
|
|
|
raise RuntimeError('Trial dir is not a dir: %r' % self.dir)
|
|
|
|
|
|
|
|
checksums = self.dir.child(FILE_CHECKSUMS)
|
|
|
|
if not self.dir.isfile(FILE_CHECKSUMS):
|
|
|
|
raise RuntimeError('No checksums file in trial dir: %r', checksums)
|
|
|
|
|
|
|
|
with open(checksums, 'r') as f:
|
|
|
|
line_nr = 0
|
|
|
|
for line in [l.strip() for l in f.readlines()]:
|
|
|
|
line_nr += 1
|
|
|
|
if not line:
|
|
|
|
continue
|
2020-05-25 17:49:29 +00:00
|
|
|
md5, relpath = line.split(' ')
|
|
|
|
file_path = self.dir.child(relpath)
|
2017-03-28 12:30:28 +00:00
|
|
|
|
2020-05-25 17:49:29 +00:00
|
|
|
if not self.dir.isfile(relpath):
|
2017-03-28 12:30:28 +00:00
|
|
|
raise RuntimeError('File listed in checksums file but missing in trials dir:'
|
|
|
|
' %r vs. %r line %d' % (file_path, checksums, line_nr))
|
|
|
|
|
|
|
|
if md5 != util.md5_of_file(file_path):
|
|
|
|
raise RuntimeError('Checksum mismatch for %r vs. %r line %d'
|
|
|
|
% (file_path, checksums, line_nr))
|
|
|
|
|
2020-05-25 17:49:29 +00:00
|
|
|
if relpath.endswith('.tgz') or relpath.endswith('.tar.gz'):
|
|
|
|
(label, name) = os.path.split(relpath)
|
|
|
|
#print('label: %s, name: %s' % (label, name))
|
|
|
|
li = self.bin_tars.get(label, [])
|
|
|
|
li.append(name)
|
|
|
|
self.bin_tars[label] = li
|
2017-03-28 12:30:28 +00:00
|
|
|
|
2020-05-25 17:49:29 +00:00
|
|
|
def has_bin_tar(self, bin_name, run_label):
|
2017-03-28 12:30:28 +00:00
|
|
|
bin_tar_start = '%s.' % bin_name
|
2020-05-25 17:49:29 +00:00
|
|
|
matches = [t for t in self.bin_tars[run_label] if t.startswith(bin_tar_start)]
|
|
|
|
self.dbg('has bin_tar', run_label=run_label, bin_name=bin_name, matches=matches)
|
2017-03-28 12:30:28 +00:00
|
|
|
if not matches:
|
|
|
|
return None
|
|
|
|
if len(matches) > 1:
|
2020-05-25 17:49:29 +00:00
|
|
|
raise RuntimeError('More than one match for bin name %r on run_label \'%s\': %r' % (bin_name, run_label, matches))
|
2017-03-28 12:30:28 +00:00
|
|
|
bin_tar = matches[0]
|
2020-05-25 17:49:29 +00:00
|
|
|
bin_tar_path = self.dir.child(os.path.join(run_label, bin_tar))
|
2017-03-28 12:30:28 +00:00
|
|
|
if not os.path.isfile(bin_tar_path):
|
|
|
|
raise RuntimeError('Not a file or missing: %r' % bin_tar_path)
|
|
|
|
return bin_tar_path
|
|
|
|
|
2020-05-25 17:49:29 +00:00
|
|
|
def get_inst(self, bin_name, run_label=None):
|
|
|
|
if run_label is None:
|
|
|
|
run_label = ''
|
|
|
|
bin_tar = self.has_bin_tar(bin_name, run_label)
|
2017-03-28 12:30:28 +00:00
|
|
|
if not bin_tar:
|
2017-04-08 16:52:39 +00:00
|
|
|
raise RuntimeError('No such binary available: %r' % bin_name)
|
2020-05-25 17:49:29 +00:00
|
|
|
inst_dir = self.inst_dir.child(os.path.join(run_label, bin_name))
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
if os.path.isdir(inst_dir):
|
|
|
|
# already unpacked
|
|
|
|
return inst_dir
|
|
|
|
|
|
|
|
t = None
|
|
|
|
try:
|
|
|
|
os.makedirs(inst_dir)
|
|
|
|
t = tarfile.open(bin_tar)
|
|
|
|
t.extractall(inst_dir)
|
|
|
|
return inst_dir
|
|
|
|
|
|
|
|
except:
|
|
|
|
shutil.rmtree(inst_dir)
|
|
|
|
raise
|
|
|
|
finally:
|
|
|
|
if t:
|
|
|
|
try:
|
|
|
|
t.close()
|
|
|
|
except:
|
|
|
|
pass
|
|
|
|
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
def add_suite_run(self, suite_scenario_str, suite_def, scenarios):
|
|
|
|
suite_run = suite.SuiteRun(self, suite_scenario_str, suite_def, scenarios)
|
2017-05-15 16:24:35 +00:00
|
|
|
self.suites.append(suite_run)
|
|
|
|
|
|
|
|
def run_suites(self, names=None):
|
|
|
|
self.status = Trial.UNKNOWN
|
2017-06-06 21:08:07 +00:00
|
|
|
try:
|
|
|
|
for suite_run in self.suites:
|
|
|
|
try:
|
|
|
|
suite_run.run_tests(names)
|
|
|
|
except BaseException as e:
|
|
|
|
# when the program is aborted by a signal (like Ctrl-C), escalate to abort all.
|
|
|
|
self.err('TRIAL RUN ABORTED: %s' % type(e).__name__)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
# log the traceback before the trial's logging is ended
|
|
|
|
log.log_exn()
|
2017-06-06 21:08:07 +00:00
|
|
|
raise
|
|
|
|
finally:
|
|
|
|
if suite_run.status != suite.SuiteRun.PASS:
|
|
|
|
self.status = Trial.FAIL
|
|
|
|
if self.status == Trial.UNKNOWN:
|
|
|
|
self.status = Trial.PASS
|
|
|
|
finally:
|
|
|
|
junit_path = self.get_run_dir().new_file(self.name()+'.xml')
|
|
|
|
self.log('Storing JUnit report in', junit_path)
|
|
|
|
report.trial_to_junit_write(self, junit_path)
|
2017-05-15 16:24:35 +00:00
|
|
|
|
2020-06-05 11:19:02 +00:00
|
|
|
def get_all_inst_hash_info(self):
|
|
|
|
d = {}
|
|
|
|
pathlist = pathlib.Path(str(self.inst_dir)).glob('**/*git_hashes.txt')
|
|
|
|
for path in pathlist:
|
|
|
|
# because path is object not string
|
|
|
|
abs_path_str = str(path) # because path is object not string
|
|
|
|
dir, file = os.path.split(abs_path_str)
|
|
|
|
reldir = os.path.relpath(dir, str(self.inst_dir)).rstrip(os.sep)
|
|
|
|
with open(abs_path_str, 'r') as f:
|
|
|
|
for line in [l.strip() for l in f.readlines()]:
|
|
|
|
if not line:
|
|
|
|
continue
|
|
|
|
hash, proj = tuple(line.split(' ', 1))
|
|
|
|
d[os.path.join(reldir,proj)] = hash
|
|
|
|
return d
|
|
|
|
|
2017-05-15 16:24:35 +00:00
|
|
|
def log_report(self):
|
refactor: fix error handling; fix log.Origin; only one trial
A bit of refactoring to fix logging and error reporting, and simplify the code.
This transmogrifies some of the things committed in
0ffb41440661631fa1d520c152be4cf8ebd4c46b "Add JUnit XML reports; refactor test
reporting", which did not fully match the code structuring ideas used in
osmo-gsm-tester. Also solve some problems present from the start of the code
base.
Though this is a bit of a code bomb, it would take a lot of time to separate
this into smaller bits: these changes are closely related and resulted
incrementally from testing error handling and logging details. I hope it's ok.
Things changed / problems fixed:
Allow only a single trial to be run per cmdline invocation: unbloat trial and
suite invocation in osmo-gsm-tester.py.
There is a SuiteDefinition, intended to be immutable, and a mutable SuiteRun.
SuiteDefinition had a list of tests, which was modified by the SuiteRun to
record test results. Instead, have only the test basenames in the
SuiteDefinition and create a new set of Test() instances for each SuiteRun, to
ensure that no state leaks between separate suite runs.
State leaking across runs can be seen in
http://jenkins.osmocom.org/jenkins/view/osmo-gsm-tester/job/osmo-gsm-tester_run/453/
where an earlier sms test for sysmo succeeds, but its state gets overwritten by
the later sms test for trx that fails. The end result is that both tests
failed, although the first run was successful.
Fix a problem with Origin: log.Origin allowed to be __enter__ed more than once,
skipping the second entry. The problem there is that we'd still __exit__ twice
or more, popping the Origin off the stack even though it should still remain.
We could count __enter__ recurrences, but instead, completely disallow entering
a second time.
A code path should have one 'with' statement per object, at pivotal points like
run_suites or run_tests. Individual utility functions should not do 'with' on a
central object. The structure needed is, in pseudo code:
try:
with trial:
try:
with suite_run:
try:
with test:
test_actions()
The 'with' needs to be inside the 'try', so that the exception can be handled
in __exit__ before it reaches the exception logging.
To clarify this, like test exceptions caught in Test.run(), also move suite
exception handling from Trial into SuiteRun.run_tests(). There are 'with self'
in Test.run() and SuiteRun.run_tests(), which are well placed, because these
are pivotal points in the main code path.
Log output: clearly separate logging of distinct suites and test scripts, by
adding more large_separator() calls at the start of each test. Place these
separator calls in more logical places. Add separator size and spacing args.
Log output: print tracebacks only once, for the test script where they happen.
Have less state that duplicates other state: drop SuiteRun.test_failed_ctr and
suite.test_skipped_ctr, instead add SuiteRun.count_test_results().
For test failure reporting, store the traceback text in a separate member var.
In the text report, apply above changes and unclutter to achieve a brief and
easy to read result overview: print less filler characters, drop the starting
times, drop the tracebacks. This can be found in the individual test logs.
Because the tracebacks are no longer in the text report, the suite_test.py can
just print the reports and expect that output instead of asserting individual
contents.
In the text report, print duration in precision of .1 seconds.
Add origin information and a traceback text to the junit XML result to give
more context when browsing the result XML. For 'AssertionError', add the source
line of where the assertion hit.
Drop the explicit Failure exception. We don't need one specific exception to
mark a failure, instead any arbitrary exception is treated as a failure. Use
the exception's class name as fail_type.
Though my original idea was to use raising exceptions as the only way to cause
a test failure, I'm keeping the set_fail() function as an alternative way,
because it allows test specific cleanup and may come in handy later. To have
both ways integrate seamlessly, shift some result setting into 'finally'
clauses and make sure higher levels (suite, trial) count the contained items'
stati.
Minor tweak: write the 'pass' and 'skip' reports in lower case so that the
'FAIL' stands out.
Minor tweak: pass the return code that the program exit should return further
outward, so that the exit(1) call does not cause a SystemExit exception to be
logged.
The aims of this patch are:
- Logs are readable so that it is clear which logging belongs to which test and
suite.
- The logging origins are correct (vs. parents gone missing as previously)
- A single test error does not cause following tests or suites to be skipped.
- An exception "above" Exception, i.e. SystemExit and the like, *does*
immediately abort all tests and suites, and the results for tests that were
not run are reported as "unknown" (rather than skipped on purpose):
- Raising a SystemExit aborts all.
- Hitting ctrl-c aborts all.
- The resulting summary in the log is brief and readable.
Change-Id: Ibf0846d457cab26f54c25e6906a8bb304724e2d8
2017-06-06 17:41:17 +00:00
|
|
|
log.large_separator(self.name(), self.status)
|
2017-05-15 16:24:35 +00:00
|
|
|
self.log(report.trial_to_text(self))
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
# vim: expandtab tabstop=4 shiftwidth=4
|