2017-03-28 12:30:28 +00:00
|
|
|
# osmo_gsm_tester: read and manage config files and global config
|
2017-03-28 10:16:58 +00:00
|
|
|
#
|
|
|
|
# Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH
|
|
|
|
#
|
|
|
|
# Author: Neels Hofmeyr <neels@hofmeyr.de>
|
|
|
|
#
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
2017-06-03 07:51:45 +00:00
|
|
|
# it under the terms of the GNU General Public License as
|
2017-03-28 10:16:58 +00:00
|
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
|
|
# License, or (at your option) any later version.
|
|
|
|
#
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
2017-06-03 07:51:45 +00:00
|
|
|
# GNU General Public License for more details.
|
2017-03-28 10:16:58 +00:00
|
|
|
#
|
2017-06-03 07:51:45 +00:00
|
|
|
# You should have received a copy of the GNU General Public License
|
2017-03-28 10:16:58 +00:00
|
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
# discussion for choice of config file format:
|
|
|
|
#
|
|
|
|
# Python syntax is insane, because it allows the config file to run arbitrary
|
|
|
|
# python commands.
|
|
|
|
#
|
|
|
|
# INI file format is nice and simple, but it doesn't allow having the same
|
|
|
|
# section numerous times (e.g. to define several modems or BTS models) and does
|
|
|
|
# not support nesting.
|
|
|
|
#
|
|
|
|
# JSON has too much braces and quotes to be easy to type
|
|
|
|
#
|
2017-03-28 12:30:28 +00:00
|
|
|
# YAML formatting is lean, but:
|
|
|
|
# - too powerful. The normal load() allows arbitrary code execution. There is
|
|
|
|
# safe_load().
|
|
|
|
# - allows several alternative ways of formatting, better to have just one
|
|
|
|
# authoritative style.
|
|
|
|
# - tries to detect types. It would be better to receive every setting as
|
|
|
|
# simple string rather than e.g. an IMSI as an integer.
|
|
|
|
# - e.g. an IMSI starting with a zero is interpreted as octal value, resulting
|
|
|
|
# in super confusing error messages if the user merely forgets to quote it.
|
|
|
|
# - does not tell me which line a config item came from, so no detailed error
|
|
|
|
# message is possible.
|
2017-03-28 10:16:58 +00:00
|
|
|
#
|
2017-03-28 12:30:28 +00:00
|
|
|
# The Python ConfigParserShootout page has numerous contestants, but many of
|
|
|
|
# those seem to be not widely used / standardized or even tested.
|
2017-03-28 10:16:58 +00:00
|
|
|
# https://wiki.python.org/moin/ConfigParserShootout
|
|
|
|
#
|
|
|
|
# The optimum would be a stripped down YAML format.
|
|
|
|
# In the lack of that, we shall go with yaml.load_safe() + a round trip
|
|
|
|
# (feeding back to itself), converting keys to lowercase and values to string.
|
2017-03-28 12:30:28 +00:00
|
|
|
# There is no solution for octal interpretations nor config file source lines
|
|
|
|
# unless, apparently, we implement our own config parser.
|
2017-03-28 10:16:58 +00:00
|
|
|
|
|
|
|
import yaml
|
|
|
|
import os
|
2017-09-12 11:43:40 +00:00
|
|
|
import copy
|
2017-03-28 10:16:58 +00:00
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
from . import log, schema, util
|
|
|
|
from .util import is_dict, is_list, Dir, get_tempdir
|
|
|
|
|
|
|
|
ENV_PREFIX = 'OSMO_GSM_TESTER_'
|
|
|
|
ENV_CONF = os.getenv(ENV_PREFIX + 'CONF')
|
|
|
|
|
2017-06-05 16:03:53 +00:00
|
|
|
override_conf = None
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
DEFAULT_CONFIG_LOCATIONS = [
|
|
|
|
'.',
|
2017-04-08 16:52:39 +00:00
|
|
|
os.path.join(os.getenv('HOME'), '.config', 'osmo-gsm-tester'),
|
|
|
|
'/usr/local/etc/osmo-gsm-tester',
|
|
|
|
'/etc/osmo-gsm-tester'
|
2017-03-28 12:30:28 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
PATHS_CONF = 'paths.conf'
|
2017-04-08 13:56:31 +00:00
|
|
|
DEFAULT_SUITES_CONF = 'default-suites.conf'
|
2017-03-28 12:30:28 +00:00
|
|
|
PATH_STATE_DIR = 'state_dir'
|
|
|
|
PATH_SUITES_DIR = 'suites_dir'
|
|
|
|
PATH_SCENARIOS_DIR = 'scenarios_dir'
|
|
|
|
PATHS_SCHEMA = {
|
|
|
|
PATH_STATE_DIR: schema.STR,
|
|
|
|
PATH_SUITES_DIR: schema.STR,
|
|
|
|
PATH_SCENARIOS_DIR: schema.STR,
|
|
|
|
}
|
2017-03-28 10:16:58 +00:00
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
PATHS_TEMPDIR_STR = '$TEMPDIR'
|
|
|
|
|
|
|
|
PATHS = None
|
|
|
|
|
2017-04-08 16:52:39 +00:00
|
|
|
def _get_config_file(basename, fail_if_missing=True):
|
2017-06-05 16:03:53 +00:00
|
|
|
if override_conf:
|
|
|
|
locations = [ override_conf ]
|
|
|
|
elif ENV_CONF:
|
2017-03-28 12:30:28 +00:00
|
|
|
locations = [ ENV_CONF ]
|
|
|
|
else:
|
|
|
|
locations = DEFAULT_CONFIG_LOCATIONS
|
|
|
|
|
|
|
|
for l in locations:
|
2017-05-04 14:39:29 +00:00
|
|
|
real_l = os.path.realpath(l)
|
|
|
|
p = os.path.realpath(os.path.join(real_l, basename))
|
2017-03-28 12:30:28 +00:00
|
|
|
if os.path.isfile(p):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg('Found config file', basename, 'as', p, 'in', l, 'which is', real_l, _category=log.C_CNF)
|
2017-05-04 14:39:29 +00:00
|
|
|
return (p, real_l)
|
2017-03-28 12:30:28 +00:00
|
|
|
if not fail_if_missing:
|
2017-04-08 16:52:39 +00:00
|
|
|
return None, None
|
2017-03-28 12:30:28 +00:00
|
|
|
raise RuntimeError('configuration file not found: %r in %r' % (basename,
|
|
|
|
[os.path.abspath(p) for p in locations]))
|
|
|
|
|
2017-04-08 16:52:39 +00:00
|
|
|
def get_config_file(basename, fail_if_missing=True):
|
|
|
|
path, found_in = _get_config_file(basename, fail_if_missing)
|
|
|
|
return path
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def read_config_file(basename, validation_schema=None, if_missing_return=False):
|
|
|
|
fail_if_missing = True
|
|
|
|
if if_missing_return is not False:
|
|
|
|
fail_if_missing = False
|
|
|
|
path = get_config_file(basename, fail_if_missing=fail_if_missing)
|
2017-04-08 16:52:39 +00:00
|
|
|
if path is None:
|
|
|
|
return if_missing_return
|
2017-03-28 12:30:28 +00:00
|
|
|
return read(path, validation_schema=validation_schema, if_missing_return=if_missing_return)
|
|
|
|
|
|
|
|
def get_configured_path(label, allow_unset=False):
|
|
|
|
global PATHS
|
|
|
|
|
|
|
|
env_name = ENV_PREFIX + label.upper()
|
|
|
|
env_path = os.getenv(env_name)
|
|
|
|
if env_path:
|
2017-05-04 14:39:29 +00:00
|
|
|
real_env_path = os.path.realpath(env_path)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg('Found path', label, 'as', env_path, 'in', '$' + env_name, 'which is', real_env_path, _category=log.C_CNF)
|
2017-05-04 14:39:29 +00:00
|
|
|
return real_env_path
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
if PATHS is None:
|
2017-04-08 16:52:39 +00:00
|
|
|
paths_file, found_in = _get_config_file(PATHS_CONF)
|
2017-03-28 12:30:28 +00:00
|
|
|
PATHS = read(paths_file, PATHS_SCHEMA)
|
2017-05-04 14:39:29 +00:00
|
|
|
# sorted for deterministic regression test results
|
|
|
|
for key, path in sorted(PATHS.items()):
|
2017-04-08 16:52:39 +00:00
|
|
|
if not path.startswith(os.pathsep):
|
2017-05-04 14:39:29 +00:00
|
|
|
PATHS[key] = os.path.realpath(os.path.join(found_in, path))
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg(paths_file + ': relative path', path, 'is', PATHS[key], _category=log.C_CNF)
|
2017-03-28 12:30:28 +00:00
|
|
|
p = PATHS.get(label)
|
|
|
|
if p is None and not allow_unset:
|
|
|
|
raise RuntimeError('missing configuration in %s: %r' % (PATHS_CONF, label))
|
|
|
|
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg('Found path', label, 'as', p, _category=log.C_CNF)
|
2017-03-28 12:30:28 +00:00
|
|
|
if p.startswith(PATHS_TEMPDIR_STR):
|
|
|
|
p = os.path.join(get_tempdir(), p[len(PATHS_TEMPDIR_STR):])
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg('Path', label, 'contained', PATHS_TEMPDIR_STR, 'and becomes', p, _category=log.C_CNF)
|
2017-03-28 12:30:28 +00:00
|
|
|
return p
|
|
|
|
|
|
|
|
def get_state_dir():
|
|
|
|
return Dir(get_configured_path(PATH_STATE_DIR))
|
|
|
|
|
|
|
|
def get_suites_dir():
|
|
|
|
return Dir(get_configured_path(PATH_SUITES_DIR))
|
|
|
|
|
|
|
|
def get_scenarios_dir():
|
|
|
|
return Dir(get_configured_path(PATH_SCENARIOS_DIR))
|
|
|
|
|
|
|
|
def read(path, validation_schema=None, if_missing_return=False):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.ctx(path)
|
|
|
|
if not os.path.isfile(path) and if_missing_return is not False:
|
|
|
|
return if_missing_return
|
|
|
|
with open(path, 'r') as f:
|
|
|
|
config = yaml.safe_load(f)
|
|
|
|
config = _standardize(config)
|
|
|
|
if validation_schema:
|
|
|
|
schema.validate(config, validation_schema)
|
|
|
|
return config
|
2017-03-28 10:16:58 +00:00
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def write(path, config):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.ctx(path)
|
|
|
|
with open(path, 'w') as f:
|
|
|
|
f.write(tostr(config))
|
2017-03-28 12:30:28 +00:00
|
|
|
|
2017-03-28 10:16:58 +00:00
|
|
|
def tostr(config):
|
|
|
|
return _tostr(_standardize(config))
|
|
|
|
|
|
|
|
def _tostr(config):
|
|
|
|
return yaml.dump(config, default_flow_style=False)
|
|
|
|
|
|
|
|
def _standardize_item(item):
|
|
|
|
if isinstance(item, (tuple, list)):
|
|
|
|
return [_standardize_item(i) for i in item]
|
|
|
|
if isinstance(item, dict):
|
|
|
|
return dict([(key.lower(), _standardize_item(val)) for key,val in item.items()])
|
|
|
|
return str(item)
|
|
|
|
|
|
|
|
def _standardize(config):
|
|
|
|
config = yaml.safe_load(_tostr(_standardize_item(config)))
|
|
|
|
return config
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def get_defaults(for_kind):
|
2017-04-14 02:18:06 +00:00
|
|
|
defaults = read_config_file('defaults.conf', if_missing_return={})
|
2017-03-28 12:30:28 +00:00
|
|
|
return defaults.get(for_kind, {})
|
|
|
|
|
2017-04-13 01:11:59 +00:00
|
|
|
class Scenario(log.Origin, dict):
|
|
|
|
def __init__(self, name, path):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
super().__init__(log.C_TST, name)
|
2017-04-13 01:11:59 +00:00
|
|
|
self.path = path
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def get_scenario(name, validation_schema=None):
|
|
|
|
scenarios_dir = get_scenarios_dir()
|
|
|
|
if not name.endswith('.conf'):
|
|
|
|
name = name + '.conf'
|
|
|
|
path = scenarios_dir.child(name)
|
|
|
|
if not os.path.isfile(path):
|
|
|
|
raise RuntimeError('No such scenario file: %r' % path)
|
2017-04-13 01:11:59 +00:00
|
|
|
sc = Scenario(name, path)
|
|
|
|
sc.update(read(path, validation_schema=validation_schema))
|
|
|
|
return sc
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
def add(dest, src):
|
|
|
|
if is_dict(dest):
|
|
|
|
if not is_dict(src):
|
|
|
|
raise ValueError('cannot add to dict a value of type: %r' % type(src))
|
|
|
|
|
|
|
|
for key, val in src.items():
|
|
|
|
dest_val = dest.get(key)
|
|
|
|
if dest_val is None:
|
|
|
|
dest[key] = val
|
|
|
|
else:
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.ctx(key=key)
|
|
|
|
add(dest_val, val)
|
2017-03-28 10:16:58 +00:00
|
|
|
return
|
2017-03-28 12:30:28 +00:00
|
|
|
if is_list(dest):
|
|
|
|
if not is_list(src):
|
|
|
|
raise ValueError('cannot add to list a value of type: %r' % type(src))
|
|
|
|
dest.extend(src)
|
|
|
|
return
|
|
|
|
if dest == src:
|
|
|
|
return
|
|
|
|
raise ValueError('cannot add dicts, conflicting items (values %r and %r)'
|
|
|
|
% (dest, src))
|
|
|
|
|
|
|
|
def combine(dest, src):
|
|
|
|
if is_dict(dest):
|
|
|
|
if not is_dict(src):
|
|
|
|
raise ValueError('cannot combine dict with a value of type: %r' % type(src))
|
|
|
|
|
|
|
|
for key, val in src.items():
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.ctx(key=key)
|
2017-03-28 12:30:28 +00:00
|
|
|
dest_val = dest.get(key)
|
|
|
|
if dest_val is None:
|
|
|
|
dest[key] = val
|
|
|
|
else:
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
combine(dest_val, val)
|
2017-03-28 12:30:28 +00:00
|
|
|
return
|
|
|
|
if is_list(dest):
|
|
|
|
if not is_list(src):
|
|
|
|
raise ValueError('cannot combine list with a value of type: %r' % type(src))
|
config: Fix combination of lists
This commit fixes combination of resources containing lists.
For lists containing complex types, it has been decided to handle them
as sorted list, where position in list matters. In this case, combine is
called recursively for each element in dest and src sharing position in
the list, and assumes that if one list is shorter than the other, then
it has to be combined against empty set for that tye.
For instance this is useful when defining trx_list properties, where a
BTS can have a different amount of TRX but we may be interested in
restricting the first TRX and don't care about extra TRX.
For lists containing simple types (eg. integers or strings), we just want
to merge both lists and we only need to check if the value is already there,
ie. handle them as unsortered sets. This case won't work if we call combine
for each element of the list because for a simple case it will just end up
checking if a[i] == b[i].
This kind of operation for simple types is needed in later commits where
cipher attribute is introduced. Without this patch, having following 2
scenarios and trying them to use together "-s foosuite:cipher-a50+ciphera51"
will fail:
cipher_a50.conf:
bts:
- ciphers:
- 'a5 0'
cipher_a51.conf
bts:
- ciphers:
- 'a5 1'
ValueError: cannot combine dicts, conflicting items (values 'a5 0' and 'a5 1')
Change-Id: Ib7a38f10eb9de338a77bf1fa3afceb9df1532015
2017-08-28 12:04:07 +00:00
|
|
|
# Validate that all elements in both lists are of the same type:
|
|
|
|
t = util.list_validate_same_elem_type(src + dest)
|
|
|
|
if t is None:
|
|
|
|
return # both lists are empty, return
|
|
|
|
# For lists of complex objects, we expect them to be sorted lists:
|
|
|
|
if t in (dict, list, tuple):
|
|
|
|
for i in range(len(dest)):
|
|
|
|
log.ctx(idx=i)
|
|
|
|
src_it = src[i] if i < len(src) else util.empty_instance_type(t)
|
|
|
|
combine(dest[i], src_it)
|
|
|
|
for i in range(len(dest), len(src)):
|
|
|
|
log.ctx(idx=i)
|
|
|
|
dest.append(src[i])
|
|
|
|
else: # for lists of basic elements, we handle them as unsorted sets:
|
|
|
|
for elem in src:
|
|
|
|
if elem not in dest:
|
|
|
|
dest.append(elem)
|
2017-03-28 12:30:28 +00:00
|
|
|
return
|
|
|
|
if dest == src:
|
|
|
|
return
|
|
|
|
raise ValueError('cannot combine dicts, conflicting items (values %r and %r)'
|
|
|
|
% (dest, src))
|
|
|
|
|
|
|
|
def overlay(dest, src):
|
|
|
|
if is_dict(dest):
|
|
|
|
if not is_dict(src):
|
|
|
|
raise ValueError('cannot combine dict with a value of type: %r' % type(src))
|
|
|
|
|
|
|
|
for key, val in src.items():
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.ctx(key=key)
|
2017-03-28 12:30:28 +00:00
|
|
|
dest_val = dest.get(key)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
dest[key] = overlay(dest_val, val)
|
2017-03-28 12:30:28 +00:00
|
|
|
return dest
|
|
|
|
if is_list(dest):
|
|
|
|
if not is_list(src):
|
|
|
|
raise ValueError('cannot combine list with a value of type: %r' % type(src))
|
|
|
|
for i in range(len(src)):
|
2017-08-28 15:21:34 +00:00
|
|
|
log.ctx(idx=i)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
dest[i] = overlay(dest[i], src[i])
|
2017-03-28 12:30:28 +00:00
|
|
|
return dest
|
|
|
|
return src
|
2017-09-12 11:43:40 +00:00
|
|
|
|
|
|
|
def replicate_times(d):
|
2017-09-10 14:33:10 +00:00
|
|
|
'''
|
|
|
|
replicate items that have a "times" > 1
|
|
|
|
|
|
|
|
'd' is a dict matching WANT_SCHEMA, which is the same as
|
|
|
|
the RESOURCES_SCHEMA, except each entity that can be reserved has a 'times'
|
|
|
|
field added, to indicate how many of those should be reserved.
|
|
|
|
'''
|
2017-09-12 11:43:40 +00:00
|
|
|
d = copy.deepcopy(d)
|
|
|
|
for key, item_list in d.items():
|
2017-09-12 13:02:25 +00:00
|
|
|
idx = 0
|
|
|
|
while idx < len(item_list):
|
|
|
|
item = item_list[idx]
|
|
|
|
times = int(item.pop('times', 1))
|
|
|
|
for j in range(1, times):
|
|
|
|
item_list.insert(idx + j, copy.deepcopy(item))
|
|
|
|
idx += times
|
2017-09-12 11:43:40 +00:00
|
|
|
return d
|
|
|
|
|
2017-03-28 10:16:58 +00:00
|
|
|
# vim: expandtab tabstop=4 shiftwidth=4
|