2017-03-28 12:30:28 +00:00
|
|
|
# osmo_gsm_tester: read and manage config files and global config
|
2017-03-28 10:16:58 +00:00
|
|
|
#
|
|
|
|
# Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH
|
|
|
|
#
|
|
|
|
# Author: Neels Hofmeyr <neels@hofmeyr.de>
|
|
|
|
#
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
2017-06-03 07:51:45 +00:00
|
|
|
# it under the terms of the GNU General Public License as
|
2017-03-28 10:16:58 +00:00
|
|
|
# published by the Free Software Foundation, either version 3 of the
|
|
|
|
# License, or (at your option) any later version.
|
|
|
|
#
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
2017-06-03 07:51:45 +00:00
|
|
|
# GNU General Public License for more details.
|
2017-03-28 10:16:58 +00:00
|
|
|
#
|
2017-06-03 07:51:45 +00:00
|
|
|
# You should have received a copy of the GNU General Public License
|
2017-03-28 10:16:58 +00:00
|
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
# discussion for choice of config file format:
|
|
|
|
#
|
|
|
|
# Python syntax is insane, because it allows the config file to run arbitrary
|
|
|
|
# python commands.
|
|
|
|
#
|
|
|
|
# INI file format is nice and simple, but it doesn't allow having the same
|
|
|
|
# section numerous times (e.g. to define several modems or BTS models) and does
|
|
|
|
# not support nesting.
|
|
|
|
#
|
|
|
|
# JSON has too much braces and quotes to be easy to type
|
|
|
|
#
|
2017-03-28 12:30:28 +00:00
|
|
|
# YAML formatting is lean, but:
|
|
|
|
# - too powerful. The normal load() allows arbitrary code execution. There is
|
|
|
|
# safe_load().
|
|
|
|
# - allows several alternative ways of formatting, better to have just one
|
|
|
|
# authoritative style.
|
|
|
|
# - tries to detect types. It would be better to receive every setting as
|
|
|
|
# simple string rather than e.g. an IMSI as an integer.
|
|
|
|
# - e.g. an IMSI starting with a zero is interpreted as octal value, resulting
|
|
|
|
# in super confusing error messages if the user merely forgets to quote it.
|
|
|
|
# - does not tell me which line a config item came from, so no detailed error
|
|
|
|
# message is possible.
|
2017-03-28 10:16:58 +00:00
|
|
|
#
|
2017-03-28 12:30:28 +00:00
|
|
|
# The Python ConfigParserShootout page has numerous contestants, but many of
|
|
|
|
# those seem to be not widely used / standardized or even tested.
|
2017-03-28 10:16:58 +00:00
|
|
|
# https://wiki.python.org/moin/ConfigParserShootout
|
|
|
|
#
|
|
|
|
# The optimum would be a stripped down YAML format.
|
|
|
|
# In the lack of that, we shall go with yaml.load_safe() + a round trip
|
|
|
|
# (feeding back to itself), converting keys to lowercase and values to string.
|
2017-03-28 12:30:28 +00:00
|
|
|
# There is no solution for octal interpretations nor config file source lines
|
|
|
|
# unless, apparently, we implement our own config parser.
|
2017-03-28 10:16:58 +00:00
|
|
|
|
|
|
|
import yaml
|
|
|
|
import os
|
2017-09-12 11:43:40 +00:00
|
|
|
import copy
|
2017-03-28 10:16:58 +00:00
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
from . import log, schema, util
|
|
|
|
from .util import is_dict, is_list, Dir, get_tempdir
|
|
|
|
|
|
|
|
ENV_PREFIX = 'OSMO_GSM_TESTER_'
|
|
|
|
ENV_CONF = os.getenv(ENV_PREFIX + 'CONF')
|
|
|
|
|
2017-06-05 16:03:53 +00:00
|
|
|
override_conf = None
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
DEFAULT_CONFIG_LOCATIONS = [
|
|
|
|
'.',
|
2017-04-08 16:52:39 +00:00
|
|
|
os.path.join(os.getenv('HOME'), '.config', 'osmo-gsm-tester'),
|
|
|
|
'/usr/local/etc/osmo-gsm-tester',
|
|
|
|
'/etc/osmo-gsm-tester'
|
2017-03-28 12:30:28 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
PATHS_CONF = 'paths.conf'
|
2017-04-08 13:56:31 +00:00
|
|
|
DEFAULT_SUITES_CONF = 'default-suites.conf'
|
2017-03-28 12:30:28 +00:00
|
|
|
PATH_STATE_DIR = 'state_dir'
|
|
|
|
PATH_SUITES_DIR = 'suites_dir'
|
|
|
|
PATH_SCENARIOS_DIR = 'scenarios_dir'
|
|
|
|
PATHS_SCHEMA = {
|
|
|
|
PATH_STATE_DIR: schema.STR,
|
|
|
|
PATH_SUITES_DIR: schema.STR,
|
|
|
|
PATH_SCENARIOS_DIR: schema.STR,
|
|
|
|
}
|
2017-03-28 10:16:58 +00:00
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
PATHS_TEMPDIR_STR = '$TEMPDIR'
|
|
|
|
|
|
|
|
PATHS = None
|
|
|
|
|
2017-04-08 16:52:39 +00:00
|
|
|
def _get_config_file(basename, fail_if_missing=True):
|
2017-06-05 16:03:53 +00:00
|
|
|
if override_conf:
|
|
|
|
locations = [ override_conf ]
|
|
|
|
elif ENV_CONF:
|
2017-03-28 12:30:28 +00:00
|
|
|
locations = [ ENV_CONF ]
|
|
|
|
else:
|
|
|
|
locations = DEFAULT_CONFIG_LOCATIONS
|
|
|
|
|
|
|
|
for l in locations:
|
2017-05-04 14:39:29 +00:00
|
|
|
real_l = os.path.realpath(l)
|
|
|
|
p = os.path.realpath(os.path.join(real_l, basename))
|
2017-03-28 12:30:28 +00:00
|
|
|
if os.path.isfile(p):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg('Found config file', basename, 'as', p, 'in', l, 'which is', real_l, _category=log.C_CNF)
|
2017-05-04 14:39:29 +00:00
|
|
|
return (p, real_l)
|
2017-03-28 12:30:28 +00:00
|
|
|
if not fail_if_missing:
|
2017-04-08 16:52:39 +00:00
|
|
|
return None, None
|
2017-03-28 12:30:28 +00:00
|
|
|
raise RuntimeError('configuration file not found: %r in %r' % (basename,
|
|
|
|
[os.path.abspath(p) for p in locations]))
|
|
|
|
|
2017-04-08 16:52:39 +00:00
|
|
|
def get_config_file(basename, fail_if_missing=True):
|
|
|
|
path, found_in = _get_config_file(basename, fail_if_missing)
|
|
|
|
return path
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def read_config_file(basename, validation_schema=None, if_missing_return=False):
|
|
|
|
fail_if_missing = True
|
|
|
|
if if_missing_return is not False:
|
|
|
|
fail_if_missing = False
|
|
|
|
path = get_config_file(basename, fail_if_missing=fail_if_missing)
|
2017-04-08 16:52:39 +00:00
|
|
|
if path is None:
|
|
|
|
return if_missing_return
|
2017-03-28 12:30:28 +00:00
|
|
|
return read(path, validation_schema=validation_schema, if_missing_return=if_missing_return)
|
|
|
|
|
|
|
|
def get_configured_path(label, allow_unset=False):
|
|
|
|
global PATHS
|
|
|
|
|
|
|
|
env_name = ENV_PREFIX + label.upper()
|
|
|
|
env_path = os.getenv(env_name)
|
|
|
|
if env_path:
|
2017-05-04 14:39:29 +00:00
|
|
|
real_env_path = os.path.realpath(env_path)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg('Found path', label, 'as', env_path, 'in', '$' + env_name, 'which is', real_env_path, _category=log.C_CNF)
|
2017-05-04 14:39:29 +00:00
|
|
|
return real_env_path
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
if PATHS is None:
|
2017-04-08 16:52:39 +00:00
|
|
|
paths_file, found_in = _get_config_file(PATHS_CONF)
|
2017-03-28 12:30:28 +00:00
|
|
|
PATHS = read(paths_file, PATHS_SCHEMA)
|
2017-05-04 14:39:29 +00:00
|
|
|
# sorted for deterministic regression test results
|
|
|
|
for key, path in sorted(PATHS.items()):
|
2017-04-08 16:52:39 +00:00
|
|
|
if not path.startswith(os.pathsep):
|
2017-05-04 14:39:29 +00:00
|
|
|
PATHS[key] = os.path.realpath(os.path.join(found_in, path))
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg(paths_file + ': relative path', path, 'is', PATHS[key], _category=log.C_CNF)
|
2017-03-28 12:30:28 +00:00
|
|
|
p = PATHS.get(label)
|
|
|
|
if p is None and not allow_unset:
|
|
|
|
raise RuntimeError('missing configuration in %s: %r' % (PATHS_CONF, label))
|
|
|
|
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg('Found path', label, 'as', p, _category=log.C_CNF)
|
2017-03-28 12:30:28 +00:00
|
|
|
if p.startswith(PATHS_TEMPDIR_STR):
|
|
|
|
p = os.path.join(get_tempdir(), p[len(PATHS_TEMPDIR_STR):])
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.dbg('Path', label, 'contained', PATHS_TEMPDIR_STR, 'and becomes', p, _category=log.C_CNF)
|
2017-03-28 12:30:28 +00:00
|
|
|
return p
|
|
|
|
|
|
|
|
def get_state_dir():
|
|
|
|
return Dir(get_configured_path(PATH_STATE_DIR))
|
|
|
|
|
|
|
|
def get_suites_dir():
|
|
|
|
return Dir(get_configured_path(PATH_SUITES_DIR))
|
|
|
|
|
|
|
|
def get_scenarios_dir():
|
|
|
|
return Dir(get_configured_path(PATH_SCENARIOS_DIR))
|
|
|
|
|
|
|
|
def read(path, validation_schema=None, if_missing_return=False):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.ctx(path)
|
|
|
|
if not os.path.isfile(path) and if_missing_return is not False:
|
|
|
|
return if_missing_return
|
|
|
|
with open(path, 'r') as f:
|
|
|
|
config = yaml.safe_load(f)
|
|
|
|
config = _standardize(config)
|
|
|
|
if validation_schema:
|
|
|
|
schema.validate(config, validation_schema)
|
|
|
|
return config
|
2017-03-28 10:16:58 +00:00
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def write(path, config):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.ctx(path)
|
|
|
|
with open(path, 'w') as f:
|
|
|
|
f.write(tostr(config))
|
2017-03-28 12:30:28 +00:00
|
|
|
|
2017-03-28 10:16:58 +00:00
|
|
|
def tostr(config):
|
|
|
|
return _tostr(_standardize(config))
|
|
|
|
|
|
|
|
def _tostr(config):
|
|
|
|
return yaml.dump(config, default_flow_style=False)
|
|
|
|
|
|
|
|
def _standardize_item(item):
|
|
|
|
if isinstance(item, (tuple, list)):
|
|
|
|
return [_standardize_item(i) for i in item]
|
|
|
|
if isinstance(item, dict):
|
|
|
|
return dict([(key.lower(), _standardize_item(val)) for key,val in item.items()])
|
|
|
|
return str(item)
|
|
|
|
|
|
|
|
def _standardize(config):
|
|
|
|
config = yaml.safe_load(_tostr(_standardize_item(config)))
|
|
|
|
return config
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def get_defaults(for_kind):
|
2017-04-14 02:18:06 +00:00
|
|
|
defaults = read_config_file('defaults.conf', if_missing_return={})
|
2017-03-28 12:30:28 +00:00
|
|
|
return defaults.get(for_kind, {})
|
|
|
|
|
2017-04-13 01:11:59 +00:00
|
|
|
class Scenario(log.Origin, dict):
|
|
|
|
def __init__(self, name, path):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
super().__init__(log.C_TST, name)
|
2017-04-13 01:11:59 +00:00
|
|
|
self.path = path
|
|
|
|
|
2017-03-28 12:30:28 +00:00
|
|
|
def get_scenario(name, validation_schema=None):
|
|
|
|
scenarios_dir = get_scenarios_dir()
|
|
|
|
if not name.endswith('.conf'):
|
|
|
|
name = name + '.conf'
|
|
|
|
path = scenarios_dir.child(name)
|
|
|
|
if not os.path.isfile(path):
|
|
|
|
raise RuntimeError('No such scenario file: %r' % path)
|
2017-04-13 01:11:59 +00:00
|
|
|
sc = Scenario(name, path)
|
|
|
|
sc.update(read(path, validation_schema=validation_schema))
|
|
|
|
return sc
|
2017-03-28 12:30:28 +00:00
|
|
|
|
|
|
|
def add(dest, src):
|
|
|
|
if is_dict(dest):
|
|
|
|
if not is_dict(src):
|
|
|
|
raise ValueError('cannot add to dict a value of type: %r' % type(src))
|
|
|
|
|
|
|
|
for key, val in src.items():
|
|
|
|
dest_val = dest.get(key)
|
|
|
|
if dest_val is None:
|
|
|
|
dest[key] = val
|
|
|
|
else:
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.ctx(key=key)
|
|
|
|
add(dest_val, val)
|
2017-03-28 10:16:58 +00:00
|
|
|
return
|
2017-03-28 12:30:28 +00:00
|
|
|
if is_list(dest):
|
|
|
|
if not is_list(src):
|
|
|
|
raise ValueError('cannot add to list a value of type: %r' % type(src))
|
|
|
|
dest.extend(src)
|
|
|
|
return
|
|
|
|
if dest == src:
|
|
|
|
return
|
|
|
|
raise ValueError('cannot add dicts, conflicting items (values %r and %r)'
|
|
|
|
% (dest, src))
|
|
|
|
|
|
|
|
def combine(dest, src):
|
|
|
|
if is_dict(dest):
|
|
|
|
if not is_dict(src):
|
|
|
|
raise ValueError('cannot combine dict with a value of type: %r' % type(src))
|
|
|
|
|
|
|
|
for key, val in src.items():
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.ctx(key=key)
|
2017-03-28 12:30:28 +00:00
|
|
|
dest_val = dest.get(key)
|
|
|
|
if dest_val is None:
|
|
|
|
dest[key] = val
|
|
|
|
else:
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
combine(dest_val, val)
|
2017-03-28 12:30:28 +00:00
|
|
|
return
|
|
|
|
if is_list(dest):
|
|
|
|
if not is_list(src):
|
|
|
|
raise ValueError('cannot combine list with a value of type: %r' % type(src))
|
|
|
|
for i in range(len(src)):
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.ctx(idx=i)
|
|
|
|
combine(dest[i], src[i])
|
2017-03-28 12:30:28 +00:00
|
|
|
return
|
|
|
|
if dest == src:
|
|
|
|
return
|
|
|
|
raise ValueError('cannot combine dicts, conflicting items (values %r and %r)'
|
|
|
|
% (dest, src))
|
|
|
|
|
|
|
|
def overlay(dest, src):
|
|
|
|
if is_dict(dest):
|
|
|
|
if not is_dict(src):
|
|
|
|
raise ValueError('cannot combine dict with a value of type: %r' % type(src))
|
|
|
|
|
|
|
|
for key, val in src.items():
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
log.ctx(key=key)
|
2017-03-28 12:30:28 +00:00
|
|
|
dest_val = dest.get(key)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
dest[key] = overlay(dest_val, val)
|
2017-03-28 12:30:28 +00:00
|
|
|
return dest
|
|
|
|
if is_list(dest):
|
|
|
|
if not is_list(src):
|
|
|
|
raise ValueError('cannot combine list with a value of type: %r' % type(src))
|
|
|
|
for i in range(len(src)):
|
2017-08-28 15:21:34 +00:00
|
|
|
log.ctx(idx=i)
|
fix and refactor logging: drop 'with', simplify
With the recent fix of the junit report related issues, another issue arose:
the 'with log.Origin' was changed to disallow __enter__ing an object twice to
fix problems, now still code would fail because it tries to do 'with' on the
same object twice. The only reason is to ensure that logging is associated with
a given object. Instead of complicating even more, implement differently.
Refactor logging to simplify use: drop the 'with Origin' style completely, and
instead use the python stack to determine which objects are created by which,
and which object to associate a log statement with.
The new way: we rely on the convention that each class instance has a local
'self' referencing the object instance. If we need to find an origin as a new
object's parent, or to associate a log message with, we traverse each stack
frame, fetching the first local 'self' object that is a log.Origin class
instance.
How to use:
Simply call log.log() anywhere, and it finds an Origin object to log for, from
the stack. Alternatively call self.log() for any Origin() object to skip the
lookup.
Create classes as child class of log.Origin and make sure to call
super().__init__(category, name). This constructor will magically find a parent
Origin on the stack.
When an exception happens, we first escalate the exception up through call
scopes to where ever it is handled by log.log_exn(). This then finds an Origin
object in the traceback's stack frames, no need to nest in 'with' scopes.
Hence the 'with log.Origin' now "happens implicitly", we can write pure natural
python code, no more hassles with scope ordering.
Furthermore, any frame can place additional logging information in a frame by
calling log.ctx(). This is automatically inserted in the ancestry associated
with a log statement / exception.
Change-Id: I5f9b53150f2bb6fa9d63ce27f0806f0ca6a45e90
2017-06-09 23:18:27 +00:00
|
|
|
dest[i] = overlay(dest[i], src[i])
|
2017-03-28 12:30:28 +00:00
|
|
|
return dest
|
|
|
|
return src
|
2017-09-12 11:43:40 +00:00
|
|
|
|
|
|
|
def replicate_times(d):
|
2017-09-10 14:33:10 +00:00
|
|
|
'''
|
|
|
|
replicate items that have a "times" > 1
|
|
|
|
|
|
|
|
'd' is a dict matching WANT_SCHEMA, which is the same as
|
|
|
|
the RESOURCES_SCHEMA, except each entity that can be reserved has a 'times'
|
|
|
|
field added, to indicate how many of those should be reserved.
|
|
|
|
'''
|
2017-09-12 11:43:40 +00:00
|
|
|
d = copy.deepcopy(d)
|
|
|
|
for key, item_list in d.items():
|
2017-09-12 13:02:25 +00:00
|
|
|
idx = 0
|
|
|
|
while idx < len(item_list):
|
|
|
|
item = item_list[idx]
|
|
|
|
times = int(item.pop('times', 1))
|
|
|
|
for j in range(1, times):
|
|
|
|
item_list.insert(idx + j, copy.deepcopy(item))
|
|
|
|
idx += times
|
2017-09-12 11:43:40 +00:00
|
|
|
return d
|
|
|
|
|
2017-03-28 10:16:58 +00:00
|
|
|
# vim: expandtab tabstop=4 shiftwidth=4
|