Drop python2 support (again)
Re-apply reverted commit Iabda95073faa2191fd117e9637e0858c589e9d9e
("Drop python2 support"), but with additional changes to make the
scripts actually work with python3 and to make it build without python2.
I have verified, that the contrib/jenkins.sh scripts of all Osmocom
repositories (with their python3 patches on top) are working with this
patch and that all Osmocom repositories with the python3 patches build
in OBS (tested in own namespace).
All related patches for changing from python2 to 3 in other repositories
must be merged shortly after this one, as soon as the build slaves were
(automatically) updated to have the new osmo-python-tests installed:
https://gerrit.osmocom.org/q/topic:drop-py2
New fixes:
* osmopy/obscvty.py: verify: fix compare
Comparing maps in python3 does not work the same as in python2. Convert
them to lists first, so the compare works as intended again.
Fix error:
File "/home/user/code/osmo-dev/src/osmo-python-tests/scripts/osmotestvty.py", line 57, in test_history
assert(self.vty.w_verify(test_str, [t1]))
AssertionError
* osmopy/obscvty.py: use enc/dec with send/recv
Fix error:
self.socket.send("%s\r" % request)
TypeError: a bytes-like object is required, not 'str'
* scripts/osmotestconfig.py: use encode() before writing to file
Fix error:
File "/home/user/code/osmo-dev/src/osmo-python-tests/scripts/osmotestconfig.py", line 91, in copy_config
tmpfile.write(open(config).read())
File "/usr/lib/python3.5/tempfile.py", line 622, in func_wrapper
return func(*args, **kwargs)
TypeError: a bytes-like object is required, not 'str'
* debian/control: add --buildsystem=pybuild. Otherwise "--with python3"
is ignored and the build fails if python2 is not installed, with:
Can't exec "pyversions": No such file or directory at /usr/[...]/python_distutils.pm line 120.
Related: OS#2819
Change-Id: I3ffc3519bf6c22536a49dad7a966188ddad351a7
2019-12-09 13:41:14 +00:00
|
|
|
#!/usr/bin/env python3
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
# (C) 2013 by Katerina Barone-Adesi <kat.obsc@gmail.com>
|
|
|
|
# This program is free software: you can redistribute it and/or modify
|
|
|
|
# it under the terms of the GNU General Public License as published by
|
|
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
|
|
# (at your option) any later version.
|
|
|
|
|
|
|
|
# This program is distributed in the hope that it will be useful,
|
|
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
# GNU General Public License for more details.
|
|
|
|
|
|
|
|
# You should have received a copy of the GNU General Public License
|
|
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import os
|
|
|
|
import os.path
|
2020-07-15 06:53:10 +00:00
|
|
|
import shutil
|
2013-04-04 15:31:13 +00:00
|
|
|
import time
|
2016-04-13 15:29:51 +00:00
|
|
|
import sys, shutil, stat
|
2013-04-04 15:31:13 +00:00
|
|
|
import tempfile
|
|
|
|
|
|
|
|
import osmopy.obscvty as obscvty
|
|
|
|
import osmopy.osmoutil as osmoutil
|
|
|
|
|
|
|
|
|
|
|
|
# Return true iff all the tests for the given config pass
|
|
|
|
def test_config(app_desc, config, tmpdir, verbose=True):
|
|
|
|
try:
|
2014-07-04 18:43:38 +00:00
|
|
|
err = 0
|
2015-01-31 20:15:06 +00:00
|
|
|
if test_config_atest(app_desc, config, verify_doc, verbose)[0] > 0:
|
2014-07-04 18:43:38 +00:00
|
|
|
err += 1
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
newconfig = copy_config(tmpdir, config)
|
2014-07-04 18:43:38 +00:00
|
|
|
if test_config_atest(app_desc, newconfig, write_config, verbose) > 0:
|
|
|
|
err += 1
|
|
|
|
|
|
|
|
if test_config_atest(app_desc, newconfig, token_vty_command, verbose) > 0:
|
|
|
|
err += 1
|
|
|
|
|
|
|
|
return err
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
# If there's a socket error, skip the rest of the tests for this config
|
|
|
|
except IOError:
|
|
|
|
return 1
|
|
|
|
|
|
|
|
|
|
|
|
def test_config_atest(app_desc, config, run_test, verbose=True):
|
|
|
|
proc = None
|
|
|
|
ret = None
|
2017-02-24 19:49:34 +00:00
|
|
|
vty = None
|
2013-04-04 15:31:13 +00:00
|
|
|
try:
|
2016-03-30 12:58:17 +00:00
|
|
|
cmd = app_desc[1].split(' ') + [ "-c", config]
|
2013-04-04 15:31:13 +00:00
|
|
|
if verbose:
|
2018-01-03 16:35:59 +00:00
|
|
|
print("Verifying %s, test %s" % (' '.join(cmd), run_test.__name__))
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
proc = osmoutil.popen_devnull(cmd)
|
|
|
|
end = app_desc[2]
|
|
|
|
port = app_desc[0]
|
|
|
|
vty = obscvty.VTYInteract(end, "127.0.0.1", port)
|
|
|
|
ret = run_test(vty)
|
|
|
|
|
|
|
|
except IOError as se:
|
2018-01-03 16:35:59 +00:00
|
|
|
print("Failed to verify %s" % ' '.join(cmd), file=sys.stderr)
|
|
|
|
print("Current directory: %s" % os.getcwd(), file=sys.stderr)
|
|
|
|
print("Error was %s" % se, file=sys.stderr)
|
|
|
|
print("Config was\n%s" % open(config).read(), file=sys.stderr)
|
2013-04-04 15:31:13 +00:00
|
|
|
raise se
|
|
|
|
|
|
|
|
finally:
|
|
|
|
if proc:
|
|
|
|
osmoutil.end_proc(proc)
|
2017-02-24 19:49:34 +00:00
|
|
|
if vty:
|
|
|
|
vty._close_socket()
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
return ret
|
|
|
|
|
|
|
|
def copy_config(dirname, config):
|
2016-04-07 12:11:25 +00:00
|
|
|
shutil.rmtree(dirname, True)
|
|
|
|
ign = shutil.ignore_patterns('*.cfg')
|
|
|
|
shutil.copytree(os.path.dirname(config), dirname, ignore=ign)
|
2016-04-13 15:29:51 +00:00
|
|
|
os.chmod(dirname, stat.S_IRWXU)
|
2016-04-07 12:11:25 +00:00
|
|
|
|
2013-04-04 15:31:13 +00:00
|
|
|
try:
|
|
|
|
os.stat(dirname)
|
|
|
|
except OSError:
|
|
|
|
os.mkdir(dirname)
|
|
|
|
|
|
|
|
prefix = os.path.basename(config)
|
|
|
|
tmpfile = tempfile.NamedTemporaryFile(
|
|
|
|
dir=dirname, prefix=prefix, delete=False)
|
Drop python2 support (again)
Re-apply reverted commit Iabda95073faa2191fd117e9637e0858c589e9d9e
("Drop python2 support"), but with additional changes to make the
scripts actually work with python3 and to make it build without python2.
I have verified, that the contrib/jenkins.sh scripts of all Osmocom
repositories (with their python3 patches on top) are working with this
patch and that all Osmocom repositories with the python3 patches build
in OBS (tested in own namespace).
All related patches for changing from python2 to 3 in other repositories
must be merged shortly after this one, as soon as the build slaves were
(automatically) updated to have the new osmo-python-tests installed:
https://gerrit.osmocom.org/q/topic:drop-py2
New fixes:
* osmopy/obscvty.py: verify: fix compare
Comparing maps in python3 does not work the same as in python2. Convert
them to lists first, so the compare works as intended again.
Fix error:
File "/home/user/code/osmo-dev/src/osmo-python-tests/scripts/osmotestvty.py", line 57, in test_history
assert(self.vty.w_verify(test_str, [t1]))
AssertionError
* osmopy/obscvty.py: use enc/dec with send/recv
Fix error:
self.socket.send("%s\r" % request)
TypeError: a bytes-like object is required, not 'str'
* scripts/osmotestconfig.py: use encode() before writing to file
Fix error:
File "/home/user/code/osmo-dev/src/osmo-python-tests/scripts/osmotestconfig.py", line 91, in copy_config
tmpfile.write(open(config).read())
File "/usr/lib/python3.5/tempfile.py", line 622, in func_wrapper
return func(*args, **kwargs)
TypeError: a bytes-like object is required, not 'str'
* debian/control: add --buildsystem=pybuild. Otherwise "--with python3"
is ignored and the build fails if python2 is not installed, with:
Can't exec "pyversions": No such file or directory at /usr/[...]/python_distutils.pm line 120.
Related: OS#2819
Change-Id: I3ffc3519bf6c22536a49dad7a966188ddad351a7
2019-12-09 13:41:14 +00:00
|
|
|
tmpfile.write(open(config).read().encode())
|
2013-04-04 15:31:13 +00:00
|
|
|
tmpfile.close()
|
|
|
|
# This works around the precautions NamedTemporaryFile is made for...
|
|
|
|
return tmpfile.name
|
|
|
|
|
|
|
|
|
|
|
|
def write_config(vty):
|
|
|
|
new_config = vty.enabled_command("write")
|
2016-08-15 10:31:46 +00:00
|
|
|
if not new_config.startswith("Configuration saved to "):
|
|
|
|
print(new_config)
|
|
|
|
return 1, [new_config]
|
2015-01-31 20:15:06 +00:00
|
|
|
return 0
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
# The only purpose of this function is to verify a working vty
|
|
|
|
def token_vty_command(vty):
|
|
|
|
vty.command("help")
|
2014-07-04 18:43:38 +00:00
|
|
|
return 0
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
# This may warn about the same doc missing multiple times, by design
|
|
|
|
def verify_doc(vty):
|
|
|
|
xml = vty.command("show online-help")
|
|
|
|
split_at = "<command"
|
|
|
|
all_errs = []
|
|
|
|
for command in xml.split(split_at):
|
|
|
|
if "(null)" in command:
|
|
|
|
lines = command.split("\n")
|
|
|
|
cmd_line = split_at + lines[0]
|
|
|
|
err_lines = []
|
|
|
|
for line in lines:
|
|
|
|
if '(null)' in line:
|
|
|
|
err_lines.append(line)
|
|
|
|
|
|
|
|
all_errs.append(err_lines)
|
|
|
|
|
2018-01-03 16:35:59 +00:00
|
|
|
print("Documentation error (missing docs): \n%s\n%s\n" % (
|
|
|
|
cmd_line, '\n'.join(err_lines)), file=sys.stderr)
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
return (len(all_errs), all_errs)
|
|
|
|
|
|
|
|
|
|
|
|
# Skip testing the configurations of anything that hasn't been compiled
|
|
|
|
def app_exists(app_desc):
|
2016-03-30 12:58:17 +00:00
|
|
|
cmd = app_desc[1].split(' ')[0]
|
2013-04-04 15:31:13 +00:00
|
|
|
return os.path.exists(cmd)
|
|
|
|
|
|
|
|
|
|
|
|
def remove_tmpdir(tmpdir):
|
2020-07-15 06:53:10 +00:00
|
|
|
shutil.rmtree(tmpdir)
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
|
2016-03-30 12:58:16 +00:00
|
|
|
def check_configs_tested(basedir, app_configs, ignore_configs):
|
2013-04-04 15:31:13 +00:00
|
|
|
configs = []
|
|
|
|
for root, dirs, files in os.walk(basedir):
|
|
|
|
for f in files:
|
2016-03-30 12:58:16 +00:00
|
|
|
if f.endswith(".cfg") and f not in ignore_configs:
|
2013-04-04 15:31:13 +00:00
|
|
|
configs.append(os.path.join(root, f))
|
|
|
|
for config in configs:
|
|
|
|
found = False
|
|
|
|
for app in app_configs:
|
|
|
|
if config in app_configs[app]:
|
|
|
|
found = True
|
|
|
|
if not found:
|
2018-01-03 16:35:59 +00:00
|
|
|
print("Warning: %s is not being tested" % config, file=sys.stderr)
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_all_apps(apps, app_configs, tmpdir="writtenconfig", verbose=True,
|
2016-03-30 12:58:16 +00:00
|
|
|
confpath=".", rmtmp=False, ignore_configs=[]):
|
|
|
|
check_configs_tested("doc/examples/", app_configs, ignore_configs)
|
2013-04-04 15:31:13 +00:00
|
|
|
errors = 0
|
|
|
|
for app in apps:
|
|
|
|
if not app_exists(app):
|
2018-01-03 16:35:59 +00:00
|
|
|
print("Skipping app %s (not found)" % app[1], file=sys.stderr)
|
2013-04-04 15:31:13 +00:00
|
|
|
continue
|
|
|
|
|
|
|
|
configs = app_configs[app[3]]
|
|
|
|
for config in configs:
|
2013-04-05 18:19:17 +00:00
|
|
|
config = os.path.join(confpath, config)
|
2017-02-27 00:06:44 +00:00
|
|
|
errors += test_config(app, config, tmpdir, verbose)
|
2013-04-04 15:31:13 +00:00
|
|
|
|
2013-04-05 18:19:17 +00:00
|
|
|
if rmtmp or not errors:
|
2013-04-04 15:31:13 +00:00
|
|
|
remove_tmpdir(tmpdir)
|
|
|
|
|
2017-02-27 00:06:44 +00:00
|
|
|
if errors:
|
2018-01-03 16:35:59 +00:00
|
|
|
print("ERRORS: %d" % errors, file=sys.stderr)
|
2013-04-04 15:31:13 +00:00
|
|
|
return errors
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
import argparse
|
|
|
|
|
|
|
|
confpath = "."
|
2013-04-05 18:19:17 +00:00
|
|
|
wordir = "."
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
parser = argparse.ArgumentParser()
|
|
|
|
parser.add_argument("--e1nitb", action="store_true", dest="e1nitb")
|
|
|
|
parser.add_argument("-v", "--verbose", dest="verbose",
|
|
|
|
action="store_true", help="verbose mode")
|
|
|
|
parser.add_argument("-p", "--pythonconfpath", dest="p",
|
|
|
|
help="searchpath for config")
|
2013-04-05 18:19:17 +00:00
|
|
|
parser.add_argument("-w", "--workdir", dest="w",
|
|
|
|
help="Working directory to run in")
|
|
|
|
|
2013-04-04 15:31:13 +00:00
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
|
|
if args.p:
|
|
|
|
confpath = args.p
|
|
|
|
|
2013-04-05 18:19:17 +00:00
|
|
|
if args.w:
|
|
|
|
workdir = args.w
|
|
|
|
|
2013-04-05 19:34:52 +00:00
|
|
|
osmoappdesc = osmoutil.importappconf_or_quit(confpath, "osmoappdesc",
|
|
|
|
args.p)
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
apps = osmoappdesc.apps
|
|
|
|
configs = osmoappdesc.app_configs
|
2016-03-30 12:58:16 +00:00
|
|
|
ignores = getattr(osmoappdesc, 'ignore_configs', [])
|
2013-04-04 15:31:13 +00:00
|
|
|
|
|
|
|
if args.e1nitb:
|
|
|
|
configs['nitb'].extend(osmoappdesc.nitb_e1_configs)
|
|
|
|
|
2013-04-05 18:19:17 +00:00
|
|
|
os.chdir(workdir)
|
2016-03-30 12:58:16 +00:00
|
|
|
sys.exit(test_all_apps(apps, configs, ignore_configs=ignores,
|
|
|
|
confpath=confpath, verbose=args.verbose))
|