check_dependencies: Import modules dynamically and find related debian packages
This way we don't need to manually add new imports here or drop unusued
ones. It also makes sure local imports in all our py files is correct.
For instance, running the script already caught an issue which is added
to this patch (osmo_ms_driver/__main__.py).
This new version of the script also allows specifying subsets of
features to skip when checking for dependencies. This way, for instance
somebody not willing to use a sispm powersupply can stil check all the
needed dependencies are fine.
This new tool will make it easier to slowly make some dependencies only
used by some object test classes optional (for instance, python-smpplib
if user doesn't want to run an ESME node).
It also allows to retrieve the required debian/manually installed
packages when run with "-p" option:
"""
Debian packages:
libpython3.5-minimal:amd64
python3-gi
python3-six
libpython3.5-stdlib:amd64
python3-pygments
python3-yaml
python3-mako
python3-numpy
python3-markupsafe
Modules without debian package (pip or setuptools?):
usb [dpkg-query: no path found matching pattern /usr/local/lib/python3.5/dist-packages/usb/_interop.py]
pydbus [dpkg-query: no path found matching pattern /usr/local/lib/python3.5/dist-packages/pydbus/proxy.py]
smpplib [dpkg-query: no path found matching pattern /usr/local/lib/python3.5/dist-packages/smpplib/command_codes.py]
sispm [dpkg-query: no path found matching pattern /usr/local/lib/python3.5/dist-packages/sispm/__init__.py]
"""
Change-Id: I29ddf8971837754abd930d847bd1036e8e510de6
This commit is contained in:
parent
2b959580b9
commit
045245d5ae
|
|
@ -3,31 +3,128 @@
|
|||
# just import all python3 modules used by osmo-gsm-tester to make sure they are
|
||||
# installed.
|
||||
|
||||
from inspect import getframeinfo, stack
|
||||
from mako.lookup import TemplateLookup
|
||||
from mako.template import Template
|
||||
import argparse
|
||||
import contextlib
|
||||
import copy
|
||||
import difflib
|
||||
import fcntl
|
||||
import inspect
|
||||
import io
|
||||
import os
|
||||
import pprint
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
import yaml
|
||||
import pydbus
|
||||
import sqlite3
|
||||
import sispm
|
||||
import smpplib
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree
|
||||
import numpy
|
||||
|
||||
print('dependencies ok')
|
||||
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import pprint
|
||||
import subprocess
|
||||
|
||||
feature_module_map = {
|
||||
'powersupply_intellinet' : ['powersupply_intellinet'],
|
||||
'powersupply_sispm' : ['powersupply_sispm'],
|
||||
'rfemu_amarisoftctrl': ['rfemu_amarisoftctrl'],
|
||||
'rfemu_minicircuits': ['rfemu_minicircuits'],
|
||||
}
|
||||
|
||||
def skip_features_to_skip_modules(skip_features):
|
||||
skip_obj_modules = []
|
||||
|
||||
for skip_feature in skip_features:
|
||||
if skip_feature not in feature_module_map:
|
||||
raise Exception('feature %s doesn\'t exist!' % skip_feature)
|
||||
for skip_module in feature_module_map[skip_feature]:
|
||||
skip_obj_modules.append(skip_module)
|
||||
return skip_obj_modules
|
||||
|
||||
def import_runtime_dependencies():
|
||||
# we don't have any right now, but in the future if we import a module during runtime (eg inside a function), then we need to place it here:
|
||||
# import foobar
|
||||
pass
|
||||
|
||||
def import_all_py_in_dir(rel_path, skip_modules=[]):
|
||||
selfdir = os.path.dirname(os.path.abspath(__file__))
|
||||
dir = os.path.join(selfdir, rel_path)
|
||||
print('importing files in directory %s' % dir)
|
||||
for entry in os.listdir(dir):
|
||||
full_entry = os.path.join(selfdir, rel_path, entry)
|
||||
if not os.path.isfile(full_entry):
|
||||
if args.verbose:
|
||||
print('skipping entry %s' % full_entry)
|
||||
continue
|
||||
if not full_entry.endswith('.py'):
|
||||
if args.verbose:
|
||||
print('skipping file %s' % full_entry)
|
||||
continue
|
||||
modulename = entry[:-3]
|
||||
if modulename in skip_modules:
|
||||
if args.verbose:
|
||||
print('skipping module %s' % modulename)
|
||||
continue
|
||||
modulepath = rel_path.replace('/', '.') + '.' + modulename
|
||||
print('importing %s' % modulepath)
|
||||
__import__(modulepath, globals(), locals())
|
||||
|
||||
def get_module_names():
|
||||
all_modules=sys.modules.items()
|
||||
all_modules_filtered = {}
|
||||
for mname, m in all_modules:
|
||||
if not hasattr(m, '__file__'):
|
||||
continue # skip built-in modules
|
||||
if mname.startswith('_'):
|
||||
continue # skip internal modules
|
||||
if mname.startswith('src.osmo_') or 'osmo_gsm_tester' in mname or 'osmo_ms_driver' in mname:
|
||||
continue # skip our own local modules
|
||||
mname = mname.split('.')[0] # store only main module
|
||||
if m not in all_modules_filtered.values():
|
||||
all_modules_filtered[mname] = m
|
||||
return all_modules_filtered
|
||||
|
||||
def print_deb_packages(modules):
|
||||
packages_deb = []
|
||||
modules_err = []
|
||||
for mname, m in modules.items():
|
||||
proc = subprocess.Popen(["dpkg", "-S", m.__file__], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
outs, errs = proc.communicate()
|
||||
if args.verbose:
|
||||
print('out: %s, err: %s' %(outs, errs))
|
||||
if len(errs): # error -> package not found (installed through pip?)
|
||||
modules_err.append((mname, errs.decode('utf-8')))
|
||||
elif len(outs):
|
||||
outs = outs.decode('utf-8')
|
||||
outs = outs.split()[0].rstrip(':') # first part is debian package name
|
||||
if not outs in packages_deb:
|
||||
packages_deb.append(outs)
|
||||
else:
|
||||
print('WARNING: dpkg returns empty!')
|
||||
|
||||
print('Debian packages:')
|
||||
for pkgname in packages_deb:
|
||||
print("\t" + pkgname)
|
||||
print()
|
||||
print('Modules without debian package (pip or setuptools?):')
|
||||
for mname, err in modules_err:
|
||||
print("\t" + mname.ljust(20) + " [" + err.rstrip() +"]")
|
||||
|
||||
parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter)
|
||||
parser.add_argument('-s', '--skip-feature', dest='skip_features', choices=feature_module_map.keys(), action='append',
|
||||
help='''All osmo-gsm-tester features not used by the user running the script''')
|
||||
parser.add_argument('-p', '--distro-packages', dest='distro_packages', action='store_true',
|
||||
help='Print distro packages installing modules')
|
||||
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
|
||||
help='Print a lot more information')
|
||||
args = parser.parse_args()
|
||||
|
||||
skip_obj_modules = skip_features_to_skip_modules(list(args.skip_features or []))
|
||||
|
||||
print('Skip checking modules: %r' % skip_obj_modules)
|
||||
|
||||
# We need to add it for cross-references between osmo_ms_driver and osmo_gsm_tester to work:
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 'src/'))
|
||||
import_all_py_in_dir('src/osmo_ms_driver')
|
||||
import_all_py_in_dir('src/osmo_gsm_tester/core')
|
||||
import_all_py_in_dir('src/osmo_gsm_tester/obj', skip_obj_modules)
|
||||
import_all_py_in_dir('src/osmo_gsm_tester')
|
||||
import_runtime_dependencies()
|
||||
print('Importing dependencies ok, all installed')
|
||||
|
||||
print('Retreiving list of imported modules...')
|
||||
modules = get_module_names()
|
||||
if args.verbose:
|
||||
for mname, m in modules.items():
|
||||
print('%s --> %s' %(mname, m.__file__))
|
||||
|
||||
if args.distro_packages:
|
||||
print('Generating distro package list from imported module list...')
|
||||
print_deb_packages(modules)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ from .cdf import cdfs
|
|||
from .starter import BinaryOptions, MobileTestStarter
|
||||
from .test_support import imsi_ki_gen
|
||||
from osmo_gsm_tester.core import log, util
|
||||
from osmo_gsm_tester import ms_osmo_mobile
|
||||
from osmo_gsm_tester.obj import ms_osmo_mobile
|
||||
|
||||
# System modules
|
||||
from datetime import timedelta
|
||||
|
|
|
|||
Loading…
Reference in New Issue