From 151b08a4108342b9873d6028e7f4751501f506d8 Mon Sep 17 00:00:00 2001 From: Pau Espin Pedrol Date: Mon, 2 Mar 2020 14:14:27 +0100 Subject: [PATCH] srsue: Introduce metrics verification procedures Change-Id: Ib1da58615cdc4f53ac1a27080e94e5b47760c508 --- check_dependencies.py | 1 + src/osmo_gsm_tester/srs_enb.py | 35 +++++++++++++- src/osmo_gsm_tester/srs_ue.py | 84 ++++++++++++++++++++++++++++++++-- suites/4g/iperf3.py | 6 ++- suites/4g/suite.conf | 3 ++ 5 files changed, 122 insertions(+), 7 deletions(-) diff --git a/check_dependencies.py b/check_dependencies.py index 28bfdf7a..c3b1d64a 100755 --- a/check_dependencies.py +++ b/check_dependencies.py @@ -28,5 +28,6 @@ import sispm import smpplib import urllib.request import xml.etree.ElementTree +import numpy print('dependencies ok') diff --git a/src/osmo_gsm_tester/srs_enb.py b/src/osmo_gsm_tester/srs_enb.py index 1cfd212b..1fb2db15 100644 --- a/src/osmo_gsm_tester/srs_enb.py +++ b/src/osmo_gsm_tester/srs_enb.py @@ -76,6 +76,7 @@ class srsENB(log.Origin): self.remote_config_drb_file = None self.remote_log_file = None self._num_prb = 0 + self._txmode = 0 self.suite_run = suite_run self.remote_user = conf.get('remote_user', None) if not rf_type_valid(conf.get('rf_dev_type', None)): @@ -179,10 +180,13 @@ class srsENB(log.Origin): config.overlay(values, dict(enb=self._conf)) config.overlay(values, dict(enb={ 'mme_addr': self.epc.addr() })) + self._num_prb = int(values['enb'].get('num_prb', None)) + assert self._num_prb + self._txmode = int(values['enb'].get('transmission_mode', None)) + assert self._txmode + # We need to set some specific variables programatically here to match IP addresses: if self._conf.get('rf_dev_type') == 'zmq': - self._num_prb = int(values['enb'].get('num_prb', None)) - assert self._num_prb base_srate = num_prb2base_srate(self._num_prb) rf_dev_args = 'fail_on_disconnect=true,tx_port=tcp://' + self.addr() \ + ':2000,rx_port=tcp://' + self.ue.addr() \ @@ -222,4 +226,31 @@ class srsENB(log.Origin): def num_prb(self): return self._num_prb + def ue_max_rate(self, downlink=True): + # The max rate for a single UE per PRB in TM1 + max_phy_rate_tm1_dl = { 6 : 2.3e6, + 15 : 8e6, + 25 : 16e6, + 50 : 36e6, + 75 : 54e6, + 100 : 75e6 } + # TODO: proper values for this table: + max_phy_rate_tm1_ul = { 6 : 0.23e6, + 15 : 0.8e6, + 25 : 1.6e6, + 50 : 3.6e6, + 75 : 5.4e6, + 100 : 7.5e6 } + if downlink: + max_rate = max_phy_rate_tm1_dl[self.num_prb()] + else: + max_rate = max_phy_rate_tm1_ul[self.num_prb()] + #TODO: calculate for non-standard prb numbers. + if self._txmode > 2: + max_rate *= 2 + # We use 3 control symbols for 6, 15 and 25 PRBs which results in lower max rate + if self.num_prb() < 50: + max_rate *= 0.9 + return max_rate + # vim: expandtab tabstop=4 shiftwidth=4 diff --git a/src/osmo_gsm_tester/srs_ue.py b/src/osmo_gsm_tester/srs_ue.py index cbd8a68e..f90ea326 100644 --- a/src/osmo_gsm_tester/srs_ue.py +++ b/src/osmo_gsm_tester/srs_ue.py @@ -22,6 +22,7 @@ import pprint from . import log, util, config, template, process, remote from .run_node import RunNode +from .event_loop import MainLoop from .ms import MS def rf_type_valid(rf_type_str): @@ -91,10 +92,6 @@ class srsUE(MS): self.rem_host.scpfrom('scp-back-pcap', self.remote_pcap_file, self.pcap_file) except Exception as e: self.log(repr(e)) - try: - self.rem_host.scpfrom('scp-back-metrics', self.remote_metrics_file, self.metrics_file) - except Exception as e: - self.log(repr(e)) def setup_runs_locally(self): return self.remote_user is None @@ -102,6 +99,9 @@ class srsUE(MS): def netns(self): return "srsue1" + def stop(self): + self.suite_run.stop_process(self.process) + def connect(self, enb): self.log('Starting srsue') self.enb = enb @@ -247,4 +247,80 @@ class srsUE(MS): proc = self.rem_host.RemoteNetNSProcess(name, self.netns(), popen_args, env={}) proc.launch_sync() + def verify_metric(self, value, operation='avg', metric='dl_brate', criterion='gt'): + # file is not properly flushed until the process has stopped. + if self.running(): + self.stop() + # metrics file is not flushed immediatelly by the OS during process + # tear down, we need to wait some extra time: + MainLoop.sleep(self, 2) + if not self.setup_runs_locally(): + try: + self.rem_host.scpfrom('scp-back-metrics', self.remote_metrics_file, self.metrics_file) + except Exception as e: + self.err('Failed copying back metrics file from remote host') + raise e + metrics = srsUEMetrics(self.metrics_file) + return metrics.verify(value, operation, metric, criterion) + +import numpy + +class srsUEMetrics(log.Origin): + + VALID_OPERATIONS = ['avg', 'sum'] + VALID_CRITERION = ['eq','gt','lt'] + CRITERION_TO_SYM = { 'eq' : '==', 'gt' : '>', 'lt' : '<' } + CRYTERION_TO_SYM_OPPOSITE = { 'eq' : '!=', 'gt' : '<=', 'lt' : '>=' } + + + def __init__(self, metrics_file): + super().__init__(log.C_RUN, 'srsue_metrics') + self.raw_data = None + self.metrics_file = metrics_file + # read CSV, guessing data type with first row being the legend + try: + self.raw_data = numpy.genfromtxt(self.metrics_file, names=True, delimiter=';', dtype=None) + except (ValueError, IndexError, IOError) as error: + self.err("Error parsing metrics CSV file %s" % self.metrics_file) + raise error + + def verify(self, value, operation='avg', metric='dl_brate', criterion='gt'): + if operation not in self.VALID_OPERATIONS: + raise log.Error('Unknown operation %s not in %r' % (operation, self.VALID_OPERATIONS)) + if criterion not in self.VALID_CRITERION: + raise log.Error('Unknown operation %s not in %r' % (operation, self.VALID_CRITERION)) + # check if given metric exists in data + try: + sel_data = self.raw_data[metric] + except ValueError as err: + print('metric %s not available' % metric) + raise err + + if operation == 'avg': + result = numpy.average(sel_data) + elif operation == 'sum': + result = numpy.sum(sel_data) + self.dbg(result=result, value=value) + + success = False + if criterion == 'eq' and result == value or \ + criterion == 'gt' and result > value or \ + criterion == 'lt' and result < value: + success = True + + # Convert bitrate in Mbit/s: + if metric.find('brate') > 0: + result /= 1e6 + value /= 1e6 + mbit_str = ' Mbit/s' + else: + mbit_str = '' + + if not success: + result_msg = "{:.2f}{} {} {:.2f}{}".format(result, mbit_str, self.CRYTERION_TO_SYM_OPPOSITE[criterion], value, mbit_str) + raise log.Error(result_msg) + result_msg = "{:.2f}{} {} {:.2f}{}".format(result, mbit_str, self.CRITERION_TO_SYM[criterion], value, mbit_str) + # TODO: overwrite test system-out with this text. + return result_msg + # vim: expandtab tabstop=4 shiftwidth=4 diff --git a/suites/4g/iperf3.py b/suites/4g/iperf3.py index 371376f0..20489b51 100755 --- a/suites/4g/iperf3.py +++ b/suites/4g/iperf3.py @@ -32,7 +32,7 @@ print('ENB is connected to EPC') ue.connect(enb) iperf3srv.start() -proc = iperf3cli.prepare_test_proc(False, ue.netns()) +proc = iperf3cli.prepare_test_proc(False, ue.netns(), time_sec=60) print('waiting for UE to attach...') wait(ue.is_connected, None) @@ -42,3 +42,7 @@ print("Running iperf3 client to %s through %s" % (str(iperf3cli), ue.netns())) proc.launch_sync() iperf3srv.stop() print_results(iperf3cli.get_results(), iperf3srv.get_results()) + +max_rate = enb.ue_max_rate(downlink=False) +res_str = ue.verify_metric(max_rate * 0.9, operation='avg', metric='ul_brate', criterion='gt') +print(res_str + '\n') diff --git a/suites/4g/suite.conf b/suites/4g/suite.conf index 352293ab..59e393aa 100644 --- a/suites/4g/suite.conf +++ b/suites/4g/suite.conf @@ -7,3 +7,6 @@ resources: modem: - times: 1 type: srsue + +defaults: + timeout: 180s