srsue: Introduce metrics verification procedures

Change-Id: Ib1da58615cdc4f53ac1a27080e94e5b47760c508

check_dependencies.py

@@ -28,5 +28,6 @@ import sispm
 import smpplib
 import urllib.request
 import xml.etree.ElementTree
+import numpy
 
 print('dependencies ok')

src/osmo_gsm_tester/srs_enb.py

@@ -76,6 +76,7 @@ class srsENB(log.Origin):
         self.remote_config_drb_file = None
         self.remote_log_file = None
         self._num_prb = 0
+        self._txmode = 0
         self.suite_run = suite_run
         self.remote_user = conf.get('remote_user', None)
         if not rf_type_valid(conf.get('rf_dev_type', None)):
@@ -179,10 +180,13 @@ class srsENB(log.Origin):
         config.overlay(values, dict(enb=self._conf))
         config.overlay(values, dict(enb={ 'mme_addr': self.epc.addr() }))
 
+        self._num_prb = int(values['enb'].get('num_prb', None))
+        assert self._num_prb
+        self._txmode = int(values['enb'].get('transmission_mode', None))
+        assert self._txmode
+
         # We need to set some specific variables programatically here to match IP addresses:
         if self._conf.get('rf_dev_type') == 'zmq':
-            self._num_prb = int(values['enb'].get('num_prb', None))
-            assert self._num_prb
             base_srate = num_prb2base_srate(self._num_prb)
             rf_dev_args = 'fail_on_disconnect=true,tx_port=tcp://' + self.addr() \
                         + ':2000,rx_port=tcp://' + self.ue.addr() \
@@ -222,4 +226,31 @@ class srsENB(log.Origin):
     def num_prb(self):
         return self._num_prb
 
+    def ue_max_rate(self, downlink=True):
+        # The max rate for a single UE per PRB in TM1
+        max_phy_rate_tm1_dl = { 6 : 2.3e6,
+                               15 : 8e6,
+                               25 : 16e6,
+                               50 : 36e6,
+                               75 : 54e6,
+                               100 : 75e6 }
+        # TODO: proper values for this table:
+        max_phy_rate_tm1_ul = { 6 : 0.23e6,
+                               15 : 0.8e6,
+                               25 : 1.6e6,
+                               50 : 3.6e6,
+                               75 : 5.4e6,
+                               100 : 7.5e6 }
+        if downlink:
+            max_rate = max_phy_rate_tm1_dl[self.num_prb()]
+        else:
+            max_rate = max_phy_rate_tm1_ul[self.num_prb()]
+        #TODO: calculate for non-standard prb numbers.
+        if self._txmode > 2:
+            max_rate *= 2
+        # We use 3 control symbols for 6, 15 and 25 PRBs which results in lower max rate
+        if self.num_prb() < 50:
+          max_rate *= 0.9
+        return max_rate
+
 # vim: expandtab tabstop=4 shiftwidth=4

src/osmo_gsm_tester/srs_ue.py

@@ -22,6 +22,7 @@ import pprint
 
 from . import log, util, config, template, process, remote
 from .run_node import RunNode
+from .event_loop import MainLoop
 from .ms import MS
 
 def rf_type_valid(rf_type_str):
@@ -91,10 +92,6 @@ class srsUE(MS):
             self.rem_host.scpfrom('scp-back-pcap', self.remote_pcap_file, self.pcap_file)
         except Exception as e:
             self.log(repr(e))
-        try:
-            self.rem_host.scpfrom('scp-back-metrics', self.remote_metrics_file, self.metrics_file)
-        except Exception as e:
-            self.log(repr(e))
 
     def setup_runs_locally(self):
         return self.remote_user is None
@@ -102,6 +99,9 @@ class srsUE(MS):
     def netns(self):
         return "srsue1"
 
+    def stop(self):
+        self.suite_run.stop_process(self.process)
+
     def connect(self, enb):
         self.log('Starting srsue')
         self.enb = enb
@@ -247,4 +247,80 @@ class srsUE(MS):
             proc = self.rem_host.RemoteNetNSProcess(name, self.netns(), popen_args, env={})
         proc.launch_sync()
 
+    def verify_metric(self, value, operation='avg', metric='dl_brate', criterion='gt'):
+        # file is not properly flushed until the process has stopped.
+        if self.running():
+            self.stop()
+            # metrics file is not flushed immediatelly by the OS during process
+            # tear down, we need to wait some extra time:
+            MainLoop.sleep(self, 2)
+            if not self.setup_runs_locally():
+                try:
+                    self.rem_host.scpfrom('scp-back-metrics', self.remote_metrics_file, self.metrics_file)
+                except Exception as e:
+                    self.err('Failed copying back metrics file from remote host')
+                    raise e
+        metrics = srsUEMetrics(self.metrics_file)
+        return metrics.verify(value, operation, metric, criterion)
+
+import numpy
+
+class srsUEMetrics(log.Origin):
+
+    VALID_OPERATIONS = ['avg', 'sum']
+    VALID_CRITERION = ['eq','gt','lt']
+    CRITERION_TO_SYM = { 'eq' : '==', 'gt' : '>', 'lt' : '<' }
+    CRYTERION_TO_SYM_OPPOSITE = { 'eq' : '!=', 'gt' : '<=', 'lt' : '>=' }
+
+
+    def __init__(self, metrics_file):
+        super().__init__(log.C_RUN, 'srsue_metrics')
+        self.raw_data = None
+        self.metrics_file = metrics_file
+        # read CSV, guessing data type with first row being the legend
+        try:
+            self.raw_data = numpy.genfromtxt(self.metrics_file, names=True, delimiter=';', dtype=None)
+        except (ValueError, IndexError, IOError) as error:
+            self.err("Error parsing metrics CSV file %s" % self.metrics_file)
+            raise error
+
+    def verify(self, value, operation='avg', metric='dl_brate', criterion='gt'):
+        if operation not in self.VALID_OPERATIONS:
+            raise log.Error('Unknown operation %s not in %r' % (operation, self.VALID_OPERATIONS))
+        if criterion not in self.VALID_CRITERION:
+            raise log.Error('Unknown operation %s not in %r' % (operation, self.VALID_CRITERION))
+        # check if given metric exists in data
+        try:
+            sel_data = self.raw_data[metric]
+        except ValueError as err:
+            print('metric %s not available' % metric)
+            raise err
+
+        if operation == 'avg':
+            result = numpy.average(sel_data)
+        elif operation == 'sum':
+            result = numpy.sum(sel_data)
+        self.dbg(result=result, value=value)
+
+        success = False
+        if criterion == 'eq' and result == value or \
+           criterion == 'gt' and result > value or \
+           criterion == 'lt' and result < value:
+            success = True
+
+        # Convert bitrate in Mbit/s:
+        if metric.find('brate') > 0:
+            result /= 1e6
+            value /= 1e6
+            mbit_str = ' Mbit/s'
+        else:
+            mbit_str = ''
+
+        if not success:
+            result_msg = "{:.2f}{} {} {:.2f}{}".format(result, mbit_str, self.CRYTERION_TO_SYM_OPPOSITE[criterion], value, mbit_str)
+            raise log.Error(result_msg)
+        result_msg = "{:.2f}{} {} {:.2f}{}".format(result, mbit_str, self.CRITERION_TO_SYM[criterion], value, mbit_str)
+        # TODO: overwrite test system-out with this text.
+        return result_msg
+
 # vim: expandtab tabstop=4 shiftwidth=4

suites/4g/iperf3.py

@@ -32,7 +32,7 @@ print('ENB is connected to EPC')
 ue.connect(enb)
 
 iperf3srv.start()
-proc = iperf3cli.prepare_test_proc(False, ue.netns())
+proc = iperf3cli.prepare_test_proc(False, ue.netns(), time_sec=60)
 
 print('waiting for UE to attach...')
 wait(ue.is_connected, None)
@@ -42,3 +42,7 @@ print("Running iperf3 client to %s through %s" % (str(iperf3cli), ue.netns()))
 proc.launch_sync()
 iperf3srv.stop()
 print_results(iperf3cli.get_results(), iperf3srv.get_results())
+
+max_rate = enb.ue_max_rate(downlink=False)
+res_str = ue.verify_metric(max_rate * 0.9, operation='avg', metric='ul_brate', criterion='gt')
+print(res_str + '\n')

suites/4g/suite.conf

@@ -7,3 +7,6 @@ resources:
   modem:
   - times: 1
     type: srsue
+
+defaults:
+  timeout: 180s