Support identifying different tgz files based on run_label attribute

tgz files in trials can be categorized in subdirectories, allowing to
select different bianry files at runtime based on the target run node
which is going to run them. This way for instance one can have a binary
linked against libs for eg. CentOS under run_label "centos/" or an ARM
target under "arm", and then use "run_label: arm" on the resource using
it.

Change-Id: Iaf2e97da3aff693395f44f0e93b184d4846cf6da

.gitignore

@@ -4,13 +4,16 @@ pid
 .version
 _version.py
 tags
-set_pythonpath
-test_work
 state
 *.pyc
 sysmocom/resources.conf
 sysmocom/ttcn3/resources.conf
 
+# selftest:
+set_pythonpath
+test_work
+selftest/trial_test/run_label/inst/
+
 # manuals
 doc/manuals/*.html
 doc/manuals/*.svg

doc/manuals/chapters/trial.adoc

@@ -48,13 +48,22 @@ The script in 'contrib/jenkins-run.sh' takes care of related tasks such as
 
 {app-name} tests create objects to manage the allocated resources during test
 lifetime. These objects, in turn, usually run and manage processes started from
-the trail's sysroot binaries. {app-name} provide APIs for those object classes
+the trial's sysroot binaries. {app-name} provide APIs for those object classes
 to discover, unpack and run those binaries. An object class simply needs to
 request the name of the sysroot it wants to use (for instance 'osmo-bsc'), and
 {app-name} will take care of preparing everything and providing the sysroot path
 to it. It's a duty of the resource class to copy over the sysroot to the
 destination if the intention is to run the binary remotely on another host.
 
+Moreover, {app-name} supports working with several different versions of a given
+sysroot by means of storing them in different subdirectories, which are later
+referenced by an object's class 'run_label' attribute named after that
+subdirectory. This way, for instance, a sysroot can be provided containing
+binaries linked against libraries present on a CentOS distribution, and other
+sysroots with the same name can also be provided which are linked against
+different versions of CentOS, or a different distro like Debian, or even a
+different arch like ARM.
+
 When seeking a sysroot of a given name '<inst-name>' in the 'inst/' directory,
 {app-name} will look for 'tgz' files starting with the pattern '<inst-name>.'
 (up to the first dot). That means, suffixes are available for {app-name} user to
@@ -63,3 +72,6 @@ commit hash. Hence, these example files are considered valid and will be
 selected by {app-name} for 'osmo-bsc': 'osmo-bsc.tgz', 'osmo-bsc.build-23.tgz',
 'osmo-bsc.5f3e0dd2.tgz', 'osmo-bsc.armv7.build-2.tgz'. If either none or more
 than one valid file is found matching the pattern, an exception will be thrown.
+If a 'run_label=foobar' is provided, {app-name} will look up for the 'tgz' in
+exactly the same way, but this time in 'inst/foobar/' directory instead of
+'inst/'.

selftest/Makefile

@@ -12,6 +12,7 @@ set_pythonpath:
 clean:
 	@find . -name "*__pycache__" -type d -print0 | xargs -0 rm -rvf
 	@find . -name "*test_work" -type d -print0 | xargs -0 rm -rvf
+	@rm -rfv ./trial_test/run_label/inst
 	@rm -fv ./set_pythonpath
 
 # vim: noexpandtab tabstop=8 shiftwidth=8

selftest/trial_test/run_label/checksums.md5

@@ -0,0 +1,2 @@
+b13c9c94d41c45caaa08ad9ad3ee0f11  sample.tar.gz
+430ec62329747b5f15fe6e2cb141b909  foobar/sample.tar.gz

selftest/trial_test/trial_test.ok

@@ -14,3 +14,12 @@ None
 ok, got RuntimeError: Checksum mismatch for '[PATH]/trial_test/invalid_checksum/file2' vs. '[PATH]/trial_test/invalid_checksum/checksums.md5' line 2
 - detect missing file
 ok, got RuntimeError: File listed in checksums file but missing in trials dir: '[PATH]/trial_test/missing_file/file2' vs. '[PATH]/trial_test/missing_file/checksums.md5' line 2
+- Verify trials based on run_label
+tst run_label: DBG: has bin_tar {bin_name='sample', matches=['sample.tar.gz'], run_label='foobar'}
+inst: [PATH]/trial_test/run_label/inst/foobar/sample
+content file2: subhello
+
+tst run_label: DBG: has bin_tar {bin_name='sample', matches=['sample.tar.gz'], run_label=''}
+inst: [PATH]/trial_test/run_label/inst/sample
+content file1: hello
+

selftest/trial_test/trial_test.ok.ign

@@ -1,3 +1,3 @@
 /tmp/[^/]*	[TMP]
 ....-..-.._..-..-..	[TIMESTAMP]
-'[^']*/trial_test	'[PATH]/trial_test
+/[^ ]*/trial_test/	[PATH]/trial_test/

selftest/trial_test/trial_test.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 
-import time
 import _prep
+
+import time
 import os
+
 from osmo_gsm_tester.core import util
 from osmo_gsm_tester.core.trial import Trial
 
@@ -46,4 +48,17 @@ try:
 except RuntimeError as e:
     print('ok, got RuntimeError: %s' % str(e))
 
+print('- Verify trials based on run_label')
+d = util.Dir('trial_test')
+t = Trial(d.child('run_label'))
+t.verify()
+inst = util.Dir(t.get_inst('sample', 'foobar'))
+print('inst: ' + str(inst))
+with open(inst.child('file2'), 'r') as f:
+    print('content file2: %s' % f.read())
+inst = util.Dir( t.get_inst('sample'))
+print('inst: ' + str(inst))
+with open(inst.child('file1'), 'r') as f:
+    print('content file1: %s' % f.read())
+
 # vim: expandtab tabstop=4 shiftwidth=4

src/osmo_gsm_tester/core/trial.py

@@ -57,7 +57,7 @@ class Trial(log.Origin):
         super().__init__(log.C_TST, os.path.basename(self.path))
         self.dir = util.Dir(self.path)
         self.inst_dir = util.Dir(self.dir.child('inst'))
-        self.bin_tars = []
+        self.bin_tars = {}
         self.suites = []
         self.status = Trial.UNKNOWN
         self._run_dir = None
@@ -125,10 +125,10 @@ class Trial(log.Origin):
                 line_nr += 1
                 if not line:
                     continue
-                md5, filename = line.split('  ')
-                file_path = self.dir.child(filename)
+                md5, relpath = line.split('  ')
+                file_path = self.dir.child(relpath)
 
-                if not self.dir.isfile(filename):
+                if not self.dir.isfile(relpath):
                     raise RuntimeError('File listed in checksums file but missing in trials dir:'
                                        ' %r vs. %r line %d' % (file_path, checksums, line_nr))
 
@@ -136,28 +136,34 @@ class Trial(log.Origin):
                     raise RuntimeError('Checksum mismatch for %r vs. %r line %d'
                                        % (file_path, checksums, line_nr))
 
-                if filename.endswith('.tgz'):
-                    self.bin_tars.append(filename)
+                if relpath.endswith('.tgz') or relpath.endswith('.tar.gz'):
+                    (label, name) = os.path.split(relpath)
+                    #print('label: %s, name: %s' % (label, name))
+                    li = self.bin_tars.get(label, [])
+                    li.append(name)
+                    self.bin_tars[label] = li
 
-    def has_bin_tar(self, bin_name):
+    def has_bin_tar(self, bin_name, run_label):
         bin_tar_start = '%s.' % bin_name
-        matches = [t for t in self.bin_tars if t.startswith(bin_tar_start)]
-        self.dbg(bin_name=bin_name, matches=matches)
+        matches = [t for t in self.bin_tars[run_label] if t.startswith(bin_tar_start)]
+        self.dbg('has bin_tar', run_label=run_label, bin_name=bin_name, matches=matches)
         if not matches:
             return None
         if len(matches) > 1:
-            raise RuntimeError('More than one match for bin name %r: %r' % (bin_name, matches))
+            raise RuntimeError('More than one match for bin name %r on run_label \'%s\': %r' % (bin_name, run_label, matches))
         bin_tar = matches[0]
-        bin_tar_path = self.dir.child(bin_tar)
+        bin_tar_path = self.dir.child(os.path.join(run_label, bin_tar))
         if not os.path.isfile(bin_tar_path):
             raise RuntimeError('Not a file or missing: %r' % bin_tar_path)
         return bin_tar_path
 
-    def get_inst(self, bin_name):
-        bin_tar = self.has_bin_tar(bin_name)
+    def get_inst(self, bin_name, run_label=None):
+        if run_label is None:
+            run_label = ''
+        bin_tar = self.has_bin_tar(bin_name, run_label)
         if not bin_tar:
             raise RuntimeError('No such binary available: %r' % bin_name)
-        inst_dir = self.inst_dir.child(bin_name)
+        inst_dir = self.inst_dir.child(os.path.join(run_label, bin_name))
 
         if os.path.isdir(inst_dir):
             # already unpacked

src/osmo_gsm_tester/obj/run_node.py

@@ -26,6 +26,7 @@ def on_register_schemas():
         'run_addr': schema.IPV4,
         'ssh_user': schema.STR,
         'ssh_addr': schema.IPV4,
+        'run_label': schema.STR,
         }
     schema.register_resource_schema('run_node', resource_schema)
 
@@ -35,12 +36,13 @@ class RunNode(log.Origin):
     T_LOCAL = 'local'
     T_REM_SSH = 'ssh'
 
-    def __init__(self, type=None, run_addr=None, ssh_user=None, ssh_addr=None):
+    def __init__(self, type=None, run_addr=None, ssh_user=None, ssh_addr=None, run_label=None):
         super().__init__(log.C_RUN, 'runnode')
         self._type = type
         self._run_addr = run_addr
         self._ssh_user = ssh_user
         self._ssh_addr = ssh_addr
+        self._run_label = run_label
         if not self._type:
             raise log.Error('run_type not set')
         if not self._run_addr:
@@ -57,7 +59,9 @@ class RunNode(log.Origin):
 
     @classmethod
     def from_conf(cls, conf):
-        return cls(conf.get('run_type', None), conf.get('run_addr', None), conf.get('ssh_user', None), conf.get('ssh_addr', None))
+        return cls(conf.get('run_type', None), conf.get('run_addr', None),
+                   conf.get('ssh_user', None), conf.get('ssh_addr', None),
+                   conf.get('run_label', None))
 
     def is_local(self):
         return self._type == RunNode.T_LOCAL
@@ -77,4 +81,7 @@ class RunNode(log.Origin):
     def ssh_addr(self):
         return self._ssh_addr
 
+    def run_label(self):
+        return self._run_label
+
 # vim: expandtab tabstop=4 shiftwidth=4