Handle termination signals to exit gracefully and prevent resource leak

Make sure we free the reserved resources and kill launched subprocesses
before stopping. Before this patch it was not the case for instance if we
received a SIGTREM signal from kill.

Change-Id: I039e4d1908a04bf606b101ddc6a186ba67e6178e

src/osmo-gsm-tester.py

@@ -68,11 +68,23 @@ created, which will collect logs and reports.
 
 import sys
 import argparse
+from signal import *
 from osmo_gsm_tester import __version__
 from osmo_gsm_tester import trial, suite, log, config
 
+def sig_handler_cleanup(signum, frame):
+    print("killed by signal %d" % signum)
+    # This sys.exit() will raise a SystemExit base exception at the current
+    # point of execution. Code must be prepared to clean system-wide resources
+    # by using the "finally" section. This allows at the end 'atexit' hooks to
+    # be called before exiting.
+    sys.exit(1)
+
 def main():
 
+    for sig in (SIGINT, SIGTERM, SIGQUIT, SIGPIPE, SIGHUP):
+        signal(sig, sig_handler_cleanup)
+
     parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter)
     # Note: since we're using RawTextHelpFormatter to keep nicely separate
     # paragraphs in the long help text, we unfortunately also need to take care
@@ -184,7 +196,8 @@ optional.''')
                 if status == trial.Trial.FAIL:
                     any_failed = True
                 trials_run.append(current_trial)
-        except:
+        except Exception:
+            # Do not catch here subclasses of BaseException such as SystemExit, let them finish the program
             current_trial.log_exn()
 
     sys.stderr.flush()

src/osmo_gsm_tester/suite.py

@@ -223,21 +223,26 @@ class SuiteRun(log.Origin):
 
     def run_tests(self, names=None):
         self.log('Suite run start')
-        self.mark_start()
-        if not self.reserved_resources:
-            self.reserve_resources()
-        for test in self.definition.tests:
-            if names and not test.name() in names:
-                test.set_skip()
-                self.test_skipped_ctr += 1
-                self.tests.append(test)
-                continue
-            with self:
-                st = test.run(self)
-                if st == Test.FAIL:
-                    self.test_failed_ctr += 1
-                self.tests.append(test)
-        self.stop_processes()
+        try:
+            self.mark_start()
+            if not self.reserved_resources:
+                self.reserve_resources()
+            for test in self.definition.tests:
+                if names and not test.name() in names:
+                    test.set_skip()
+                    self.test_skipped_ctr += 1
+                    self.tests.append(test)
+                    continue
+                with self:
+                    st = test.run(self)
+                    if st == Test.FAIL:
+                        self.test_failed_ctr += 1
+                    self.tests.append(test)
+        finally:
+            # if sys.exit() called from signal handler (e.g. SIGINT), SystemExit
+            # base exception is raised. Make sure to stop processes in this
+            # finally section. Resources are automatically freed with 'atexit'.
+            self.stop_processes()
         self.duration = time.time() - self.start_timestamp
         if self.test_failed_ctr:
             self.status = SuiteRun.FAIL