oeqa: Test failure/cleanup improvements

Currently, if qemu segfaults, the tests merrily continue trying to execute
which takes time for them to timeout and is a bit silly. Worse, no logs about
the segfault are shown to the user, its silent!

This patch tries to unravel the tangled web of issues and ensures that we:

* install a SIGCHLD handler which tells the user qemu exited
* check if qemu is running, if it isn't fail the test outright
* don't leave processes behind in sshcontrol which would hold
  bitbake.lock and block shutdown

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

4 files changed, 43 insertions, 12 deletions

diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py
index 9724325083..ff62c30268 100644
--- a/meta/lib/oeqa/oetest.py
+++ b/meta/lib/oeqa/oetest.py
@@ -144,6 +144,9 @@ class oeRuntimeTest(oeTest):
         self.target = oeRuntimeTest.tc.target
         super(oeRuntimeTest, self).__init__(methodName)
 
+    def setUp(self):
+        self.assertTrue(self.target.check(), msg = "Qemu not running?")
+
     def tearDown(self):
         # If a test fails or there is an exception
         if not exc_info() == (None, None, None):
diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py
index 542e259112..edc0d01c1e 100644
--- a/meta/lib/oeqa/targetcontrol.py
+++ b/meta/lib/oeqa/targetcontrol.py
@@ -188,6 +188,9 @@ class QemuTarget(BaseTarget):
                     bb.error("Qemu log output from %s:\n%s" % (self.qemulog, f.read()))
             raise bb.build.FuncFailed("%s - FAILED to start qemu - check the task log and the boot log" % self.pn)
 
+    def check(self):
+        return self.runner.is_alive()
+
     def stop(self):
         self.runner.stop()
         self.connection = None
diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py
index 3ad747a503..2d485379e8 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -64,6 +64,22 @@ class QemuRunner:
             with open(self.logfile, "a") as f:
                 f.write("%s" % msg)
 
+    def getOutput(self, o):
+        import fcntl
+        fl = fcntl.fcntl(o, fcntl.F_GETFL)
+        fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK)
+        return os.read(o.fileno(), 1000000)
+
+
+    def handleSIGCHLD(self, signum, frame):
+        if self.runqemu and self.runqemu.poll():
+            if self.runqemu.returncode:
+                logger.info('runqemu exited with code %d' % self.runqemu.returncode)
+                logger.info("Output from runqemu:\n%s" % self.getOutput(self.runqemu.stdout))
+                self.stop()
+                self._dump_host()
+                raise SystemExit
+
     def start(self, qemuparams = None):
         if self.display:
             os.environ["DISPLAY"] = self.display
@@ -98,11 +114,8 @@ class QemuRunner:
         if qemuparams:
             self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"'
 
-        def getOutput(o):
-            import fcntl
-            fl = fcntl.fcntl(o, fcntl.F_GETFL)
-            fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK)
-            return os.read(o.fileno(), 1000000)
+        self.origchldhandler = signal.getsignal(signal.SIGCHLD)
+        signal.signal(signal.SIGCHLD, self.handleSIGCHLD)
 
         launch_cmd = 'runqemu tcpserial=%s %s %s %s' % (self.serverport, self.machine, self.rootfs, self.qemuparams)
         # FIXME: We pass in stdin=subprocess.PIPE here to work around stty
@@ -122,7 +135,7 @@ class QemuRunner:
                     logger.info('runqemu exited with code %d' % self.runqemu.returncode)
                     self._dump_host()
                     self.stop()
-                    logger.info("Output from runqemu:\n%s" % getOutput(output))
+                    logger.info("Output from runqemu:\n%s" % self.getOutput(output))
                     return False
             time.sleep(1)
 
@@ -139,7 +152,7 @@ class QemuRunner:
                     self.ip = ips[0]
                     self.server_ip = ips[1]
             except IndexError, ValueError:
-                logger.info("Couldn't get ip from qemu process arguments! Here is the qemu command line used:\n%s\nand output from runqemu:\n%s" % (cmdline, getOutput(output)))
+                logger.info("Couldn't get ip from qemu process arguments! Here is the qemu command line used:\n%s\nand output from runqemu:\n%s" % (cmdline, self.getOutput(output)))
                 self._dump_host()
                 self.stop()
                 return False
@@ -154,7 +167,7 @@ class QemuRunner:
                 logger.error("Didn't receive a console connection from qemu. "
                              "Here is the qemu command line used:\n%s\nand "
                              "output from runqemu:\n%s" % (cmdline,
-                                                           getOutput(output)))
+                                                           self.getOutput(output)))
                 self.stop_thread()
                 return False
 
@@ -213,15 +226,15 @@ class QemuRunner:
             logger.info("Qemu pid didn't appeared in %s seconds" % self.runqemutime)
             self._dump_host()
             self.stop()
-            logger.info("Output from runqemu:\n%s" % getOutput(output))
+            logger.info("Output from runqemu:\n%s" % self.getOutput(output))
             return False
 
         return self.is_alive()
 
     def stop(self):
-
         self.stop_thread()
         if self.runqemu:
+            signal.signal(signal.SIGCHLD, self.origchldhandler)
             logger.info("Sending SIGTERM to runqemu")
             try:
                 os.killpg(self.runqemu.pid, signal.SIGTERM)
@@ -255,6 +268,8 @@ class QemuRunner:
         return False
 
     def is_alive(self):
+        if not self.runqemu:
+            return False
         qemu_child = self.find_child(str(self.runqemu.pid))
         if qemu_child:
             self.qemupid = qemu_child[0]
diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py
index 6ed48badc8..00f5051053 100644
--- a/meta/lib/oeqa/utils/sshcontrol.py
+++ b/meta/lib/oeqa/utils/sshcontrol.py
@@ -42,7 +42,7 @@ class SSHProcess(object):
             with open(self.logfile, "a") as f:
                f.write("%s" % msg)
 
-    def run(self, command, timeout=None, logfile=None):
+    def _run(self, command, timeout=None, logfile=None):
         self.logfile = logfile
         self.starttime = time.time()
         output = ''
@@ -79,8 +79,18 @@ class SSHProcess(object):
 
         self.status = self.process.wait()
         self.output = output.rstrip()
-        return (self.status, self.output)
 
+    def run(self, command, timeout=None, logfile=None):
+        try:
+            self._run(command, timeout, logfile)
+        except:
+            # Need to guard against a SystemExit or other exception occuring whilst running
+            # and ensure we don't leave a process behind.
+            if self.process.poll() is None:
+                self.process.kill()
+                self.status = self.process.wait()
+            raise
+        return (self.status, self.output)
 
 class SSHControl(object):
     def __init__(self, ip, logfile=None, timeout=300, user='root', port=None):