qemu-testlib: Add python helper and simplify shell

The current code has a race since it greps for *any* qemu process
running, even if it isn't the one we started. This leads to some sanity
tests potentially failing on machines where multiple sets of sanity tests
are running.

To resovle this and some other ugly code issues, add a python script
to accurately walk the process tree and find the qemu process. We can
then replace all the shell functions attempting this which happen to
work in many cases but not all.

Also clean up some of the error handling so its more legible.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

3 files changed, 117 insertions, 158 deletions

diff --git a/scripts/qemuimage-testlib b/scripts/qemuimage-testlib
index f8a5299a25..2c22bf2fa6 100755
--- a/scripts/qemuimage-testlib
+++ b/scripts/qemuimage-testlib
@@ -25,8 +25,9 @@ TOOLCHAIN_PROJECTS="$COREBASE/scripts/qemuimage-tests/toolchain_projects"
 # Test Directory on target for testing
 TARGET_TEST_DIR="/opt/test"
 
-# Global variable for process id
-PID=0
+# Global variables for process id
+XTERMPID=0
+QEMUPID=0
 
 # Global variable for target ip address
 TARGET_IPADDR=0
@@ -212,90 +213,33 @@ Test_Print_Result()
 # $1 is qemu process id, which needs to be killed
 Test_Kill_Qemu()
 {
-	local ret=0
-	local ppid=0
-	local i=0
 	local index=0
 	local total=0
-	declare local pid
-
-	# Check if $1 pid exists and is a qemu process
-	ps -wwfp $PID | grep -iq "qemu"
-
-	# Find all children pid of the pid $1
-	if [ $? -eq 0 ]; then
-
-		# Check if there is any child pid of the pid $PID
-		ppid=$PID
-		ps -f --ppid $ppid
-		ret=$?
-
-		while [ $ret -eq 0 ]
-		do
-			# If yes, get the child pid and check if the child pid has other child pid
-			# Continue the while loop until there is no child pid found
-			pid[$i]=`ps -f --ppid $ppid | awk '{if ($2 != "PID") print $2}'`
-			ppid=${pid[$i]}
-			i=$((i+1))
-			ps -f --ppid $ppid
-			ret=$?
-		done
-
-		# When TEST_SERIALIZE is set, qemu process will not be
-		# killed until all the cases are finished
-		if [ ${TEST_SERIALIZE} -eq 1 -a -e ${TEST_STATUS} ]; then
-			index=`sed -n 2p ${TEST_STATUS} | awk '{print $3}'`
-			total=`sed -n 2p ${TEST_STATUS} | awk '{print $4}'`
-			if [ ${index} != ${total} ]; then
-				Test_Info "Do not kill the qemu process and use it for later testing"
-				Test_Update_IPSAVE $PID $TARGET_IPADDR
-			else
-				# If it is the last case, let's kill it
-				while [ $i -ne 0 ]
-				do
-					i=$((i-1))
-					kill ${pid[$i]}
-					sleep 2
-				done
-
-				# Kill the parent id
-				kill $PID
-			fi
 
+	# When TEST_SERIALIZE is set, qemu process will not be
+	# killed until all the cases are finished
+	if [ ${TEST_SERIALIZE} -eq 1 -a -e ${TEST_STATUS} ]; then
+		index=`sed -n 2p ${TEST_STATUS} | awk '{print $3}'`
+		total=`sed -n 2p ${TEST_STATUS} | awk '{print $4}'`
+		if [ ${index} != ${total} ]; then
+			Test_Info "Do not kill the qemu process and use it for later testing"
+			Test_Update_IPSAVE $XTERMPID $TARGET_IPADDR
 		else
-			# Kill these children pids from the last one
-			while [ $i -ne 0 ]
-			do
-				i=$((i-1))
-				kill ${pid[$i]}
-				sleep 2
-			done
-
-			# Kill the parent id
-			kill $PID
+			kill -$QEMUPID
+			kill -$XTERMPID
 		fi
+	else
+		kill -$QEMUPID
+		kill -$XTERMPID
 	fi
 
 	return
 }
 
-# function to check if there is any qemu process
-Test_Check_Qemu_UP()
-{
-	local count=`ps -eo command | cut -d " " -f 1 | grep -c "\(^qemu\|.*/qemu\)"`
-	if [ ${count} -lt 1 ]; then
-		Test_Info "There is no Qemu process"
-		return 1
-	else
-		Test_Info "There is at least one Qemu process running"
-		return 0
-	fi
-}
-
 # function to check if network is up
 Test_Check_IP_UP()
 {
-	ping -c1 $1
+	ping -c1 $1 1> /dev/null
 	if [ $? -ne 0 ]; then
 		Test_Info "IP $1 is not up"
 		return 1
@@ -377,46 +321,12 @@ Test_Find_Image()
 Test_Fetch_Target_IP()
 {
 	local opid=$1
-	local ppid=0
 	local ip_addr=0
-	local i=0
-	declare local pid
 
 	# Check if $1 pid exists and contains ipaddr of target
 	ps -wwfp $opid | grep -oq "192\.168\.7\.[0-9]*::"
 
-	# Find all children pid of the pid $1
-	# and check if they contain ipaddr of target
-	if [ $? -ne 0 ]; then
-		# Check if there is any child pid of the pid $1
-		ppid=$opid
-		ps -f --ppid $ppid > /dev/zero
-		ret=$?
-
-		while [ $ret -eq 0 ]
-		do
-			# If yes, get the child pid and check if the child pid has other child pid
-			# Continue the while loop until there is no child pid found
-			pid[$i]=`ps -f --ppid $ppid | awk '{if ($2 != "PID") print $2}'`
-			ppid=${pid[$i]}
-			i=$((i+1))
-			ps -f --ppid $ppid > /dev/zero
-			ret=$?
-		done
-
-		# Check these children pids, if they have ipaddr included in command line
-		while [ $i -ne 0 ]
-		do
-			i=$((i-1))
-			ps -wwfp ${pid[$i]} | grep -oq "192\.168\.7\.[0-9]*::"
-			if [ $? -eq 0 ]; then
-				ip_addr=`ps -wwfp ${pid[$i]} | grep -o "192\.168\.7\.[0-9]*::" | awk -F":" '{print $1}'`
-			fi
-			sleep 1
-		done
-	else
-		ip_addr=`ps -wwfp $opid | grep -o "192\.168\.7\.[0-9]*::" | awk -F":" '{print $1}'`
-	fi
+	ip_addr=`ps -wwfp $opid | grep -o "192\.168\.7\.[0-9]*::" | awk -F":" '{print $1}'`
 
 	echo $ip_addr
 
@@ -427,13 +337,10 @@ Test_Fetch_Target_IP()
 Test_Create_Qemu()
 {
 	local timeout=$1
-	local ret=1
 	local up_time=0
 
-	which runqemu
-	if [ $? -eq 0 ]; then
-		RUNQEMU=`which runqemu`
-	else
+	RUNQEMU=`which runqemu`
+	if [ $? -ne 0 ]; then
 		Test_Error "Can not find runqemu in \$PATH, return fail"
 		return 1
 	fi
@@ -449,7 +356,7 @@ Test_Create_Qemu()
 	# If there is no kernel image found, return failed directly
 	if [ $? -eq 1 ]; then
 		Test_Info "No kernel image file found under ${DEPLOY_DIR}/images for ${QEMUARCH}, pls. have a check"
-		return $ret
+		return 1
 	fi
 	
 	ROOTFS_IMAGE=$(Test_Find_Image -l ${DEPLOY_DIR}/images -t ${QEMUTARGET} -a ${QEMUARCH})
@@ -457,7 +364,7 @@ Test_Create_Qemu()
 	# If there is no rootfs image found, return failed directly
 	if [ $? -eq 1 ]; then
 		Test_Info "No ${QEMUTARGET} rootfs image file found under ${DEPLOY_DIR}/images for ${QEMUARCH}, pls. have a check"
-		return $ret
+		return 1
 	fi
 
 	TEST_ROOTFS_IMAGE="${TEST_TMP}/${QEMUTARGET}-${QEMUARCH}-test.ext3"
@@ -467,7 +374,7 @@ Test_Create_Qemu()
 	# When TEST_SERIALIZE is set, we use the existing image under tmp folder
 	if [ ${TEST_SERIALIZE} -eq 1 -a -e "$TARGET_IPSAVE" ]; then
 		# If TARGET_IPSAVE exists, check PID of the qemu process from it
-		PID=`awk '{print $1}' $TARGET_IPSAVE`
+		XTERMPID=`awk '{print $1}' $TARGET_IPSAVE`
 		timeout=50
 	else
 		rm -rf $TEST_ROOTFS_IMAGE
@@ -475,22 +382,22 @@ Test_Create_Qemu()
 		$CP $ROOTFS_IMAGE $TEST_ROOTFS_IMAGE
 		if [ $? -ne 0 ]; then
 			Test_Info "Image ${ROOTFS_IMAGE} copy to ${TEST_ROOTFS_IMAGE} failed, return fail"
-			return $ret
+			return 1
 		fi
 
 		export MACHINE=$QEMUARCH
 
 		# Create Qemu in localhost VNC Port 1
-		echo "Running xterm -display ${DISPLAY} -e 'OE_TMPDIR=${OE_TMPDIR} ${RUNQEMU} ${KERNEL} ${TEST_ROOTFS_IMAGE} && /bin/sleep 60' &"
-		xterm -display ${DISPLAY} -e "OE_TMPDIR=${OE_TMPDIR} ${RUNQEMU} ${KERNEL} ${TEST_ROOTFS_IMAGE} && /bin/sleep 60" &
+		echo "Running xterm -display ${DISPLAY} -e 'OE_TMPDIR=${OE_TMPDIR} ${RUNQEMU} ${KERNEL} ${TEST_ROOTFS_IMAGE} || /bin/sleep 60' &"
+		xterm -display ${DISPLAY} -e "OE_TMPDIR=${OE_TMPDIR} ${RUNQEMU} ${KERNEL} ${TEST_ROOTFS_IMAGE} || /bin/sleep 60" &
 	
 		# Get the pid of the xterm processor, which will be used in Test_Kill_Qemu
-		PID=$!
+		XTERMPID=$!
 	fi
 
 	while [ ${up_time} -lt 10 ]
 	do
-		Test_Check_Qemu_UP
+		QEMUPID=`qemuimage-testlib-pythonhelper --findqemu $XTERMPID`
 		if [ $? -ne 0 ]; then
 			Test_Info "Wait for qemu up..."
 			up_time=`expr $up_time + 5`
@@ -501,15 +408,17 @@ Test_Create_Qemu()
 		fi
 	done
 
+	if [ ${up_time} == 10 ]; then
+		Test_Info "No qemu process appeared to start, exiting"
+		return 1
+	fi
+
 	# Parse IP address of target from the qemu command line
-	if [ ${up_time} -lt ${timeout} ]; then
-		sleep 5
-		TARGET_IPADDR=`Test_Fetch_Target_IP $PID`
-		# If IP address is 0, means there is no qemu process found
-		if [ ${TARGET_IPADDR} == "0" ]; then
-			Test_Info "There is no qemu process or qemu ip address found, return failed"
-			return $ret
-		fi
+	TARGET_IPADDR=`Test_Fetch_Target_IP $QEMUPID`
+	# If IP address is 0, means there is no qemu process found
+	if [ ${TARGET_IPADDR} == "0" ]; then
+		Test_Info "There is no qemu process or qemu ip address found, return failed"
+		return 1
 	fi
 
 	while [ ${up_time} -lt ${timeout} ]
@@ -517,8 +426,7 @@ Test_Create_Qemu()
 		Test_Check_IP_UP ${TARGET_IPADDR}
 		if [ $? -eq 0 ]; then
 			Test_Info "Qemu Network is up, ping with ${TARGET_IPADDR} is OK within ${up_time} seconds"
-			ret=0
-			break
+			return 0
 		else
 			Test_Info "Wait for Qemu Network up"
 			up_time=`expr $up_time + 5`
@@ -526,14 +434,9 @@ Test_Create_Qemu()
 		fi
 	done
 
-	if [ $ret -eq 0 ]; then
-		Test_Info "Qemu and its network is up"
-		return $ret
-	else
-		Test_Info "Qemu or its network is not up in ${timeout} seconds"
-		Test_Update_IPSAVE $PID $TARGET_IPADDR
-		return $ret
-	fi
+	Test_Info "Qemu or its network is not up in ${timeout} seconds"
+	Test_Update_IPSAVE $XTERMPID $TARGET_IPADDR
+	return 1
 }
 
 # Function to prepare test project for toolchain test
@@ -542,41 +445,36 @@ Test_Create_Qemu()
 Test_Project_Prepare()
 {
 	local toolchain_dir=$1
-	local ret=1
 
 	if [ ! -d ${toolchain_dir} ]; then
 		mkdir -p ${toolchain_dir}
-		ret=$?
-
-		if [ $ret -ne 0 ]; then
+		if [ $? -ne 0 ]; then
+			ret=$?
 			Test_Info "Create ${toolchain_dir} fail, return"
 			return $ret
 		fi
 	fi
 
-	ret=0
 	# Download test project tarball if it does not exist
 	if [ ! -f ${toolchain_dir}/${2}-${PROJECT_PV}.${suffix} ]; then
 		wget -c -t 5 $PROJECT_DOWNLOAD_URL -O ${toolchain_dir}/${2}-${PROJECT_PV}.${suffix}
-		ret=$?
+		if [ $? -ne 0 ]; then
+			ret=$?
+			Test_Info "Fail to download ${2}-${PROJECT_PV}.${suffix} from $PROJECT_DOWNLOAD_URL"
+			rm -rf ${toolchain_dir}/${2}-${PROJECT_PV}.${suffix}
+			return $ret
+		fi
 	fi
 
 	# Extract the test project into ${TEST_TMP}
-	if [ $ret -eq 0 ]; then
-		tar jxf ${toolchain_dir}/${2}-${PROJECT_PV}.${suffix} -C ${TEST_TMP}
+	tar jxf ${toolchain_dir}/${2}-${PROJECT_PV}.${suffix} -C ${TEST_TMP}
+	if [ $? -ne 0 ]; then
 		ret=$?
-		if [ $ret -eq 0 ]; then
-			Test_Info "Extract ${2}-${PROJECT_PV}.${suffix} into ${TEST_TMP} successfully"
-			return $ret
-		else
-			Test_Info "Fail to extract ${2}-${PROJECT_PV}.${suffix} into ${TEST_TMP}"
-			return $ret
-		fi
-	else
-		Test_Info "Fail to download ${2}-${PROJECT_PV}.${suffix} from $PROJECT_DOWNLOAD_URL"
-		rm -rf ${toolchain_dir}/${2}-${PROJECT_PV}.${suffix}
+		Test_Info "Fail to extract ${2}-${PROJECT_PV}.${suffix} into ${TEST_TMP}"
 		return $ret
 	fi
+	Test_Info "Extract ${2}-${PROJECT_PV}.${suffix} into ${TEST_TMP} successfully"
+	return 0
 }
 
 # Function to prepare toolchain environment
diff --git a/scripts/qemuimage-testlib-pythonhelper b/scripts/qemuimage-testlib-pythonhelper
new file mode 100755
index 0000000000..2ca61ca06a
--- /dev/null
+++ b/scripts/qemuimage-testlib-pythonhelper
@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+import optparse
+import subprocess
+import sys
+
+parser = optparse.OptionParser(
+    usage = """
+  %prog [options]
+""")
+
+parser.add_option("-q", "--findqemu",
+        help = "find a qemu beneath the process <pid>",
+        action="store", dest="findqemu")
+
+options, args = parser.parse_args(sys.argv)
+
+if options.findqemu:
+    #
+    # Walk the process tree from the process specified looking for a qemu-system. Return its pid.
+    #
+    ps = subprocess.Popen(['ps', 'ax', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0]
+    processes = ps.split('\n')
+    nfields = len(processes[0].split()) - 1
+    pids = {}
+    commands = {}
+    for row in processes[1:]:
+        data = row.split(None, nfields)
+        if len(data) != 3:
+            continue
+        if data[1] not in pids:
+            pids[data[1]] = []
+        pids[data[1]].append(data[0])
+        commands[data[0]] = data[2]
+
+    if options.findqemu not in pids:
+        print "No children found matching %s" % options.findqemu
+        sys.exit(1)
+
+    parents = []
+    newparents = pids[options.findqemu]
+    while newparents:
+        next = []
+        for p in newparents:
+            if p in pids:
+                for n in pids[p]:
+                    if n not in parents and n not in next:
+                        next.append(n)
+
+            if p not in parents:
+                parents.append(p)
+        newparents = next
+    #print "Children matching %s:" % str(parents)
+    for p in parents:
+        if "qemu-system" in commands[p]:
+            print p
+            sys.exit(0)
+    sys.exit(1)
+else:
+    parser.print_help()
+
diff --git a/scripts/qemuimage-tests/sanity/shutdown b/scripts/qemuimage-tests/sanity/shutdown
index d55c859410..c9e931c4c2 100755
--- a/scripts/qemuimage-tests/sanity/shutdown
+++ b/scripts/qemuimage-tests/sanity/shutdown
@@ -45,7 +45,7 @@ if [ $RET -eq 0 ]; then
 	# If qemu start up process ends up, it means shutdown completes
 	while [ $i -lt $TIMEOUT ]
 	do
-		ps -fp $PID > /dev/null
+		ps -fp $QEMUPID > /dev/null 2> /dev/null
 		if [ $? -ne 0 ]; then
 			RET=0
 			break