summaryrefslogtreecommitdiff
path: root/scripts/qemuimage-testlib
blob: 1fa9b4edbdf2f7d5137cd160dec5eec4d294cb6e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
#!/bin/bash
# Common function for test
# Expect should be installed for SSH Testing
# To execute `runqemu`, NOPASSWD needs to be set in /etc/sudoers for user
# For example, for user "builder", /etc/sudoers can be like following:
# #########
# #Members of the admin group may gain root privileges
# %builder ALL=(ALL) NOPASSWD: NOPASSWD: ALL
# #########
#
# Author: Jiajun Xu <jiajun.xu@intel.com>
#
# This file is licensed under the GNU General Public License,
# Version 2.
#

# The folder to hold all scripts running on targets
TOOLS="$COREBASE/scripts/qemuimage-tests/tools"

# The folder to hold all projects for toolchain testing
TOOLCHAIN_PROJECTS="$COREBASE/scripts/qemuimage-tests/toolchain_projects"

# Test Directory on target for testing
TARGET_TEST_DIR="/opt/test"

# Global variables for process id
XTERMPID=0
QEMUPID=0

# Global variable for target ip address
TARGET_IPADDR=0

# Global variable for test project version during toolchain test
# Version of cvs is 1.12.13
# Version of iptables is 1.4.11
# Version of sudoku-savant is 1.3
PROJECT_PV=0

# Global variable for test project download URL during toolchain test
# URL of cvs is http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2
# URL of iptables is http://netfilter.org/projects/iptables/files/iptables-1.4.11.tar.bz2
# URL of sudoku-savant is http://downloads.sourceforge.net/project/sudoku-savant/sudoku-savant/sudoku-savant-1.3/sudoku-savant-1.3.tar.bz2
PROJECT_DOWNLOAD_URL=0

# SDK folder to hold toolchain tarball
TOOLCHAIN_DIR="${DEPLOY_DIR}/sdk"

# Toolchain test folder to hold extracted toolchain tarball
TOOLCHAIN_TEST="/opt"

# common function for information print
Test_Error()
{
	echo -e "\tTest_Error: $*"
}

Test_Info()
{
	echo -e "\tTest_Info: $*"
}

# function to update target ip address
# $1 is the process id of the process, which starts the qemu target
# $2 is the ip address of the target
Test_Update_IPSAVE()
{
	local pid=$1
	local ip_addr=$2

	if [ "$TEST_SERIALIZE" -eq 1 -a "$pid" != "0" -a "$pid" != "" -a "$ip_addr" != "" -a "$ip_addr" != "" ]; then
		echo "Saving $pid $ip_addr to $TARGET_IPSAVE"
		echo "$pid $ip_addr" > $TARGET_IPSAVE
	fi
}

# function to copy files from host into target
# $1 is the ip address of target
# $2 is the files, which need to be copied into target
# $3 is the path on target, where files are copied into
Test_SCP()
{
	local ip_addr=$1
	local src=$2
	local des=$3
	local time_out=60
	local ret=0

	# We use expect to interactive with target by ssh
	local exp_cmd=`cat << EOF
eval spawn scp -o UserKnownHostsFile=/dev/null  -o StrictHostKeyChecking=no "$src" root@$ip_addr:"$des"
set timeout $time_out
expect {
    "*assword:" { send "\r"; exp_continue}
    "*(yes/no)?" { send "yes\r"; exp_continue }
    eof          { exit [ lindex [wait] 3 ] }
}
EOF`

	expect=`which expect`
	if [ ! -x "$expect" ]; then
		Test_Error "ERROR: Please install expect"
		return 1
	fi

	expect -c "$exp_cmd"
	ret=$?
	return $ret
}

# function to copy files from target to host
# $1 is the ip address of target
# $2 is the files, which need to be copied into target
# $3 is the path on target, where files are copied into
Test_SCP_From()
{
	local ip_addr=$1
	local src=$2
	local des=$3
	local time_out=60
	local ret=0

	# We use expect to interactive with target by ssh
	local exp_cmd=`cat << EOF
eval spawn scp -o UserKnownHostsFile=/dev/null  -o StrictHostKeyChecking=no root@$ip_addr:"$src" "$des"
set timeout $time_out
expect {
    "*assword:" { send "\r"; exp_continue}
    "*(yes/no)?" { send "yes\r"; exp_continue }
    eof          { exit [ lindex [wait] 3 ] }
}
EOF`

	expect=`which expect`
	if [ ! -x "$expect" ]; then
		Test_Error "ERROR: Please install expect"
		return 1
	fi

	expect -c "$exp_cmd"
	ret=$?
	return $ret
}

# function to run command in $ip_addr via ssh
Test_SSH()
{
	local ip_addr=$1
	shift
	local command=$@
	local time_out=60
	local ret=0
	local exp_cmd=`cat << EOF
eval spawn ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@$ip_addr "$command"
set timeout $time_out
expect {
    "*assword:" { send "\r"; exp_continue}
    "*(yes/no)?" { send "yes\r"; exp_continue }
    eof          { exit [ lindex [wait] 3 ] }
}
EOF`

	expect=`which expect`
	if [ ! -x "$expect" ]; then
		Test_Error "ERROR: Please install expect"
		return 1
	fi

	expect -c "$exp_cmd"
	ret=$?
	return $ret
}

# function to check if ssh is up in $ip_addr
Test_SSH_UP()
{
	local ip_addr=$1
	local timeout=$2
	local interval=0

	# If TEST_SERIALIZE is set, use existing running qemu for testing
	if [ ${TEST_SERIALIZE} -eq 1 -a -e ${TARGET_IPSAVE} ]; then
		timeout=50
	fi

	while [ ${interval} -lt ${timeout} ]
	do
		Test_SSH ${ip_addr} "hostname"
		if [ $? -ne 0 ]; then
			interval=`expr $interval + 10`
			sleep 10
		else
			Test_Info "We can ssh on ${ip_addr} within ${interval} seconds"
			return 0
		fi

	done

	Test_Info "We can not ssh on ${ip_addr} in ${timeout} seconds"
	return 1
}

# function to prepare target test environment
# $1 is the ip address of target system
# $2 is the files, which needs to be copied into target
Test_Target_Pre()
{
	local ip_addr=$1
	local testscript=$2

	# Create a pre-defined folder for test scripts
	Test_SSH $ip_addr "mkdir -p $TARGET_TEST_DIR"
	if [ $? -eq 0 ]; then
		# Copy test scripts into target
		Test_SCP $ip_addr $testscript $TARGET_TEST_DIR && return 0
	else
		Test_Error "Fail to create $TARGET_TEST_DIR on target"
		return 1
	fi

	return 1
}

# function to record test result in $TEST_RESULT/testresult.log
Test_Print_Result()
{
	local PASS=0
	local FAIL=0
	local NORESULT=0
	if [ $2 -eq 0 ]; then
		PASS=1
	elif [ $2 -eq 1 ]; then
		FAIL=1
	else
		NORESULT=1
	fi

	# Format the output of the test result
	echo -e "$1 $PASS $FAIL $NORESULT" | awk '{printf("\t"); for(i=1;i<=NF;i++) printf("%-15s",$i); printf("\n");}' >> $TEST_RESULT/testresult.log
}

# Test_Kill_Qemu to kill child pid with parent pid given
# $1 is qemu process id, which needs to be killed
Test_Kill_Qemu()
{
	local index=0
	local total=0
	local k=0

	# When TEST_SERIALIZE is set, qemu process will not be
	# killed until all the cases are finished
	if [ ${TEST_SERIALIZE} -eq 1 -a -e ${TEST_STATUS} ]; then
		index=`sed -n 2p ${TEST_STATUS} | awk '{print $3}'`
		total=`sed -n 2p ${TEST_STATUS} | awk '{print $4}'`
		if [ ${index} != ${total} ]; then
			Test_Info "Do not kill the qemu process and use it for later testing (step $index of $total)"
			Test_Update_IPSAVE $XTERMPID $TARGET_IPADDR
		else
			k=1
		fi
	else
		k=1
	fi

	if [ $k -eq 1 ]; then
		if [ "$QEMUPID" != "0" -a "$QEMUPID" != "" ]; then
			running=`ps -wwfp $QEMUPID`
			if [ $? -eq 0 ]; then
				echo "killing $QEMUPID"
				kill $QEMUPID
			fi
		fi
		if [ "$XTERMPID" != "0" -a "$XTERMPID" != "" ]; then
			running=`ps -wwfp $XTERMPID`
			if [ $? -eq 0 ]; then
				echo "killing $XTERMPID"
				kill $XTERMPID
			fi
		fi
	fi

	return
}

# function to check if network is up
Test_Check_IP_UP()
{
	ping -c1 $1 1> /dev/null
	if [ $? -ne 0 ]; then
		Test_Info "IP $1 is not up"
		return 1
	else
		Test_Info "IP $1 is up"
		return 0
	fi
}

# function to find kernel/rootfs image
Test_Find_Image()
{
	where=""
	kernel=""
	arch=""
	target=""
	extension=""
	rootfs=""

	while getopts "l:k:a:t:e:" Option
	do
		case $Option in
			l) where="$OPTARG"
			   ;;
			k) kernel="$OPTARG"
			   ;;
			a) arch="$OPTARG"
			   ;;
			t) target="$OPTARG"
			   ;;
			e) extension="$OPTARG"
			   ;;
			*) echo "invalid option: -$Option" && return 1
			   ;;
		 esac
	done

	if [ ! -z $kernel ]; then
		if [ -L ${where}/${kernel}-${arch}.${extension} ]; then
			echo ${where}/${kernel}-${arch}.${extension}
			return 0
		else
			for i in `dir ${where}`
			do
				# Exclude qemux86-64 when target is qemux86
				echo $i | grep "${kernel}.*${arch}.*\.${extension}" | grep -qv "${kernel}.*${arch}-64.*\.${extension}"
				if [ $? -eq 0 ]; then
					echo ${where}/${i}
					return 0
				fi
			done
			return 1
		fi
	fi

	if [ ! -z $target ]; then
		if [ -L ${where}/${target}-${arch}.${extension} ]; then
			rootfs=`readlink -f ${where}/${target}-${arch}.${extension}`
			echo ${rootfs}
			return 0
		else
			for i in `dir ${where}`
			do
				# Exclude qemux86-64 when target is qemux86
				echo $i | grep "${target}-${arch}.*\.${extension}" | grep -qv "${target}-${arch}-64.*\.${extension}"
				if [ $? -eq 0 ]; then
					echo ${where}/${i}
					return 0
				fi
			done
			return 1
		fi
	fi
	return 1
}

# function to parse IP address of target
# $1 is the pid of qemu startup process
Test_Fetch_Target_IP()
{
	local opid=$1
	local ip_addr=0

	if [ "$opid" = "0" -o "$opid" = "" ]; then
		echo ""
		return
	fi

	# Check if $1 pid exists and contains ipaddr of target
	ip_addr=`ps -wwfp $opid | grep -o "192\.168\.7\.[0-9]*::" | awk -F":" '{print $1}'`

	echo $ip_addr

	return
}

# function to check if qemu and its network
Test_Create_Qemu()
{
	local timeout=$1
	shift
	local extraargs="$@"
	local up_time=0

	RUNQEMU=`which runqemu`
	if [ $? -ne 0 ]; then
		Test_Error "Can not find runqemu in \$PATH, return fail"
		return 1
	fi
	
	if [ "$QEMUARCH" = "qemux86" -o "$QEMUARCH" = "qemux86-64" ]; then
		KERNEL=$(Test_Find_Image -l ${DEPLOY_DIR}/images -k bzImage -a ${QEMUARCH} -e "bin")
	elif [ "$QEMUARCH" = "qemuarm" -o "$QEMUARCH" = "spitz" -o "$QEMUARCH" = "borzoi" -o "$QEMUARCH" = "akita" -o "$QEMUARCH" = "nokia800" ]; then
		KERNEL=$(Test_Find_Image -l ${DEPLOY_DIR}/images -k zImage -a ${QEMUARCH})
	elif [ "$QEMUARCH" = "qemumips" -o "$QEMUARCH" = "qemuppc" ]; then
		KERNEL=$(Test_Find_Image -l ${DEPLOY_DIR}/images -k vmlinux -a ${QEMUARCH} -e "bin")
	fi

	# If there is no kernel image found, return failed directly
	if [ $? -eq 1 ]; then
		Test_Info "No kernel image file found under ${DEPLOY_DIR}/images for ${QEMUARCH}, pls. have a check"
		return 1
	fi
	
	Test_Info "rootfs image extension selected: $ROOTFS_EXT"
	ROOTFS_IMAGE=$(Test_Find_Image -l ${DEPLOY_DIR}/images -t ${QEMUTARGET} -a ${QEMUARCH} -e "$ROOTFS_EXT")

	# If there is no rootfs image found, return failed directly
	if [ $? -eq 1 ]; then
		Test_Info "No ${QEMUTARGET} rootfs image file found under ${DEPLOY_DIR}/images for ${QEMUARCH}, pls. have a check"
		return 1
	fi

	TEST_ROOTFS_IMAGE="${TEST_TMP}/${QEMUTARGET}-${QEMUARCH}-test.${ROOTFS_EXT}"
	
	CP=`which cp`

	# When TEST_SERIALIZE is set, we use the existing image under tmp folder
	if [ ${TEST_SERIALIZE} -eq 1 -a -e "$TARGET_IPSAVE" ]; then
		# If TARGET_IPSAVE exists, check PID of the qemu process from it
		XTERMPID=`awk '{print $1}' $TARGET_IPSAVE`
		timeout=50
	else
		rm -rf $TEST_ROOTFS_IMAGE
		echo "Copying rootfs $ROOTFS_IMAGE to $TEST_ROOTFS_IMAGE"
		$CP $ROOTFS_IMAGE $TEST_ROOTFS_IMAGE
		if [ $? -ne 0 ]; then
			Test_Info "Image ${ROOTFS_IMAGE} copy to ${TEST_ROOTFS_IMAGE} failed, return fail"
			return 1
		fi

		export MACHINE=$QEMUARCH

		# Create Qemu in localhost VNC Port 1
		echo "Running xterm -display ${DISPLAY} -e 'OE_TMPDIR=${OE_TMPDIR} ${RUNQEMU} ${KERNEL} ${TEST_ROOTFS_IMAGE} ${extraargs} 2>&1 | tee ${RUNQEMU_LOGFILE} || /bin/sleep 60' &"
		xterm -display ${DISPLAY} -e "OE_TMPDIR=${OE_TMPDIR} ${RUNQEMU} ${KERNEL} ${TEST_ROOTFS_IMAGE} ${extraargs} 2>&1 | tee ${RUNQEMU_LOGFILE} || /bin/sleep 60" &
	
		# Get the pid of the xterm processor, which will be used in Test_Kill_Qemu
		XTERMPID=$!
		echo "XTERMPID is $XTERMPID"
		# When starting, qemu can reexecute itself and change PID so wait a short while for things to settle
		sleep 5
	fi

	while [ ${up_time} -lt 30 ]
	do
		QEMUPID=`qemuimage-testlib-pythonhelper --findqemu $XTERMPID 2>/dev/null`
		if [ $? -ne 0 ]; then
			Test_Info "Wait for qemu up..."
			up_time=`expr $up_time + 5`
			sleep 5
		else
			Test_Info "Begin to check if qemu network is up"
			echo "QEMUPID is $QEMUPID"
			break
		fi
	done

	if [ ${up_time} == 30 ]; then
		Test_Info "No qemu process appeared to start, exiting"
		ps axww -O ppid
		Test_Info "Process list dumped for debugging purposes"
		Test_Info "runqemu output log:"
		cat ${RUNQEMU_LOGFILE}
		echo
		return 1
	fi

	up_time=0
	# Parse IP address of target from the qemu command line
	TARGET_IPADDR=`Test_Fetch_Target_IP $QEMUPID`
	echo "Target IP is ${TARGET_IPADDR}"
	if [ "${TARGET_IPADDR}" = "" -o "${TARGET_IPADDR}" = "0" ]; then
		Test_Info "There is no qemu process or qemu ip address found, return failed"
		ps -wwf
		ps axww -O ppid
		Test_Info "runqemu output log:"
		cat ${RUNQEMU_LOGFILE}
		echo
		return 1
	fi

	while [ ${up_time} -lt ${timeout} ]
	do
		Test_Check_IP_UP ${TARGET_IPADDR}
		if [ $? -eq 0 ]; then
			Test_Info "Qemu Network is up, ping with ${TARGET_IPADDR} is OK within ${up_time} seconds"
			return 0
		else
			Test_Info "Wait for Qemu Network up"
			up_time=`expr $up_time + 5`
			sleep 5
		fi
	done

	Test_Info "Process list dumped for debugging purposes:"
	ps axww -O ppid
	Test_Info "runqemu output log:"
	cat ${RUNQEMU_LOGFILE}
	Test_Info "Qemu or its network is not up in ${timeout} seconds"
	Test_Update_IPSAVE $XTERMPID $TARGET_IPADDR
	return 1
}

# Function to prepare test project for toolchain test
# $1 is the folder holding test project file
# $2 is the test project name
Test_Project_Prepare()
{
	local toolchain_dir=$1

	if [ ! -d ${toolchain_dir} ]; then
		mkdir -p ${toolchain_dir}
		if [ $? -ne 0 ]; then
			ret=$?
			Test_Info "Create ${toolchain_dir} fail, return"
			return $ret
		fi
	fi

	# Download test project tarball if it does not exist
	if [ ! -f ${toolchain_dir}/${2}-${PROJECT_PV}.${suffix} ]; then
		wget -c -t 5 $PROJECT_DOWNLOAD_URL -O ${toolchain_dir}/${2}-${PROJECT_PV}.${suffix}
		if [ $? -ne 0 ]; then
			ret=$?
			Test_Info "Fail to download ${2}-${PROJECT_PV}.${suffix} from $PROJECT_DOWNLOAD_URL"
			rm -rf ${toolchain_dir}/${2}-${PROJECT_PV}.${suffix}
			return $ret
		fi
	fi

	# Extract the test project into ${TEST_TMP}
	tar jxf ${toolchain_dir}/${2}-${PROJECT_PV}.${suffix} -C ${TEST_TMP}
	if [ $? -ne 0 ]; then
		ret=$?
		Test_Info "Fail to extract ${2}-${PROJECT_PV}.${suffix} into ${TEST_TMP}"
		return $ret
	fi
	Test_Info "Extract ${2}-${PROJECT_PV}.${suffix} into ${TEST_TMP} successfully"
	return 0
}

# Function to prepare toolchain environment
# $1 is toolchain directory to hold toolchain tarball
# $2 is prefix name for toolchain tarball
Test_Toolchain_Prepare()
{
	local toolchain_dir=$1
	local sdk_name=$2
	local ret=1

	if [ ! -d ${toolchain_dir} ]; then
		Test_Info "No directory ${toolchain_dir}, which holds toolchain tarballs"
		return 1
	fi

	# Check if there is any toolchain tarball under $toolchain_dir with prefix $sdk_name
	for i in `dir ${toolchain_dir}`
	do
		echo $i | grep "${sdk_name}-toolchain-gmae"
		if [ $? -eq 0 ]; then
			rm -rf ${TEST_TMP}/opt
			tar jxf ${toolchain_dir}/${i} -C ${TEST_TMP}
			ret=$?
			break
		fi
	done

	if [ $ret -eq 0 ]; then
		Test_Info "Check if /opt is accessible for non-root user"

		# Check if the non-root test user has write access of $TOOLCHAIN_TEST
		if [ -d ${TOOLCHAIN_TEST} ]; then
			touch ${TOOLCHAIN_TEST}
			if [ $? -ne 0 ]; then
				Test_Info "Has no right to modify folder $TOOLCHAIN_TEST, pls. chown it to test user"
				return 2
			fi
		else
			mkdir -p ${TOOLCHAIN_TEST}
			if [ $? -ne 0 ]; then
				Test_Info "Has no right to create folder $TOOLCHAIN_TEST, pls. create it and chown it to test user"
				return 2
			fi
		fi

		# If there is a toolchain folder under $TOOLCHAIN_TEST, let's remove it
		if [ -d ${TOOLCHAIN_TEST}/poky ]; then
			rm -rf ${TOOLCHAIN_TEST}/poky
		fi

		# Copy toolchain into $TOOLCHAIN_TEST
		cp -r ${TEST_TMP}/opt/poky ${TOOLCHAIN_TEST}
		ret=$?

		if [ $ret -eq 0 ]; then
			Test_Info "Successfully copy toolchain into $TOOLCHAIN_TEST"
			return $ret
		else
			Test_Info "Meet error when copy toolchain into $TOOLCHAIN_TEST"
			return $ret
		fi
	else
		Test_Info "No tarball named ${sdk_name}-toolchain-gmae under ${toolchain_dir}"
		return $ret
	fi
}

# Function to execute command and exit if run out of time
# $1 is timeout value
# $2 is the command to be executed
Test_Time_Out()
{
	local timeout=$1
	shift
	local command=$*
	local date=0
	local tmp=`mktemp`
	local ret=1
	local pid=0
	local ppid=0
	local i=0
	declare local pid_l

	# Run command in background
	($command; echo $? > $tmp) &
	pid=$!
	while ps -e -o pid | grep -qw $pid; do
		if [ $date -ge $timeout ]; then
			Test_Info "$timeout Timeout when running command $command"
			rm -rf $tmp

			# Find all child processes of pid and kill them
			ppid=$pid
			ps -f --ppid $ppid
			ret=$?
	
			while [ $ret -eq 0 ]
			do
				# If yes, get the child pid and check if the child pid has other child pid
				# Continue the while loop until there is no child pid found
				pid_l[$i]=`ps -f --ppid $ppid | awk '{if ($2 != "PID") print $2}'`
				ppid=${pid_l[$i]}
				i=$((i+1))
				ps -f --ppid $ppid
				ret=$?
			done
	
				# Kill these children pids from the last one
				while [ $i -ne 0 ]
				do
					i=$((i-1))
					kill ${pid_l[$i]}
					sleep 2
				done
	
				# Kill the parent id
				kill $pid
				return 1
		fi
		sleep 5
		date=`expr $date + 5`
	done
	ret=`cat $tmp`
	rm -rf $tmp
	return $ret
}

# Function to test toolchain
# $1 is test project name
# $2 is the timeout value
Test_Toolchain()
{
	local test_project=$1
	local timeout=$2
	local ret=1
	local suffix="tar.bz2"
	local env_setup=""
	local pro_install="${TEST_TMP}/pro_install"

	# Set value for PROJECT_PV and PROJECT_DOWNLOAD_URL accordingly
	if [ $test_project == "cvs" ]; then
		PROJECT_PV=1.12.13
		PROJECT_DOWNLOAD_URL="http://ftp.gnu.org/non-gnu/cvs/source/feature/1.12.13/cvs-1.12.13.tar.bz2"
	elif [ $test_project == "iptables" ]; then
		PROJECT_PV=1.4.11
		PROJECT_DOWNLOAD_URL="http://netfilter.org/projects/iptables/files/iptables-1.4.11.tar.bz2"
	elif [ $test_project == "sudoku-savant" ]; then
		PROJECT_PV=1.3
		PROJECT_DOWNLOAD_URL="http://downloads.sourceforge.net/project/sudoku-savant/sudoku-savant/sudoku-savant-1.3/sudoku-savant-1.3.tar.bz2"
	else
		Test_Info "Unknown test project name $test_project"
		return 1
	fi

	# Download test project and extract it
	Test_Project_Prepare $TOOLCHAIN_PROJECTS $test_project
	if [ $? -ne 0 ]; then
		Test_Info "Prepare test project file failed"
		return 1
	fi

	# Extract toolchain tarball into ${TEST_TMP}
	Test_Toolchain_Prepare $TOOLCHAIN_DIR $SDK_NAME
	ret=$?
	if [ $ret -ne 0 ]; then
		Test_Info "Prepare toolchain test environment failed"
		return $ret
	fi

	if [ ! -d ${pro_install} ]; then
		mkdir -p ${pro_install}
	fi

	# Begin to build test project in toolchain environment
	env_setup=`find ${TOOLCHAIN_TEST}/poky -name "environment-setup*"`

	source $env_setup

	if [ $test_project == "cvs" -o $test_project == "iptables" ]; then
		cd ${TEST_TMP}/${test_project}-${PROJECT_PV}
		Test_Time_Out $timeout ./configure ${CONFIGURE_FLAGS} || { Test_Info "configure failed with $test_project"; return 1; }
		Test_Time_Out $timeout make -j4 || { Test_Info "make failed with $test_project"; return 1; }
		Test_Time_Out $timeout make install DESTDIR=${pro_install} || { Test_Info "make failed with $test_project"; return 1; }
		cd -
		ret=0
	elif [ $test_project == "sudoku-savant" ]; then
		cd ${TEST_TMP}/${test_project}-${PROJECT_PV}
		Test_Time_Out $timeout ./configure ${CONFIGURE_FLAGS} || { Test_Info "configure failed with $test_project"; return 1; }
		Test_Time_Out $timeout make -j4 || { Test_Info "make failed with $test_project"; return 1; }
		cd -
		ret=0
	else
		Test_Info "Unknown test project $test_project"
		ret=1
	fi

	return $ret
}

Test_Display_Syslog()
{
	local tmplog=`mktemp`
        Test_SCP_From ${TARGET_IPADDR} /var/log/messages $tmplog
        echo "System logs:"
        cat $tmplog
        rm -f $tmplog
}