diff options
Diffstat (limited to 'scripts/contrib/build-perf-test.sh')
| -rwxr-xr-x | scripts/contrib/build-perf-test.sh | 400 |
1 files changed, 400 insertions, 0 deletions
diff --git a/scripts/contrib/build-perf-test.sh b/scripts/contrib/build-perf-test.sh new file mode 100755 index 0000000..7d99228 --- /dev/null +++ b/scripts/contrib/build-perf-test.sh @@ -0,0 +1,400 @@ +#!/bin/bash +# +# This script runs a series of tests (with and without sstate) and reports build time (and tmp/ size) +# +# Build performance test script +# +# Copyright 2013 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# +# AUTHORS: +# Stefan Stanacar <stefanx.stanacar@intel.com> + + +ME=$(basename $0) + +# +# usage and setup +# + +usage () { +cat << EOT +Usage: $ME [-h] + $ME [-c <commit>] [-v] [-m <val>] [-j <val>] [-t <val>] [-i <image-name>] [-d <path>] +Options: + -h + Display this help and exit. + -c <commit> + git checkout <commit> before anything else + -v + Show bitbake output, don't redirect it to a log. + -m <machine> + Value for MACHINE. Default is qemux86. + -j <val> + Value for PARALLEL_MAKE. Default is 8. + -t <val> + Value for BB_NUMBER_THREADS. Default is 8. + -i <image-name> + Instead of timing against core-image-sato, use <image-name> + -d <path> + Use <path> as DL_DIR + -p <githash> + Cherry pick githash onto the commit + +Note: current working directory must be inside a poky git clone. + +EOT +} + + +if clonedir=$(git rev-parse --show-toplevel); then + cd $clonedir +else + echo "The current working dir doesn't seem to be a poky git clone. Please cd there before running $ME" + exit 1 +fi + +IMAGE="core-image-sato" +verbose=0 +dldir= +commit= +pmake= +cherrypicks= +while getopts "hvc:m:j:t:i:d:p:" opt; do + case $opt in + h) usage + exit 0 + ;; + v) verbose=1 + ;; + c) commit=$OPTARG + ;; + m) export MACHINE=$OPTARG + ;; + j) pmake=$OPTARG + ;; + t) export BB_NUMBER_THREADS=$OPTARG + ;; + i) IMAGE=$OPTARG + ;; + d) dldir=$OPTARG + ;; + p) cherrypicks="$cherrypicks $OPTARG" + ;; + *) usage + exit 1 + ;; + esac +done + + +#drop cached credentials and test for sudo access without a password +sudo -k -n ls > /dev/null 2>&1 +reqpass=$? +if [ $reqpass -ne 0 ]; then + echo "The script requires sudo access to drop caches between builds (echo 3 > /proc/sys/vm/drop_caches)" + read -s -p "Please enter your sudo password: " pass + echo +fi + +if [ -n "$commit" ]; then + echo "git checkout -f $commit" + git pull > /dev/null 2>&1 + git checkout -f $commit || exit 1 + git pull > /dev/null 2>&1 +fi + +if [ -n "$cherrypicks" ]; then + for c in $cherrypicks; do + git cherry-pick $c + done +fi + +rev=$(git rev-parse --short HEAD) || exit 1 +OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`" +BUILDDIR="$OUTDIR/build" +resultsfile="$OUTDIR/results.log" +cmdoutput="$OUTDIR/commands.log" +myoutput="$OUTDIR/output.log" +globalres="$clonedir/build-perf-test/globalres.log" + +mkdir -p $OUTDIR || exit 1 + +log () { + local msg="$1" + echo "`date`: $msg" | tee -a $myoutput +} + + +# +# Config stuff +# + +branch=`git branch 2>&1 | grep "^* " | tr -d "* "` +gitcommit=$(git rev-parse HEAD) || exit 1 +log "Running on $branch:$gitcommit" + +source ./oe-init-build-env $OUTDIR/build >/dev/null || exit 1 +cd $OUTDIR/build + +[ -n "$MACHINE" ] || export MACHINE="qemux86" +[ -n "$BB_NUMBER_THREADS" ] || export BB_NUMBER_THREADS="8" + +if [ -n "$pmake" ]; then + export PARALLEL_MAKE="-j $pmake" +else + export PARALLEL_MAKE="-j 8" +fi + +if [ -n "$dldir" ]; then + echo "DL_DIR = \"$dldir\"" >> conf/local.conf +else + echo "DL_DIR = \"$clonedir/build-perf-test/downloads\"" >> conf/local.conf +fi + +# Sometimes I've noticed big differences in timings for the same commit, on the same machine +# Disabling the network sanity check helps a bit (because of my crappy network connection and/or proxy) +echo "CONNECTIVITY_CHECK_URIS =\"\"" >> conf/local.conf + + +# +# Functions +# + +declare -a TIMES +time_count=0 +declare -a SIZES +size_count=0 + +time_cmd () { + log " Timing: $*" + + if [ $verbose -eq 0 ]; then + /usr/bin/time -v -o $resultsfile "$@" >> $cmdoutput + else + /usr/bin/time -v -o $resultsfile "$@" + fi + ret=$? + if [ $ret -eq 0 ]; then + t=`grep wall $resultsfile | sed 's/.*m:ss): //'` + log " TIME: $t" + TIMES[(( time_count++ ))]="$t" + else + log "ERROR: exit status was non-zero, will report time as 0." + TIMES[(( time_count++ ))]="0" + fi + + #time by default overwrites the output file and we want to keep the results + #it has an append option but I don't want to clobber the results in the same file + i=`ls $OUTDIR/results.log* |wc -l` + mv $resultsfile "${resultsfile}.${i}" + log "More stats can be found in ${resultsfile}.${i}" +} + +bbtime () { + time_cmd bitbake "$@" +} + +#we don't time bitbake here +bbnotime () { + local arg="$@" + log " Running: bitbake ${arg}" + if [ $verbose -eq 0 ]; then + bitbake ${arg} >> $cmdoutput + else + bitbake ${arg} + fi + ret=$? + if [ $ret -eq 0 ]; then + log " Finished bitbake ${arg}" + else + log "ERROR: exit status was non-zero. Exit.." + exit $ret + fi + +} + +do_rmtmp() { + log " Removing tmp" + rm -rf bitbake.lock pseudodone conf/sanity_info cache tmp +} +do_rmsstate () { + log " Removing sstate-cache" + rm -rf sstate-cache +} +do_sync () { + log " Syncing and dropping caches" + sync; sync + if [ $reqpass -eq 0 ]; then + sudo sh -c "echo 3 > /proc/sys/vm/drop_caches" + else + echo "$pass" | sudo -S sh -c "echo 3 > /proc/sys/vm/drop_caches" + echo + fi + sleep 3 +} + +write_results() { + echo -n "`uname -n`,$branch:$gitcommit,`git describe`," >> $globalres + for i in "${TIMES[@]}"; do + echo -n "$i," >> $globalres + done + for i in "${SIZES[@]}"; do + echo -n "$i," >> $globalres + done + echo >> $globalres + sed -i '$ s/,$//' $globalres +} + +#### + +# +# Test 1 +# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir (w/o rm_work and w/ rm_work) +# Pre: Downloaded sources, no sstate +# Steps: +# Part1: +# - fetchall +# - clean build dir +# - time bitbake core-image-sato +# - collect data +# Part2: +# - bitbake virtual/kernel -c cleansstate +# - time bitbake virtual/kernel +# Part3: +# - add INHERIT to local.conf +# - clean build dir +# - build +# - report size, remove INHERIT + +test1_p1 () { + log "Running Test 1, part 1/3: Measure wall clock of bitbake $IMAGE and size of tmp/ dir" + bbnotime $IMAGE -c fetchall + do_rmtmp + do_rmsstate + do_sync + bbtime $IMAGE + s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'` + SIZES[(( size_count++ ))]="$s" + log "SIZE of tmp dir is: $s" + log "Buildstats are saved in $OUTDIR/buildstats-test1" + mv tmp/buildstats $OUTDIR/buildstats-test1 +} + + +test1_p2 () { + log "Running Test 1, part 2/3: bitbake virtual/kernel -c cleansstate and time bitbake virtual/kernel" + bbnotime virtual/kernel -c cleansstate + do_sync + bbtime virtual/kernel +} + +test1_p3 () { + log "Running Test 1, part 3/3: Build $IMAGE w/o sstate and report size of tmp/dir with rm_work enabled" + echo "INHERIT += \"rm_work\"" >> conf/local.conf + do_rmtmp + do_rmsstate + do_sync + bbtime $IMAGE + sed -i 's/INHERIT += \"rm_work\"//' conf/local.conf + s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'` + SIZES[(( size_count++ ))]="$s" + log "SIZE of tmp dir is: $s" + log "Buildstats are saved in $OUTDIR/buildstats-test13" + mv tmp/buildstats $OUTDIR/buildstats-test13 +} + + +# +# Test 2 +# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir +# Pre: populated sstate cache + +test2 () { + # Assuming test 1 has run + log "Running Test 2: Measure wall clock of bitbake $IMAGE -c rootfs with sstate" + do_rmtmp + do_sync + bbtime $IMAGE -c rootfs +} + + +# Test 3 +# parsing time metrics +# +# Start with +# i) "rm -rf tmp/cache; time bitbake -p" +# ii) "rm -rf tmp/cache/default-glibc/; time bitbake -p" +# iii) "time bitbake -p" + + +test3 () { + log "Running Test 3: Parsing time metrics (bitbake -p)" + log " Removing tmp/cache && cache" + rm -rf tmp/cache cache + bbtime -p + log " Removing tmp/cache/default-glibc/" + rm -rf tmp/cache/default-glibc/ + bbtime -p + bbtime -p +} + +# +# Test 4 - eSDK +# Measure: eSDK size and installation time +test4 () { + log "Running Test 4: eSDK size and installation time" + bbnotime $IMAGE -c do_populate_sdk_ext + + esdk_installer=(tmp/deploy/sdk/*-toolchain-ext-*.sh) + + if [ ${#esdk_installer[*]} -eq 1 ]; then + s=$((`stat -c %s "$esdk_installer"` / 1024)) + SIZES[(( size_count++ ))]="$s" + log "Download SIZE of eSDK is: $s kB" + + do_sync + time_cmd "$esdk_installer" -y -d "tmp/esdk-deploy" + + s=$((`du -sb "tmp/esdk-deploy" | cut -f1` / 1024)) + SIZES[(( size_count++ ))]="$s" + log "Install SIZE of eSDK is: $s kB" + else + log "ERROR: other than one sdk found (${esdk_installer[*]}), reporting size and time as 0." + SIZES[(( size_count++ ))]="0" + TIMES[(( time_count++ ))]="0" + fi + +} + + +# RUN! + +test1_p1 +test1_p2 +test1_p3 +test2 +test3 +test4 + +# if we got til here write to global results +write_results + +log "All done, cleaning up..." + +do_rmtmp +do_rmsstate |