path: root/scripts/contrib/build-perf-test.sh

#!/bin/bash
#
# This script runs a series of tests  (with and without sstate) and reports build time (and tmp/ size)
# 
# Build performance test script
#
# Copyright 2013 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#
# AUTHORS:
# Stefan Stanacar <stefanx.stanacar@intel.com>


ME=$(basename $0)

#
# usage and setup
#

usage () {
cat << EOT
Usage: $ME [-h]
       $ME [-c <commit>] [-v] [-m <val>] [-j <val>] [-t <val>] [-i <image-name>] [-d <path>]
Options:
        -h
                Display this help and exit.
        -c <commit>
                git checkout <commit> before anything else
        -v
                Show bitbake output, don't redirect it to a log.
        -m <machine>
                Value for MACHINE. Default is qemux86.
        -j <val>
                Value for PARALLEL_MAKE. Default is 8. 
        -t <val>
                Value for BB_NUMBER_THREADS. Default is 8.
        -i <image-name>
                Instead of timing against core-image-sato, use <image-name>
        -d <path>
                Use <path> as DL_DIR
        -p <githash>
                Cherry pick githash onto the commit
                
Note: current working directory must be inside a poky git clone.
 
EOT
}


if clonedir=$(git rev-parse --show-toplevel); then
        cd $clonedir
else
        echo "The current working dir doesn't seem to be a poky git clone. Please cd there before running $ME"
        exit 1
fi

IMAGE="core-image-sato"
verbose=0
dldir=
commit=
pmake=
cherrypicks=
while getopts "hvc:m:j:t:i:d:p:" opt; do
        case $opt in
                h)      usage
                        exit 0
                        ;;
                v)      verbose=1
                        ;;
                c)      commit=$OPTARG
                        ;;
                m)      export MACHINE=$OPTARG
                        ;;
                j)      pmake=$OPTARG
                        ;;
                t)      export BB_NUMBER_THREADS=$OPTARG
                        ;;
                i)      IMAGE=$OPTARG
                        ;;
                d)      dldir=$OPTARG
                        ;;
                p)      cherrypicks="$cherrypicks $OPTARG"
                        ;;
                *)      usage
                        exit 1
                        ;;                      
        esac
done


#drop cached credentials and test for sudo access without a password
sudo -k -n ls > /dev/null 2>&1
reqpass=$?
if [ $reqpass -ne 0 ]; then
    echo "The script requires sudo access to drop caches between builds (echo 3 > /proc/sys/vm/drop_caches)"
    read -s -p "Please enter your sudo password: " pass
    echo
fi

if [ -n "$commit" ]; then
            echo "git checkout -f $commit"
            git pull > /dev/null 2>&1
            git checkout -f $commit || exit 1
            git pull > /dev/null 2>&1
fi

if [ -n "$cherrypicks" ]; then
    for c in $cherrypicks; do
        git cherry-pick $c
    done
fi

rev=$(git rev-parse --short HEAD)  || exit 1
OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`"
BUILDDIR="$OUTDIR/build"
resultsfile="$OUTDIR/results.log"
cmdoutput="$OUTDIR/commands.log"
myoutput="$OUTDIR/output.log"
globalres="$clonedir/build-perf-test/globalres.log"

mkdir -p $OUTDIR || exit 1
                                
log () {
    local msg="$1"
    echo "`date`: $msg" | tee -a $myoutput
}


#
# Config stuff
#

branch=`git branch 2>&1 | grep "^* " | tr -d "* "`
gitcommit=$(git rev-parse HEAD)  || exit 1
log "Running on $branch:$gitcommit"

source ./oe-init-build-env $OUTDIR/build >/dev/null || exit 1
cd $OUTDIR/build

[ -n "$MACHINE" ] || export MACHINE="qemux86"
[ -n "$BB_NUMBER_THREADS" ] || export BB_NUMBER_THREADS="8"

if [ -n "$pmake" ]; then
        export PARALLEL_MAKE="-j $pmake"
else
        export PARALLEL_MAKE="-j 8"
fi

if [ -n "$dldir" ]; then
    echo "DL_DIR = \"$dldir\"" >> conf/local.conf
else
    echo "DL_DIR = \"$clonedir/build-perf-test/downloads\"" >> conf/local.conf
fi

# Sometimes I've noticed big differences in timings for the same commit, on the same machine
# Disabling the network sanity check helps a bit (because of my crappy network connection and/or proxy)
echo "CONNECTIVITY_CHECK_URIS =\"\"" >> conf/local.conf


#
# Functions
#

declare -a TIMES
time_count=0
declare -a SIZES
size_count=0

time_cmd () {
    log "   Timing: $*"

    if [ $verbose -eq 0 ]; then 
        /usr/bin/time -v -o $resultsfile "$@" >> $cmdoutput
    else
        /usr/bin/time -v -o $resultsfile "$@"
    fi
    ret=$?
    if [ $ret -eq 0 ]; then
        t=`grep wall $resultsfile | sed 's/.*m:ss): //'`
        log "   TIME: $t"
        TIMES[(( time_count++ ))]="$t"
    else
        log "ERROR: exit status was non-zero, will report time as 0."
        TIMES[(( time_count++ ))]="0"
    fi
    
    #time by default overwrites the output file and we  want to keep the results
    #it has an append option but I don't want to clobber the results in the same file
    i=`ls $OUTDIR/results.log* |wc -l`
    mv $resultsfile "${resultsfile}.${i}"
    log "More stats can be found in ${resultsfile}.${i}"    
}

bbtime () {
    time_cmd bitbake "$@"
}

#we don't time bitbake here
bbnotime () {
    local arg="$@"
    log "   Running: bitbake ${arg}"
    if [ $verbose -eq 0 ]; then
        bitbake ${arg} >> $cmdoutput
    else
        bitbake ${arg}
    fi
    ret=$?
    if [ $ret -eq 0 ]; then
        log "   Finished bitbake ${arg}"
    else
        log "ERROR: exit status was non-zero. Exit.."
        exit $ret
    fi

}

do_rmtmp() {
    log "   Removing tmp"
    rm -rf bitbake.lock pseudodone conf/sanity_info cache tmp
}
do_rmsstate () {
    log "   Removing sstate-cache"
    rm -rf sstate-cache
}
do_sync () {
    log "   Syncing and dropping caches"
    sync; sync
    if [ $reqpass -eq 0 ]; then
        sudo sh -c "echo 3 > /proc/sys/vm/drop_caches"
    else
        echo "$pass" | sudo -S sh -c "echo 3 > /proc/sys/vm/drop_caches"
        echo
    fi
    sleep 3
}

write_results() {
    echo -n "`uname -n`,$branch:$gitcommit,`git describe`," >> $globalres
    for i in "${TIMES[@]}"; do
        echo -n "$i," >> $globalres
    done
    for i in "${SIZES[@]}"; do
        echo -n "$i," >> $globalres
    done
    echo >> $globalres
    sed -i '$ s/,$//' $globalres
}

####

#
# Test 1
# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir (w/o rm_work and w/ rm_work)
# Pre: Downloaded sources, no sstate
# Steps:
#     Part1:
#        - fetchall 
#        - clean build dir
#        - time bitbake core-image-sato
#        - collect data
#     Part2:
#        - bitbake virtual/kernel -c cleansstate
#        - time bitbake virtual/kernel
#     Part3:
#        - add INHERIT to local.conf
#        - clean build dir
#        - build
#        - report size, remove INHERIT

test1_p1 () {
    log "Running Test 1, part 1/3: Measure wall clock of bitbake $IMAGE and size of tmp/ dir"
    bbnotime $IMAGE -c fetchall
    do_rmtmp
    do_rmsstate
    do_sync
    bbtime $IMAGE
    s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'`
    SIZES[(( size_count++ ))]="$s"
    log "SIZE of tmp dir is: $s"
    log "Buildstats are saved in $OUTDIR/buildstats-test1"
    mv tmp/buildstats $OUTDIR/buildstats-test1
}


test1_p2 () {
    log "Running Test 1, part 2/3: bitbake virtual/kernel -c cleansstate and time bitbake virtual/kernel"
    bbnotime virtual/kernel -c cleansstate
    do_sync
    bbtime virtual/kernel
}

test1_p3 () {
    log "Running Test 1, part 3/3: Build $IMAGE w/o sstate and report size of tmp/dir with rm_work enabled"
    echo "INHERIT += \"rm_work\"" >> conf/local.conf
    do_rmtmp
    do_rmsstate
    do_sync
    bbtime $IMAGE
    sed -i 's/INHERIT += \"rm_work\"//' conf/local.conf
    s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'`
    SIZES[(( size_count++ ))]="$s"
    log "SIZE of tmp dir is: $s"
    log "Buildstats are saved in $OUTDIR/buildstats-test13"
    mv tmp/buildstats $OUTDIR/buildstats-test13
}


#
# Test 2
# Measure: Wall clock of "bitbake core-image-sato" and size of tmp/dir
# Pre: populated sstate cache

test2 () {
    # Assuming test 1 has run
    log "Running Test 2: Measure wall clock of bitbake $IMAGE -c rootfs with sstate"
    do_rmtmp
    do_sync
    bbtime $IMAGE -c rootfs
}


# Test 3
# parsing time metrics
#
#  Start with
#   i) "rm -rf tmp/cache; time bitbake -p"
#  ii) "rm -rf tmp/cache/default-glibc/; time bitbake -p"
# iii) "time bitbake -p"


test3 () {
    log "Running Test 3: Parsing time metrics (bitbake -p)"
    log "   Removing tmp/cache && cache"
    rm -rf tmp/cache cache
    bbtime -p
    log "   Removing tmp/cache/default-glibc/"
    rm -rf tmp/cache/default-glibc/
    bbtime -p
    bbtime -p
}

#
# Test 4 - eSDK
# Measure: eSDK size and installation time
test4 () {
    log "Running Test 4: eSDK size and installation time"
    bbnotime $IMAGE -c do_populate_sdk_ext

    esdk_installer=(tmp/deploy/sdk/*-toolchain-ext-*.sh)

    if [ ${#esdk_installer[*]} -eq 1 ]; then
        s=$((`stat -c %s "$esdk_installer"` / 1024))
        SIZES[(( size_count++ ))]="$s"
        log "Download SIZE of eSDK is: $s kB"

        do_sync
        time_cmd "$esdk_installer" -y -d "tmp/esdk-deploy"

        s=$((`du -sb "tmp/esdk-deploy" | cut -f1` / 1024))
        SIZES[(( size_count++ ))]="$s"
        log "Install SIZE of eSDK is: $s kB"
    else
        log "ERROR: other than one sdk found (${esdk_installer[*]}), reporting size and time as 0."
        SIZES[(( size_count++ ))]="0"
        TIMES[(( time_count++ ))]="0"
    fi

}


# RUN!

test1_p1
test1_p2
test1_p3
test2
test3
test4

# if we got til here write to global results
write_results

log "All done, cleaning up..."

do_rmtmp
do_rmsstate