19 files changed, 2378 insertions, 329 deletions

diff --git a/scripts/contrib/bb-perf/bb-matrix-plot.sh b/scripts/contrib/bb-perf/bb-matrix-plot.sh
index 87e8cb1abd..136a25570d 100755
--- a/scripts/contrib/bb-perf/bb-matrix-plot.sh
+++ b/scripts/contrib/bb-perf/bb-matrix-plot.sh
@@ -115,7 +115,7 @@ set xlabel "$XLABEL"
 set ylabel "$YLABEL"
 set style line 100 lt 5 lw 1.5
 $PM3D_FRAGMENT
-set dgrid3d $PM_CNT,$BB_CNT
+set dgrid3d $PM_CNT,$BB_CNT splines
 set ticslevel 0.2
 
 set term png size $SIZE
diff --git a/scripts/contrib/bb-perf/bb-matrix.sh b/scripts/contrib/bb-perf/bb-matrix.sh
index 37721fe268..106456584d 100755
--- a/scripts/contrib/bb-perf/bb-matrix.sh
+++ b/scripts/contrib/bb-perf/bb-matrix.sh
@@ -63,6 +63,10 @@ for BB in $BB_RANGE; do
 		date
 		echo "BB=$BB PM=$PM Logging to $BB_LOG"
 
+		echo -n "  Preparing the work directory... "
+		rm -rf pseudodone tmp sstate-cache tmp-eglibc &> /dev/null
+		echo "done"
+
 		# Export the variables under test and run the bitbake command
 		# Strip any leading zeroes before passing to bitbake
 		export BB_NUMBER_THREADS=$(echo $BB | sed 's/^0*//')
@@ -70,12 +74,6 @@ for BB in $BB_RANGE; do
 		/usr/bin/time -f "$BB $PM $TIME_STR" -a -o $RUNTIME_LOG $BB_CMD &> $BB_LOG
 
 		echo "  $(tail -n1 $RUNTIME_LOG)"
-		echo -n "  Cleaning up..."
-		mv tmp/buildstats $RUNDIR/$BB-$PM-buildstats
-		rm -f pseudodone &> /dev/null
-		rm -rf tmp &> /dev/null
-		rm -rf sstate-cache &> /dev/null
-		rm -rf tmp-eglibc &> /dev/null
-		echo "done"
+		cp -a tmp/buildstats $RUNDIR/$BB-$PM-buildstats
 	done
 done
diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh b/scripts/contrib/bb-perf/buildstats-plot.sh
new file mode 100755
index 0000000000..7e8ae0410e
--- /dev/null
+++ b/scripts/contrib/bb-perf/buildstats-plot.sh
@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2011, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# DESCRIPTION
+#
+# Produces script data to be consumed by gnuplot. There are two possible plots
+# depending if either the -S parameter is present or not:
+#
+#     * without -S: Produces a histogram listing top N recipes/tasks versus
+#       stats. The first stat defined in the -s parameter is the one taken
+#       into account for ranking
+#     * -S: Produces a histogram listing tasks versus stats.  In this case,
+#       the value of each stat is the sum for that particular stat in all recipes found.
+#       Stats values  are in descending order defined by the first stat defined on -s
+#
+# EXAMPLES
+#
+# 1. Top recipes' tasks taking into account utime
+#
+#     $ buildstats-plot.sh -s utime | gnuplot -p
+#
+# 2. Tasks versus utime:stime
+#
+#     $ buildstats-plot.sh -s utime:stime -S | gnuplot -p
+#
+# 3. Tasks versus IO write_bytes:IO read_bytes
+#
+#     $ buildstats-plot.sh -s 'IO write_bytes:IO read_bytes' -S | gnuplot -p
+#
+# AUTHORS
+# Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
+#
+
+set -o nounset
+set -o errexit
+
+BS_DIR="tmp/buildstats"
+N=10
+STATS="utime"
+SUM=""
+OUTDATA_FILE="$PWD/buildstats-plot.out"
+
+function usage {
+    CMD=$(basename $0)
+    cat <<EOM
+Usage: $CMD [-b buildstats_dir] [-t do_task]
+  -b buildstats The path where the folder resides
+                (default: "$BS_DIR")
+  -n N          Top N recipes to display. Ignored if -S is present
+                (default: "$N")
+  -s stats      The stats to be matched. If more that one stat, units
+                should be the same because data is plot as histogram.
+                (see buildstats.sh -h for all options) or any other defined
+                (build)stat separated by colons, i.e. stime:utime
+                (default: "$STATS")
+  -S            Sum values for a particular stat for found recipes
+  -o            Output data file.
+                (default: "$OUTDATA_FILE")
+  -h            Display this help message
+EOM
+}
+
+# Parse and validate arguments
+while getopts "b:n:s:o:Sh" OPT; do
+	case $OPT in
+	b)
+		BS_DIR="$OPTARG"
+		;;
+	n)
+		N="$OPTARG"
+		;;
+	s)
+	        STATS="$OPTARG"
+	        ;;
+	S)
+	        SUM="y"
+	        ;;
+	o)
+	        OUTDATA_FILE="$OPTARG"
+	        ;;
+	h)
+		usage
+		exit 0
+		;;
+	*)
+		usage
+		exit 1
+		;;
+	esac
+done
+
+# Get number of stats
+IFS=':'; statsarray=(${STATS}); unset IFS
+nstats=${#statsarray[@]}
+
+# Get script folder, use to run buildstats.sh
+CD=$(dirname $0)
+
+# Parse buildstats recipes to produce a single table
+OUTBUILDSTATS="$PWD/buildstats.log"
+$CD/buildstats.sh -H -s "$STATS" -H > $OUTBUILDSTATS
+
+# Get headers
+HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp')
+
+echo -e "set boxwidth 0.9 relative"
+echo -e "set style data histograms"
+echo -e "set style fill solid 1.0 border lt -1"
+echo -e "set xtics rotate by 45 right"
+
+# Get output data
+if [ -z "$SUM" ]; then
+    cat $OUTBUILDSTATS | sed -e '1d' | sort -k3 -n -r | head -$N > $OUTDATA_FILE
+    # include task at recipe column
+    sed -i -e "1i\
+${HEADERS}" $OUTDATA_FILE
+    echo -e "set title \"Top task/recipes\""
+    echo -e "plot for [COL=3:`expr 3 + ${nstats} - 1`] '${OUTDATA_FILE}' using COL:xtic(stringcolumn(1).' '.stringcolumn(2)) title columnheader(COL)"
+else
+
+    # Construct datatamash sum argument (sum 3 sum 4 ...)
+    declare -a sumargs
+    j=0
+    for i in `seq $nstats`; do
+	sumargs[j]=sum; j=$(( $j + 1 ))
+	sumargs[j]=`expr 3 + $i - 1`;  j=$(( $j + 1 ))
+    done
+
+    # Do the processing with datamash
+    cat $OUTBUILDSTATS | sed -e '1d' | datamash -t ' ' -g1 ${sumargs[*]} | sort -k2 -n -r > $OUTDATA_FILE
+
+    # Include headers into resulted file, so we can include gnuplot xtics
+    HEADERS=$(echo $HEADERS | sed -e 's/recipe//1')
+    sed -i -e "1i\
+${HEADERS}" $OUTDATA_FILE
+
+    # Plot
+    echo -e "set title \"Sum stats values per task for all recipes\""
+    echo -e "plot for [COL=2:`expr 2 + ${nstats} - 1`] '${OUTDATA_FILE}' using COL:xtic(1) title columnheader(COL)"
+fi
+
diff --git a/scripts/contrib/bb-perf/buildstats.sh b/scripts/contrib/bb-perf/buildstats.sh
new file mode 100755
index 0000000000..8d7e2488f0
--- /dev/null
+++ b/scripts/contrib/bb-perf/buildstats.sh
@@ -0,0 +1,155 @@
+#!/bin/bash
+#
+# Copyright (c) 2011, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# DESCRIPTION
+# Given 'buildstats' data (generate by bitbake when setting
+# USER_CLASSES ?= "buildstats" on local.conf), task names and a stats values
+# (these are the ones preset on the buildstats files), outputs
+# '<task> <recipe> <value_1> <value_2> ... <value_n>'. The units are the ones
+# defined at buildstats, which in turn takes data from /proc/[pid] files
+#
+# Some useful pipelines
+#
+# 1. Tasks with largest stime (Amount of time that this process has been scheduled
+#    in kernel mode) values
+# $ buildstats.sh -b <buildstats> -s stime | sort -k3 -n -r | head
+#
+# 2. Min, max, sum utime (Amount  of  time  that  this process has been scheduled
+#    in user mode) per task (in needs GNU datamash)
+# $ buildstats.sh -b <buildstats> -s utime | datamash -t' ' -g1 min 3 max 3 sum 3 | sort -k4 -n -r
+#
+# AUTHORS
+# Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
+#
+
+# Stats, by type
+TIME="utime:stime:cutime:cstime"
+IO="IO wchar:IO write_bytes:IO syscr:IO read_bytes:IO rchar:IO syscw:IO cancelled_write_bytes"
+RUSAGE="rusage ru_utime:rusage ru_stime:rusage ru_maxrss:rusage ru_minflt:rusage ru_majflt:\
+rusage ru_inblock:rusage ru_oublock:rusage ru_nvcsw:rusage ru_nivcsw"
+
+CHILD_RUSAGE="Child rusage ru_utime:Child rusage ru_stime:Child rusage ru_maxrss:Child rusage ru_minflt:\
+Child rusage ru_majflt:Child rusage ru_inblock:Child rusage ru_oublock:Child rusage ru_nvcsw:\
+Child rusage ru_nivcsw"
+
+BS_DIR="tmp/buildstats"
+TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack"
+STATS="$TIME"
+HEADER="" # No header by default
+
+function usage {
+CMD=$(basename $0)
+cat <<EOM
+Usage: $CMD [-b buildstats_dir] [-t do_task]
+  -b buildstats The path where the folder resides
+                (default: "$BS_DIR")
+  -t tasks      The tasks to be computed
+                (default: "$TASKS")
+  -s stats      The stats to be matched. Options: TIME, IO, RUSAGE, CHILD_RUSAGE
+                or any other defined buildstat separated by colons, i.e. stime:utime
+                (default: "$STATS")
+                Default stat sets:
+                    TIME=$TIME
+                    IO=$IO
+                    RUSAGE=$RUSAGE
+                    CHILD_RUSAGE=$CHILD_RUSAGE
+  -h            Display this help message
+EOM
+}
+
+# Parse and validate arguments
+while getopts "b:t:s:Hh" OPT; do
+	case $OPT in
+	b)
+		BS_DIR="$OPTARG"
+		;;
+	t)
+		TASKS="$OPTARG"
+		;;
+	s)
+		STATS="$OPTARG"
+		;;
+	H)
+	        HEADER="y"
+	        ;;
+	h)
+		usage
+		exit 0
+		;;
+	*)
+		usage
+		exit 1
+		;;
+	esac
+done
+
+# Ensure the buildstats folder exists
+if [ ! -d "$BS_DIR" ]; then
+	echo "ERROR: $BS_DIR does not exist"
+	usage
+	exit 1
+fi
+
+stats=""
+IFS=":"
+for stat in ${STATS}; do
+	case $stat in
+	    TIME)
+		stats="${stats}:${TIME}"
+		;;
+	    IO)
+		stats="${stats}:${IO}"
+		;;
+	    RUSAGE)
+		stats="${stats}:${RUSAGE}"
+		;;
+	    CHILD_RUSAGE)
+		stats="${stats}:${CHILD_RUSAGE}"
+		;;
+	    *)
+		stats="${STATS}"
+	esac
+done
+
+# remove possible colon at the beginning
+stats="$(echo "$stats" | sed -e 's/^://1')"
+
+# Provide a header if required by the user
+[ -n "$HEADER" ] && { echo "task:recipe:$stats"; }
+
+for task in ${TASKS}; do
+    task="do_${task}"
+    for file in $(find ${BS_DIR} -type f -name ${task} | awk 'BEGIN{ ORS=""; OFS=":" } { print $0,"" }'); do
+        recipe="$(basename $(dirname $file))"
+	times=""
+	for stat in ${stats}; do
+	    [ -z "$stat" ] && { echo "empty stats"; }
+	    time=$(sed -n -e "s/^\($stat\): \\(.*\\)/\\2/p" $file)
+	    # in case the stat is not present, set the value as NA
+	    [ -z "$time" ] && { time="NA"; }
+	    # Append it to times
+	    if [ -z "$times" ]; then
+		times="${time}"
+	    else
+		times="${times} ${time}"
+	    fi
+	done
+        echo "${task} ${recipe} ${times}"
+    done
+done
diff --git a/scripts/contrib/bbvars.py b/scripts/contrib/bbvars.py
index 0896d64445..d8d0594776 100755
--- a/scripts/contrib/bbvars.py
+++ b/scripts/contrib/bbvars.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -24,12 +24,12 @@ import os.path
 import re
 
 def usage():
-    print 'Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0])
-    print '  -d FILENAME         documentation file to search'
-    print '  -h, --help          display this help and exit'
-    print '  -m METADIR          meta directory to search for recipes'
-    print '  -t FILENAME         documentation config file (for doc tags)'
-    print '  -T                  Only display variables with doc tags (requires -t)'
+    print('Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0]))
+    print('  -d FILENAME         documentation file to search')
+    print('  -h, --help          display this help and exit')
+    print('  -m METADIR          meta directory to search for recipes')
+    print('  -t FILENAME         documentation config file (for doc tags)')
+    print('  -T                  Only display variables with doc tags (requires -t)')
 
 def recipe_bbvars(recipe):
     ''' Return a unique set of every bbvar encountered in the recipe '''
@@ -37,9 +37,9 @@ def recipe_bbvars(recipe):
     vset = set()
     try:
         r = open(recipe)
-    except IOError as (errno, strerror):
-        print 'WARNING: Failed to open recipe ', recipe
-        print strerror
+    except IOError as err:
+        print('WARNING: Failed to open recipe ', recipe)
+        print(err.args[1])
 
     for line in r:
         # Strip any comments from the line
@@ -59,8 +59,8 @@ def collect_bbvars(metadir):
     for root,dirs,files in os.walk(metadir):
         for name in files:
             if name.find(".bb") >= 0:
-                for key in recipe_bbvars(os.path.join(root,name)).iterkeys():
-                    if bbvars.has_key(key):
+                for key in recipe_bbvars(os.path.join(root,name)).keys():
+                    if key in bbvars:
                         bbvars[key] = bbvars[key] + 1
                     else:
                         bbvars[key] = 1
@@ -71,9 +71,9 @@ def bbvar_is_documented(var, docfiles):
     for doc in docfiles:
         try:
             f = open(doc)
-        except IOError as (errno, strerror):
-            print 'WARNING: Failed to open doc ', doc
-            print strerror
+        except IOError as err:
+            print('WARNING: Failed to open doc ', doc)
+            print(err.args[1])
         for line in f:
             if prog.match(line):
                 return True
@@ -87,8 +87,8 @@ def bbvar_doctag(var, docconf):
 
     try:
         f = open(docconf)
-    except IOError as (errno, strerror):
-        return strerror
+    except IOError as err:
+        return err.args[1]
 
     for line in f:
         m = prog.search(line)
@@ -109,8 +109,8 @@ def main():
     # Collect and validate input
     try:
         opts, args = getopt.getopt(sys.argv[1:], "d:hm:t:T", ["help"])
-    except getopt.GetoptError, err:
-        print '%s' % str(err)
+    except getopt.GetoptError as err:
+        print('%s' % str(err))
         usage()
         sys.exit(2)
 
@@ -122,13 +122,13 @@ def main():
             if os.path.isfile(a):
                 docfiles.append(a)
             else:
-                print 'ERROR: documentation file %s is not a regular file' % (a)
+                print('ERROR: documentation file %s is not a regular file' % a)
                 sys.exit(3)
         elif o == '-m':
             if os.path.isdir(a):
                 metadirs.append(a)
             else:
-                print 'ERROR: meta directory %s is not a directory' % (a)
+                print('ERROR: meta directory %s is not a directory' % a)
                 sys.exit(4)
         elif o == "-t":
             if os.path.isfile(a):
@@ -139,31 +139,31 @@ def main():
             assert False, "unhandled option"
 
     if len(docfiles) == 0:
-        print 'ERROR: no docfile specified'
+        print('ERROR: no docfile specified')
         usage()
         sys.exit(5)
 
     if len(metadirs) == 0:
-        print 'ERROR: no metadir specified'
+        print('ERROR: no metadir specified')
         usage()
         sys.exit(6)
 
     if onlydoctags and docconf == "":
-        print 'ERROR: no docconf specified'
+        print('ERROR: no docconf specified')
         usage()
         sys.exit(7)
 
     # Collect all the variable names from the recipes in the metadirs
     for m in metadirs:
-        for key,cnt in collect_bbvars(m).iteritems():
-            if bbvars.has_key(key):
+        for key,cnt in collect_bbvars(m).items():
+            if key in bbvars:
                 bbvars[key] = bbvars[key] + cnt
             else:
                 bbvars[key] = cnt
 
     # Check each var for documentation
     varlen = 0
-    for v in bbvars.iterkeys():
+    for v in bbvars.keys():
         if len(v) > varlen:
             varlen = len(v)
         if not bbvar_is_documented(v, docfiles):
@@ -172,14 +172,14 @@ def main():
     varlen = varlen + 1
 
     # Report all undocumented variables
-    print 'Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars))
+    print('Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars)))
     header = '%s%s%s' % (str("VARIABLE").ljust(varlen), str("COUNT").ljust(6), str("DOCTAG").ljust(7))
-    print header
-    print str("").ljust(len(header), '=')
+    print(header)
+    print(str("").ljust(len(header), '='))
     for v in undocumented:
         doctag = bbvar_doctag(v, docconf)
         if not onlydoctags or not doctag == "":
-            print '%s%s%s' % (v.ljust(varlen), str(bbvars[v]).ljust(6), doctag)
+            print('%s%s%s' % (v.ljust(varlen), str(bbvars[v]).ljust(6), doctag))
 
 
 if __name__ == "__main__":
diff --git a/scripts/contrib/build-perf-test-wrapper.sh b/scripts/contrib/build-perf-test-wrapper.sh
new file mode 100755
index 0000000000..3da32532be
--- /dev/null
+++ b/scripts/contrib/build-perf-test-wrapper.sh
@@ -0,0 +1,219 @@
+#!/bin/bash
+#
+# Build performance test script wrapper
+#
+# Copyright (c) 2016, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+#
+# This script is a simple wrapper around the actual build performance tester
+# script. This script initializes the build environment, runs
+# oe-build-perf-test and archives the results.
+
+script=`basename $0`
+script_dir=$(realpath $(dirname $0))
+archive_dir=~/perf-results/archives
+
+usage () {
+cat << EOF
+Usage: $script [-h] [-c COMMITISH] [-C GIT_REPO]
+
+Optional arguments:
+  -h                show this help and exit.
+  -a ARCHIVE_DIR    archive results tarball here, give an empty string to
+                    disable tarball archiving (default: $archive_dir)
+  -c COMMITISH      test (checkout) this commit, <branch>:<commit> can be
+                    specified to test specific commit of certain branch
+  -C GIT_REPO       commit results into Git
+  -E EMAIL_ADDR     send email report
+  -P GIT_REMOTE     push results to a remote Git repository
+  -w WORK_DIR       work dir for this script
+                    (default: GIT_TOP_DIR/build-perf-test)
+  -x                create xml report (instead of json)
+EOF
+}
+
+get_os_release_var () {
+    ( source /etc/os-release; eval echo '$'$1 )
+}
+
+
+# Parse command line arguments
+commitish=""
+oe_build_perf_test_extra_opts=()
+oe_git_archive_extra_opts=()
+while getopts "ha:c:C:E:P:w:x" opt; do
+    case $opt in
+        h)  usage
+            exit 0
+            ;;
+        a)  archive_dir=`realpath -s "$OPTARG"`
+            ;;
+        c)  commitish=$OPTARG
+            ;;
+        C)  results_repo=`realpath -s "$OPTARG"`
+            ;;
+        E)  email_to="$OPTARG"
+            ;;
+        P)  oe_git_archive_extra_opts+=("--push" "$OPTARG")
+            ;;
+        w)  base_dir=`realpath -s "$OPTARG"`
+            ;;
+        x)  oe_build_perf_test_extra_opts+=("--xml")
+            ;;
+        *)  usage
+            exit 1
+            ;;
+    esac
+done
+
+# Check positional args
+shift "$((OPTIND - 1))"
+if [ $# -ne 0 ]; then
+    echo "ERROR: No positional args are accepted."
+    usage
+    exit 1
+fi
+
+# Open a file descriptor for flock and acquire lock
+LOCK_FILE="/tmp/oe-build-perf-test-wrapper.lock"
+if ! exec 3> "$LOCK_FILE"; then
+    echo "ERROR: Unable to open lock file"
+    exit 1
+fi
+if ! flock -n 3; then
+    echo "ERROR: Another instance of this script is running"
+    exit 1
+fi
+
+echo "Running on `uname -n`"
+if ! git_topdir=$(git rev-parse --show-toplevel); then
+        echo "The current working dir doesn't seem to be a git clone. Please cd there before running `basename $0`"
+        exit 1
+fi
+
+cd "$git_topdir"
+
+if [ -n "$commitish" ]; then
+    echo "Running git fetch"
+    git fetch &> /dev/null
+    git checkout HEAD^0 &> /dev/null
+
+    # Handle <branch>:<commit> format
+    if echo "$commitish" | grep -q ":"; then
+        commit=`echo "$commitish" | cut -d":" -f2`
+        branch=`echo "$commitish" | cut -d":" -f1`
+    else
+        commit="$commitish"
+        branch="$commitish"
+    fi
+
+    echo "Checking out $commitish"
+    git branch -D $branch &> /dev/null
+    if ! git checkout -f $branch &> /dev/null; then
+        echo "ERROR: Git checkout failed"
+        exit 1
+    fi
+
+    # Check that the specified branch really contains the commit
+    commit_hash=`git rev-parse --revs-only $commit --`
+    if [ -z "$commit_hash" -o "`git merge-base $branch $commit`" != "$commit_hash" ]; then
+        echo "ERROR: branch $branch does not contain commit $commit"
+        exit 1
+    fi
+    git reset --hard $commit > /dev/null
+fi
+
+# Setup build environment
+if [ -z "$base_dir" ]; then
+    base_dir="$git_topdir/build-perf-test"
+fi
+echo "Using working dir $base_dir"
+
+timestamp=`date "+%Y%m%d%H%M%S"`
+git_rev=$(git rev-parse --short HEAD)  || exit 1
+build_dir="$base_dir/build-$git_rev-$timestamp"
+results_dir="$base_dir/results-$git_rev-$timestamp"
+globalres_log="$base_dir/globalres.log"
+machine="qemux86"
+
+mkdir -p "$base_dir"
+source ./oe-init-build-env $build_dir >/dev/null || exit 1
+
+# Additional config
+auto_conf="$build_dir/conf/auto.conf"
+echo "MACHINE = \"$machine\"" > "$auto_conf"
+echo 'BB_NUMBER_THREADS = "8"' >> "$auto_conf"
+echo 'PARALLEL_MAKE = "-j 8"' >> "$auto_conf"
+echo "DL_DIR = \"$base_dir/downloads\"" >> "$auto_conf"
+# Disabling network sanity check slightly reduces the variance of timing results
+echo 'CONNECTIVITY_CHECK_URIS = ""' >> "$auto_conf"
+# Possibility to define extra settings
+if [ -f "$base_dir/auto.conf.extra" ]; then
+    cat "$base_dir/auto.conf.extra" >> "$auto_conf"
+fi
+
+# Run actual test script
+oe-build-perf-test --out-dir "$results_dir" \
+                   --globalres-file "$globalres_log" \
+                   "${oe_build_perf_test_extra_opts[@]}" \
+                   --lock-file "$base_dir/oe-build-perf.lock"
+
+case $? in
+    1)  echo "ERROR: oe-build-perf-test script failed!"
+        exit 1
+        ;;
+    2)  echo "NOTE: some tests failed!"
+        ;;
+esac
+
+# Commit results to git
+if [ -n "$results_repo" ]; then
+    echo -e "\nArchiving results in $results_repo"
+    oe-git-archive \
+        --git-dir "$results_repo" \
+        --branch-name "{hostname}/{branch}/{machine}" \
+        --tag-name "{hostname}/{branch}/{machine}/{commit_count}-g{commit}/{tag_number}" \
+        --exclude "buildstats.json" \
+        --notes "buildstats/{branch_name}" "$results_dir/buildstats.json" \
+        "${oe_git_archive_extra_opts[@]}" \
+        "$results_dir"
+
+    # Send email report
+    if [ -n "$email_to" ]; then
+        echo -e "\nEmailing test report"
+        os_name=`get_os_release_var PRETTY_NAME`
+        oe-build-perf-report -r "$results_repo" > report.txt
+        oe-build-perf-report -r "$results_repo" --html > report.html
+        "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text report.txt --html report.html "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}"
+    fi
+fi
+
+
+echo -ne "\n\n-----------------\n"
+echo "Global results file:"
+echo -ne "\n"
+
+cat "$globalres_log"
+
+if [ -n "$archive_dir" ]; then
+    echo -ne "\n\n-----------------\n"
+    echo "Archiving results in $archive_dir"
+    mkdir -p "$archive_dir"
+    results_basename=`basename "$results_dir"`
+    results_dirname=`dirname "$results_dir"`
+    tar -czf "$archive_dir/`uname -n`-${results_basename}.tar.gz" -C "$results_dirname" "$results_basename"
+fi
+
+rm -rf "$build_dir"
+rm -rf "$results_dir"
+
+echo "DONE"
diff --git a/scripts/contrib/build-perf-test.sh b/scripts/contrib/build-perf-test.sh
index ce0fb9afd8..7d99228c73 100755
--- a/scripts/contrib/build-perf-test.sh
+++ b/scripts/contrib/build-perf-test.sh
@@ -128,7 +128,7 @@ rev=$(git rev-parse --short HEAD)  || exit 1
 OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`"
 BUILDDIR="$OUTDIR/build"
 resultsfile="$OUTDIR/results.log"
-bboutput="$OUTDIR/bitbake.log"
+cmdoutput="$OUTDIR/commands.log"
 myoutput="$OUTDIR/output.log"
 globalres="$clonedir/build-perf-test/globalres.log"
 
@@ -170,6 +170,7 @@ fi
 # Disabling the network sanity check helps a bit (because of my crappy network connection and/or proxy)
 echo "CONNECTIVITY_CHECK_URIS =\"\"" >> conf/local.conf
 
+
 #
 # Functions
 #
@@ -179,14 +180,13 @@ time_count=0
 declare -a SIZES
 size_count=0
 
-bbtime () {
-    local arg="$@"
-    log "   Timing: bitbake ${arg}"
+time_cmd () {
+    log "   Timing: $*"
 
     if [ $verbose -eq 0 ]; then 
-        /usr/bin/time -v -o $resultsfile bitbake ${arg} >> $bboutput
+        /usr/bin/time -v -o $resultsfile "$@" >> $cmdoutput
     else
-        /usr/bin/time -v -o $resultsfile bitbake ${arg}
+        /usr/bin/time -v -o $resultsfile "$@"
     fi
     ret=$?
     if [ $ret -eq 0 ]; then
@@ -205,12 +205,16 @@ bbtime () {
     log "More stats can be found in ${resultsfile}.${i}"    
 }
 
+bbtime () {
+    time_cmd bitbake "$@"
+}
+
 #we don't time bitbake here
 bbnotime () {
     local arg="$@"
     log "   Running: bitbake ${arg}"
     if [ $verbose -eq 0 ]; then
-        bitbake ${arg} >> $bboutput
+        bitbake ${arg} >> $cmdoutput
     else
         bitbake ${arg}
     fi
@@ -284,7 +288,7 @@ test1_p1 () {
     do_rmsstate
     do_sync
     bbtime $IMAGE
-    s=`du -sh tmp | sed 's/tmp//'`
+    s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'`
     SIZES[(( size_count++ ))]="$s"
     log "SIZE of tmp dir is: $s"
     log "Buildstats are saved in $OUTDIR/buildstats-test1"
@@ -307,7 +311,7 @@ test1_p3 () {
     do_sync
     bbtime $IMAGE
     sed -i 's/INHERIT += \"rm_work\"//' conf/local.conf
-    s=`du -sh tmp | sed 's/tmp//'`
+    s=`du -s tmp | sed 's/tmp//' | sed 's/[ \t]*$//'`
     SIZES[(( size_count++ ))]="$s"
     log "SIZE of tmp dir is: $s"
     log "Buildstats are saved in $OUTDIR/buildstats-test13"
@@ -334,7 +338,7 @@ test2 () {
 #
 #  Start with
 #   i) "rm -rf tmp/cache; time bitbake -p"
-#  ii) "rm -rf tmp/cache/default-eglibc/; time bitbake -p"
+#  ii) "rm -rf tmp/cache/default-glibc/; time bitbake -p"
 # iii) "time bitbake -p"
 
 
@@ -343,12 +347,39 @@ test3 () {
     log "   Removing tmp/cache && cache"
     rm -rf tmp/cache cache
     bbtime -p
-    log "   Removing tmp/cache/default-eglibc/"
-    rm -rf tmp/cache/default-eglibc/
+    log "   Removing tmp/cache/default-glibc/"
+    rm -rf tmp/cache/default-glibc/
     bbtime -p
     bbtime -p
 }
 
+#
+# Test 4 - eSDK
+# Measure: eSDK size and installation time
+test4 () {
+    log "Running Test 4: eSDK size and installation time"
+    bbnotime $IMAGE -c do_populate_sdk_ext
+
+    esdk_installer=(tmp/deploy/sdk/*-toolchain-ext-*.sh)
+
+    if [ ${#esdk_installer[*]} -eq 1 ]; then
+        s=$((`stat -c %s "$esdk_installer"` / 1024))
+        SIZES[(( size_count++ ))]="$s"
+        log "Download SIZE of eSDK is: $s kB"
+
+        do_sync
+        time_cmd "$esdk_installer" -y -d "tmp/esdk-deploy"
+
+        s=$((`du -sb "tmp/esdk-deploy" | cut -f1` / 1024))
+        SIZES[(( size_count++ ))]="$s"
+        log "Install SIZE of eSDK is: $s kB"
+    else
+        log "ERROR: other than one sdk found (${esdk_installer[*]}), reporting size and time as 0."
+        SIZES[(( size_count++ ))]="0"
+        TIMES[(( time_count++ ))]="0"
+    fi
+
+}
 
 
 # RUN!
@@ -358,6 +389,7 @@ test1_p2
 test1_p3
 test2
 test3
+test4
 
 # if we got til here write to global results
 write_results
diff --git a/scripts/contrib/ddimage b/scripts/contrib/ddimage
index 93ebeafc31..ab929957a5 100755
--- a/scripts/contrib/ddimage
+++ b/scripts/contrib/ddimage
@@ -1,7 +1,8 @@
 #!/bin/sh
 
-#BLACKLIST_DEVICES="/dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde"
-BLACKLIST_DEVICES="/dev/sda"
+# Default to avoiding the first two disks on typical Linux and Mac OS installs
+# Better safe than sorry :-)
+BLACKLIST_DEVICES="/dev/sda /dev/sdb /dev/disk1 /dev/disk2"
 
 # 1MB blocksize
 BLOCKSIZE=1048576
@@ -14,9 +15,15 @@ image_details() {
 	IMG=$1
 	echo "Image details"
 	echo "============="
-	echo "    image: $(stat --printf '%N\n' $IMG)"
-	echo "     size: $(stat -L --printf '%s bytes\n' $IMG)"
-	echo " modified: $(stat -L --printf '%y\n' $IMG)"
+	echo "    image: $(basename $IMG)"
+	# stat format is different on Mac OS and Linux
+	if [ "$(uname)" = "Darwin" ]; then
+		echo "     size: $(stat -L -f '%z bytes' $IMG)"
+		echo " modified: $(stat -L -f '%Sm' $IMG)"
+	else
+		echo "     size: $(stat -L -c '%s bytes' $IMG)"
+		echo " modified: $(stat -L -c '%y' $IMG)"
+	fi
 	echo "     type: $(file -L -b $IMG)"
 	echo ""
 }
@@ -27,6 +34,14 @@ device_details() {
 
 	echo "Device details"
 	echo "=============="