#!/bin/sh # Author: Martin Jansa <martin.jansa@gmail.com> # # Copyright (c) 2013 Martin Jansa <Martin.Jansa@gmail.com> # Used to detect missing dependencies or automagically # enabled dependencies which aren't explicitly enabled # or disabled. # It does 3 builds of <target> # 1st to populate sstate-cache directory and sysroot # 2nd to rebuild each recipe with every possible # dependency found in sysroot (which stays populated # from 1st build # 3rd to rebuild each recipe only with dependencies defined # in DEPENDS # 4th (optional) repeat build like 3rd to make sure that # minimal versions of dependencies defined in DEPENDS # is also enough # Global vars tmpdir= targets= recipes= buildhistory= buildtype= default_targets="world" default_buildhistory="buildhistory" default_buildtype="1 2 3 c" usage () { cat << EOF Welcome to utility to detect missing or autoenabled dependencies. WARNING: this utility will completely remove your tmpdir (make sure you don't have important buildhistory or persistent dir there). $0 <OPTION> Options: -h, --help Display this help and exit. --tmpdir=<tmpdir> Specify tmpdir, will use the environment variable TMPDIR if it is not specified. Something like /OE/oe-core/tmp-eglibc (no / at the end). --targets=<targets> List of targets separated by space, will use the environment variable TARGETS if it is not specified. It will run "bitbake <targets>" to populate sysroots. Default value is "world". --recipes=<recipes> File with list of recipes we want to rebuild with minimal and maximal sysroot. Will use the environment variable RECIPES if it is not specified. Default value will use all packages ever recorded in buildhistory directory. --buildhistory=<buildhistory> Path to buildhistory directory, it needs to be enabled in your config, because it's used to detect different dependencies and to create list of recipes to rebuild when it's not specified. Will use the environment variable BUILDHISTORY if it is not specified. Default value is "buildhistory" --buildtype=<buildtype> There are 4 types of build: 1: build to populate sstate-cache directory and sysroot 2: build to rebuild each recipe with every possible dep 3: build to rebuild each recipe with minimal dependencies 4: build to rebuild each recipe again with minimal dependencies c: compare buildhistory directories from build 2 and 3 Will use the environment variable BUILDTYPE if it is not specified. Default value is "1 2 3 c", order is important, type 4 is optional. EOF } # Print error information and exit. echo_error () { echo "ERROR: $1" >&2 exit 1 } while [ -n "$1" ]; do case $1 in --tmpdir=*) tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e` [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir" shift ;; --targets=*) targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'` shift ;; --recipes=*) recipes=`echo $1 | sed -e 's#^--recipes="*\([^"]*\)"*#\1#'` shift ;; --buildhistory=*) buildhistory=`echo $1 | sed -e 's#^--buildhistory="*\([^"]*\)"*#\1#'` shift ;; --buildtype=*) buildtype=`echo $1 | sed -e 's#^--buildtype="*\([^"]*\)"*#\1#'` shift ;; --help|-h) usage exit 0 ;; *) echo "Invalid arguments $*" echo_error "Try '$0 -h' for more information." ;; esac done # tmpdir directory, use environment variable TMPDIR # if it was not specified, otherwise, error. [ -n "$tmpdir" ] || tmpdir=$TMPDIR [ -n "$tmpdir" ] || echo_error "No tmpdir found!" [ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\"" [ -n "$targets" ] || targets=$TARGETS [ -n "$targets" ] || targets=$default_targets [ -n "$recipes" ] || recipes=$RECIPES [ -n "$recipes" -a ! -f "$recipes" ] && echo_error "Invalid file with list of recipes to rebuild" [ -n "$recipes" ] || echo "All packages ever recorded in buildhistory directory will be rebuilt" [ -n "$buildhistory" ] || buildhistory=$BUILDHISTORY [ -n "$buildhistory" ] || buildhistory=$default_buildhistory [ -d "$buildhistory" ] || echo_error "Invalid buildhistory directory \"$buildhistory\"" [ -n "$buildtype" ] || buildtype=$BUILDTYPE [ -n "$buildtype" ] || buildtype=$default_buildtype echo "$buildtype" | grep -v '^[1234c ]*$' && echo_error "Invalid buildtype \"$buildtype\", only some combination of 1, 2, 3, 4, c separated by space is allowed" OUTPUT_BASE=test-dependencies/`date "+%s"` build_all() { echo "===== 1st build to populate sstate-cache directory and sysroot =====" OUTPUT1=${OUTPUT_BASE}/${TYPE}_all mkdir -p ${OUTPUT1} echo "Logs will be stored in ${OUTPUT1} directory" bitbake -k $targets | tee -a ${OUTPUT1}/complete.log } build_every_recipe() { if [ "${TYPE}" = "2" ] ; then echo "===== 2nd build to rebuild each recipe with every possible dep =====" OUTPUT_MAX=${OUTPUT_BASE}/${TYPE}_max OUTPUTB=${OUTPUT_MAX} else echo "===== 3rd or 4th build to rebuild each recipe with minimal dependencies =====" OUTPUT_MIN=${OUTPUT_BASE}/${TYPE}_min OUTPUTB=${OUTPUT_MIN} fi mkdir -p ${OUTPUTB} ${OUTPUTB}/failed ${OUTPUTB}/ok echo "Logs will be stored in ${OUTPUTB} directory" if [ -z "$recipes" ]; then ls -d $buildhistory/packages/*/* | xargs -n 1 basename | sort -u > ${OUTPUTB}/recipe.list recipes=${OUTPUTB}/recipe.list fi if [ "${TYPE}" != "2" ] ; then echo "!!!Removing tmpdir \"$tmpdir\"!!!" rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null fi i=1 count=`cat $recipes | wc -l` for recipe in `cat $recipes`; do echo "Building recipe: ${recipe} ($i/$count)" bitbake -c cleansstate ${recipe} > ${OUTPUTB}/log.${recipe} 2>&1; bitbake ${recipe} >> ${OUTPUTB}/log.${recipe} 2>&1; grep "ERROR: Task.*failed" ${OUTPUTB}/log.${recipe} && mv ${OUTPUTB}/log.${recipe} ${OUTPUTB}/failed/${recipe} || mv ${OUTPUTB}/log.${recipe} ${OUTPUTB}/ok/${recipe} if [ "${TYPE}" != "2" ] ; then rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null fi i=`expr $i + 1` done echo "Copying buildhistory/packages to ${OUTPUTB}" cp -ra $buildhistory/packages ${OUTPUTB} # This will be usefull to see which library is pulling new dependency echo "Copying do_package logs to ${OUTPUTB}/do_package/" mkdir ${OUTPUTB}/do_package find $tmpdir/work/ -name log.do_package | while read f; do # pn is 3 levels back, but we don't know if there is just one log per pn (only one arch and version) # dest=`echo $f | sed 's#^.*/\([^/]*\)/\([^/]*\)/\([^/]*\)/log.do_package#\1#g'` dest=`echo $f | sed "s#$tmpdir/work/##g; s#/#_#g"` cp $f ${OUTPUTB}/do_package/$dest done grep "ERROR: Task.*failed" ${OUTPUTB}/failed/* ls -1 ${OUTPUTB}/failed/* >> ${OUTPUT_BASE}/failed.recipes } compare_deps() { # you can run just compare task with command like this # OUTPUT_BASE=test-dependencies/1373140172 \ # OUTPUT_MAX=${OUTPUT_BASE}/2_max \ # OUTPUT_MIN=${OUTPUT_BASE}/3_min \ # openembedded-core/scripts/test-dependencies.sh --tmpdir=tmp-eglibc --targets=glib-2.0 --recipes=recipe_list --buildtype=c echo "===== Compare dependencies recorded in \"${OUTPUT_MAX}\" and \"${OUTPUT_MIN}\" =====" [ -n "${OUTPUTC}" ] || OUTPUTC=${OUTPUT_BASE} mkdir -p ${OUTPUTC} OUTPUT_FILE=${OUTPUTC}/dependency-changes echo "Differences will be stored in ${OUTPUT_FILE}, dot is shown for every 100 of checked packages" echo > ${OUTPUT_FILE} [ -d ${OUTPUT_MAX} ] || echo_error "Directory with output from build 2 \"${OUTPUT_MAX}\" does not exist" [ -d ${OUTPUT_MIN} ] || echo_error "Directory with output from build 3 \"${OUTPUT_MIN}\" does not exist" [ -d ${OUTPUT_MAX}/packages/ ] || echo_error "Directory with packages from build 2 \"${OUTPUT_MAX}/packages/\" does not exist" [ -d ${OUTPUT_MIN}/packages/ ] || echo_error "Directory with packages from build 3 \"${OUTPUT_MIN}/packages/\" does not exist" i=0 find ${OUTPUT_MAX}/packages/ -name latest | sed "s#${OUTPUT_MAX}/##g" | while read pkg; do max_pkg=${OUTPUT_MAX}/${pkg} min_pkg=${OUTPUT_MIN}/${pkg} if [ ! -f "${min_pkg}" ] ; then echo "ERROR: ${min_pkg} doesn't exist" | tee -a ${OUTPUT_FILE} continue fi # strip version information in parenthesis max_deps=`grep "^RDEPENDS = " ${max_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'` min_deps=`grep "^RDEPENDS = " ${min_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'` if [ "$i" = 100 ] ; then echo -n "." # cheap progressbar i=0 fi if [ "${max_deps}" = "${min_deps}" ] ; then # it's annoying long, but at least it's showing some progress, warnings are grepped at the end echo "NOTE: ${pkg} dependencies weren't changed" >> ${OUTPUT_FILE} else missing_deps= for dep in ${max_deps}; do echo "${min_deps}" | grep -q " ${dep} " || missing_deps="${missing_deps} ${dep}" done if [ -n "${missing_deps}" ] ; then echo # to get rid of dots on last line echo "WARN: ${pkg} lost dependency on ${missing_deps}" | tee -a ${OUTPUT_FILE} fi fi i=`expr $i + 1` done echo # to get rid of dots on last line echo "Found differences: " grep "^WARN: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.warn echo "Found errors: " grep "^ERROR: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.error # useful for reexecuting this script with only small subset of recipes with known issues sed 's#.*[ /]packages/\([^/]*\)/\([^/]*\)/.*#\2#g' ${OUTPUT_FILE}.warn ${OUTPUT_FILE}.error | sort -u >> ${OUTPUT_BASE}/failed.recipes } for TYPE in $buildtype; do case ${TYPE} in 1) build_all;; 2) build_every_recipe;; 3) build_every_recipe;; 4) build_every_recipe;; c) compare_deps;; *) echo_error "Invalid buildtype \"$TYPE\"" esac done