diff options
Diffstat (limited to 'scripts/contrib')
| -rwxr-xr-x | scripts/contrib/bb-perf/buildstats-plot.sh | 157 | ||||
| -rwxr-xr-x | scripts/contrib/bb-perf/buildstats.sh | 155 | ||||
| -rwxr-xr-x | scripts/contrib/bbvars.py | 62 | ||||
| -rwxr-xr-x | scripts/contrib/build-perf-test-wrapper.sh | 219 | ||||
| -rwxr-xr-x | scripts/contrib/build-perf-test.sh | 51 | ||||
| -rwxr-xr-x | scripts/contrib/ddimage | 6 | ||||
| -rwxr-xr-x | scripts/contrib/devtool-stress.py | 256 | ||||
| -rwxr-xr-x | scripts/contrib/graph-tool | 15 | ||||
| -rwxr-xr-x | scripts/contrib/list-packageconfig-flags.py | 65 | ||||
| -rwxr-xr-x | scripts/contrib/mkefidisk.sh | 130 | ||||
| -rwxr-xr-x | scripts/contrib/oe-build-perf-report-email.py | 269 | ||||
| -rwxr-xr-x | scripts/contrib/python/generate-manifest-2.7.py | 81 | ||||
| -rwxr-xr-x | scripts/contrib/python/generate-manifest-3.5.py (renamed from scripts/contrib/python/generate-manifest-3.3.py) | 147 | ||||
| -rwxr-xr-x | scripts/contrib/uncovered | 39 | ||||
| -rwxr-xr-x | scripts/contrib/verify-homepage.py | 77 |
15 files changed, 1500 insertions, 229 deletions
diff --git a/scripts/contrib/bb-perf/buildstats-plot.sh b/scripts/contrib/bb-perf/buildstats-plot.sh new file mode 100755 index 0000000000..7e8ae0410e --- /dev/null +++ b/scripts/contrib/bb-perf/buildstats-plot.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2011, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# DESCRIPTION +# +# Produces script data to be consumed by gnuplot. There are two possible plots +# depending if either the -S parameter is present or not: +# +# * without -S: Produces a histogram listing top N recipes/tasks versus +# stats. The first stat defined in the -s parameter is the one taken +# into account for ranking +# * -S: Produces a histogram listing tasks versus stats. In this case, +# the value of each stat is the sum for that particular stat in all recipes found. +# Stats values are in descending order defined by the first stat defined on -s +# +# EXAMPLES +# +# 1. Top recipes' tasks taking into account utime +# +# $ buildstats-plot.sh -s utime | gnuplot -p +# +# 2. Tasks versus utime:stime +# +# $ buildstats-plot.sh -s utime:stime -S | gnuplot -p +# +# 3. Tasks versus IO write_bytes:IO read_bytes +# +# $ buildstats-plot.sh -s 'IO write_bytes:IO read_bytes' -S | gnuplot -p +# +# AUTHORS +# Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com> +# + +set -o nounset +set -o errexit + +BS_DIR="tmp/buildstats" +N=10 +STATS="utime" +SUM="" +OUTDATA_FILE="$PWD/buildstats-plot.out" + +function usage { + CMD=$(basename $0) + cat <<EOM +Usage: $CMD [-b buildstats_dir] [-t do_task] + -b buildstats The path where the folder resides + (default: "$BS_DIR") + -n N Top N recipes to display. Ignored if -S is present + (default: "$N") + -s stats The stats to be matched. If more that one stat, units + should be the same because data is plot as histogram. + (see buildstats.sh -h for all options) or any other defined + (build)stat separated by colons, i.e. stime:utime + (default: "$STATS") + -S Sum values for a particular stat for found recipes + -o Output data file. + (default: "$OUTDATA_FILE") + -h Display this help message +EOM +} + +# Parse and validate arguments +while getopts "b:n:s:o:Sh" OPT; do + case $OPT in + b) + BS_DIR="$OPTARG" + ;; + n) + N="$OPTARG" + ;; + s) + STATS="$OPTARG" + ;; + S) + SUM="y" + ;; + o) + OUTDATA_FILE="$OPTARG" + ;; + h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done + +# Get number of stats +IFS=':'; statsarray=(${STATS}); unset IFS +nstats=${#statsarray[@]} + +# Get script folder, use to run buildstats.sh +CD=$(dirname $0) + +# Parse buildstats recipes to produce a single table +OUTBUILDSTATS="$PWD/buildstats.log" +$CD/buildstats.sh -H -s "$STATS" -H > $OUTBUILDSTATS + +# Get headers +HEADERS=$(cat $OUTBUILDSTATS | sed -n -e '1s/ /-/g' -e '1s/:/ /gp') + +echo -e "set boxwidth 0.9 relative" +echo -e "set style data histograms" +echo -e "set style fill solid 1.0 border lt -1" +echo -e "set xtics rotate by 45 right" + +# Get output data +if [ -z "$SUM" ]; then + cat $OUTBUILDSTATS | sed -e '1d' | sort -k3 -n -r | head -$N > $OUTDATA_FILE + # include task at recipe column + sed -i -e "1i\ +${HEADERS}" $OUTDATA_FILE + echo -e "set title \"Top task/recipes\"" + echo -e "plot for [COL=3:`expr 3 + ${nstats} - 1`] '${OUTDATA_FILE}' using COL:xtic(stringcolumn(1).' '.stringcolumn(2)) title columnheader(COL)" +else + + # Construct datatamash sum argument (sum 3 sum 4 ...) + declare -a sumargs + j=0 + for i in `seq $nstats`; do + sumargs[j]=sum; j=$(( $j + 1 )) + sumargs[j]=`expr 3 + $i - 1`; j=$(( $j + 1 )) + done + + # Do the processing with datamash + cat $OUTBUILDSTATS | sed -e '1d' | datamash -t ' ' -g1 ${sumargs[*]} | sort -k2 -n -r > $OUTDATA_FILE + + # Include headers into resulted file, so we can include gnuplot xtics + HEADERS=$(echo $HEADERS | sed -e 's/recipe//1') + sed -i -e "1i\ +${HEADERS}" $OUTDATA_FILE + + # Plot + echo -e "set title \"Sum stats values per task for all recipes\"" + echo -e "plot for [COL=2:`expr 2 + ${nstats} - 1`] '${OUTDATA_FILE}' using COL:xtic(1) title columnheader(COL)" +fi + diff --git a/scripts/contrib/bb-perf/buildstats.sh b/scripts/contrib/bb-perf/buildstats.sh new file mode 100755 index 0000000000..8d7e2488f0 --- /dev/null +++ b/scripts/contrib/bb-perf/buildstats.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# +# Copyright (c) 2011, Intel Corporation. +# All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# DESCRIPTION +# Given 'buildstats' data (generate by bitbake when setting +# USER_CLASSES ?= "buildstats" on local.conf), task names and a stats values +# (these are the ones preset on the buildstats files), outputs +# '<task> <recipe> <value_1> <value_2> ... <value_n>'. The units are the ones +# defined at buildstats, which in turn takes data from /proc/[pid] files +# +# Some useful pipelines +# +# 1. Tasks with largest stime (Amount of time that this process has been scheduled +# in kernel mode) values +# $ buildstats.sh -b <buildstats> -s stime | sort -k3 -n -r | head +# +# 2. Min, max, sum utime (Amount of time that this process has been scheduled +# in user mode) per task (in needs GNU datamash) +# $ buildstats.sh -b <buildstats> -s utime | datamash -t' ' -g1 min 3 max 3 sum 3 | sort -k4 -n -r +# +# AUTHORS +# Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com> +# + +# Stats, by type +TIME="utime:stime:cutime:cstime" +IO="IO wchar:IO write_bytes:IO syscr:IO read_bytes:IO rchar:IO syscw:IO cancelled_write_bytes" +RUSAGE="rusage ru_utime:rusage ru_stime:rusage ru_maxrss:rusage ru_minflt:rusage ru_majflt:\ +rusage ru_inblock:rusage ru_oublock:rusage ru_nvcsw:rusage ru_nivcsw" + +CHILD_RUSAGE="Child rusage ru_utime:Child rusage ru_stime:Child rusage ru_maxrss:Child rusage ru_minflt:\ +Child rusage ru_majflt:Child rusage ru_inblock:Child rusage ru_oublock:Child rusage ru_nvcsw:\ +Child rusage ru_nivcsw" + +BS_DIR="tmp/buildstats" +TASKS="compile:configure:fetch:install:patch:populate_lic:populate_sysroot:unpack" +STATS="$TIME" +HEADER="" # No header by default + +function usage { +CMD=$(basename $0) +cat <<EOM +Usage: $CMD [-b buildstats_dir] [-t do_task] + -b buildstats The path where the folder resides + (default: "$BS_DIR") + -t tasks The tasks to be computed + (default: "$TASKS") + -s stats The stats to be matched. Options: TIME, IO, RUSAGE, CHILD_RUSAGE + or any other defined buildstat separated by colons, i.e. stime:utime + (default: "$STATS") + Default stat sets: + TIME=$TIME + IO=$IO + RUSAGE=$RUSAGE + CHILD_RUSAGE=$CHILD_RUSAGE + -h Display this help message +EOM +} + +# Parse and validate arguments +while getopts "b:t:s:Hh" OPT; do + case $OPT in + b) + BS_DIR="$OPTARG" + ;; + t) + TASKS="$OPTARG" + ;; + s) + STATS="$OPTARG" + ;; + H) + HEADER="y" + ;; + h) + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done + +# Ensure the buildstats folder exists +if [ ! -d "$BS_DIR" ]; then + echo "ERROR: $BS_DIR does not exist" + usage + exit 1 +fi + +stats="" +IFS=":" +for stat in ${STATS}; do + case $stat in + TIME) + stats="${stats}:${TIME}" + ;; + IO) + stats="${stats}:${IO}" + ;; + RUSAGE) + stats="${stats}:${RUSAGE}" + ;; + CHILD_RUSAGE) + stats="${stats}:${CHILD_RUSAGE}" + ;; + *) + stats="${STATS}" + esac +done + +# remove possible colon at the beginning +stats="$(echo "$stats" | sed -e 's/^://1')" + +# Provide a header if required by the user +[ -n "$HEADER" ] && { echo "task:recipe:$stats"; } + +for task in ${TASKS}; do + task="do_${task}" + for file in $(find ${BS_DIR} -type f -name ${task} | awk 'BEGIN{ ORS=""; OFS=":" } { print $0,"" }'); do + recipe="$(basename $(dirname $file))" + times="" + for stat in ${stats}; do + [ -z "$stat" ] && { echo "empty stats"; } + time=$(sed -n -e "s/^\($stat\): \\(.*\\)/\\2/p" $file) + # in case the stat is not present, set the value as NA + [ -z "$time" ] && { time="NA"; } + # Append it to times + if [ -z "$times" ]; then + times="${time}" + else + times="${times} ${time}" + fi + done + echo "${task} ${recipe} ${times}" + done +done diff --git a/scripts/contrib/bbvars.py b/scripts/contrib/bbvars.py index 0896d64445..d8d0594776 100755 --- a/scripts/contrib/bbvars.py +++ b/scripts/contrib/bbvars.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,12 +24,12 @@ import os.path import re def usage(): - print 'Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0]) - print ' -d FILENAME documentation file to search' - print ' -h, --help display this help and exit' - print ' -m METADIR meta directory to search for recipes' - print ' -t FILENAME documentation config file (for doc tags)' - print ' -T Only display variables with doc tags (requires -t)' + print('Usage: %s -d FILENAME [-d FILENAME]* -m METADIR [-m MATADIR]*' % os.path.basename(sys.argv[0])) + print(' -d FILENAME documentation file to search') + print(' -h, --help display this help and exit') + print(' -m METADIR meta directory to search for recipes') + print(' -t FILENAME documentation config file (for doc tags)') + print(' -T Only display variables with doc tags (requires -t)') def recipe_bbvars(recipe): ''' Return a unique set of every bbvar encountered in the recipe ''' @@ -37,9 +37,9 @@ def recipe_bbvars(recipe): vset = set() try: r = open(recipe) - except IOError as (errno, strerror): - print 'WARNING: Failed to open recipe ', recipe - print strerror + except IOError as err: + print('WARNING: Failed to open recipe ', recipe) + print(err.args[1]) for line in r: # Strip any comments from the line @@ -59,8 +59,8 @@ def collect_bbvars(metadir): for root,dirs,files in os.walk(metadir): for name in files: if name.find(".bb") >= 0: - for key in recipe_bbvars(os.path.join(root,name)).iterkeys(): - if bbvars.has_key(key): + for key in recipe_bbvars(os.path.join(root,name)).keys(): + if key in bbvars: bbvars[key] = bbvars[key] + 1 else: bbvars[key] = 1 @@ -71,9 +71,9 @@ def bbvar_is_documented(var, docfiles): for doc in docfiles: try: f = open(doc) - except IOError as (errno, strerror): - print 'WARNING: Failed to open doc ', doc - print strerror + except IOError as err: + print('WARNING: Failed to open doc ', doc) + print(err.args[1]) for line in f: if prog.match(line): return True @@ -87,8 +87,8 @@ def bbvar_doctag(var, docconf): try: f = open(docconf) - except IOError as (errno, strerror): - return strerror + except IOError as err: + return err.args[1] for line in f: m = prog.search(line) @@ -109,8 +109,8 @@ def main(): # Collect and validate input try: opts, args = getopt.getopt(sys.argv[1:], "d:hm:t:T", ["help"]) - except getopt.GetoptError, err: - print '%s' % str(err) + except getopt.GetoptError as err: + print('%s' % str(err)) usage() sys.exit(2) @@ -122,13 +122,13 @@ def main(): if os.path.isfile(a): docfiles.append(a) else: - print 'ERROR: documentation file %s is not a regular file' % (a) + print('ERROR: documentation file %s is not a regular file' % a) sys.exit(3) elif o == '-m': if os.path.isdir(a): metadirs.append(a) else: - print 'ERROR: meta directory %s is not a directory' % (a) + print('ERROR: meta directory %s is not a directory' % a) sys.exit(4) elif o == "-t": if os.path.isfile(a): @@ -139,31 +139,31 @@ def main(): assert False, "unhandled option" if len(docfiles) == 0: - print 'ERROR: no docfile specified' + print('ERROR: no docfile specified') usage() sys.exit(5) if len(metadirs) == 0: - print 'ERROR: no metadir specified' + print('ERROR: no metadir specified') usage() sys.exit(6) if onlydoctags and docconf == "": - print 'ERROR: no docconf specified' + print('ERROR: no docconf specified') usage() sys.exit(7) # Collect all the variable names from the recipes in the metadirs for m in metadirs: - for key,cnt in collect_bbvars(m).iteritems(): - if bbvars.has_key(key): + for key,cnt in collect_bbvars(m).items(): + if key in bbvars: bbvars[key] = bbvars[key] + cnt else: bbvars[key] = cnt # Check each var for documentation varlen = 0 - for v in bbvars.iterkeys(): + for v in bbvars.keys(): if len(v) > varlen: varlen = len(v) if not bbvar_is_documented(v, docfiles): @@ -172,14 +172,14 @@ def main(): varlen = varlen + 1 # Report all undocumented variables - print 'Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars)) + print('Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars))) header = '%s%s%s' % (str("VARIABLE").ljust(varlen), str("COUNT").ljust(6), str("DOCTAG").ljust(7)) - print header - print str("").ljust(len(header), '=') + print(header) + print(str("").ljust(len(header), '=')) for v in undocumented: doctag = bbvar_doctag(v, docconf) if not onlydoctags or not doctag == "": - print '%s%s%s' % (v.ljust(varlen), str(bbvars[v]).ljust(6), doctag) + print('%s%s%s' % (v.ljust(varlen), str(bbvars[v]).ljust(6), doctag)) if __name__ == "__main__": diff --git a/scripts/contrib/build-perf-test-wrapper.sh b/scripts/contrib/build-perf-test-wrapper.sh new file mode 100755 index 0000000000..3da32532be --- /dev/null +++ b/scripts/contrib/build-perf-test-wrapper.sh @@ -0,0 +1,219 @@ +#!/bin/bash +# +# Build performance test script wrapper +# +# Copyright (c) 2016, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# +# This script is a simple wrapper around the actual build performance tester +# script. This script initializes the build environment, runs +# oe-build-perf-test and archives the results. + +script=`basename $0` +script_dir=$(realpath $(dirname $0)) +archive_dir=~/perf-results/archives + +usage () { +cat << EOF +Usage: $script [-h] [-c COMMITISH] [-C GIT_REPO] + +Optional arguments: + -h show this help and exit. + -a ARCHIVE_DIR archive results tarball here, give an empty string to + disable tarball archiving (default: $archive_dir) + -c COMMITISH test (checkout) this commit, <branch>:<commit> can be + specified to test specific commit of certain branch + -C GIT_REPO commit results into Git + -E EMAIL_ADDR send email report + -P GIT_REMOTE push results to a remote Git repository + -w WORK_DIR work dir for this script + (default: GIT_TOP_DIR/build-perf-test) + -x create xml report (instead of json) +EOF +} + +get_os_release_var () { + ( source /etc/os-release; eval echo '$'$1 ) +} + + +# Parse command line arguments +commitish="" +oe_build_perf_test_extra_opts=() +oe_git_archive_extra_opts=() +while getopts "ha:c:C:E:P:w:x" opt; do + case $opt in + h) usage + exit 0 + ;; + a) archive_dir=`realpath -s "$OPTARG"` + ;; + c) commitish=$OPTARG + ;; + C) results_repo=`realpath -s "$OPTARG"` + ;; + E) email_to="$OPTARG" + ;; + P) oe_git_archive_extra_opts+=("--push" "$OPTARG") + ;; + w) base_dir=`realpath -s "$OPTARG"` + ;; + x) oe_build_perf_test_extra_opts+=("--xml") + ;; + *) usage + exit 1 + ;; + esac +done + +# Check positional args +shift "$((OPTIND - 1))" +if [ $# -ne 0 ]; then + echo "ERROR: No positional args are accepted." + usage + exit 1 +fi + +# Open a file descriptor for flock and acquire lock +LOCK_FILE="/tmp/oe-build-perf-test-wrapper.lock" +if ! exec 3> "$LOCK_FILE"; then + echo "ERROR: Unable to open lock file" + exit 1 +fi +if ! flock -n 3; then + echo "ERROR: Another instance of this script is running" + exit 1 +fi + +echo "Running on `uname -n`" +if ! git_topdir=$(git rev-parse --show-toplevel); then + echo "The current working dir doesn't seem to be a git clone. Please cd there before running `basename $0`" + exit 1 +fi + +cd "$git_topdir" + +if [ -n "$commitish" ]; then + echo "Running git fetch" + git fetch &> /dev/null + git checkout HEAD^0 &> /dev/null + + # Handle <branch>:<commit> format + if echo "$commitish" | grep -q ":"; then + commit=`echo "$commitish" | cut -d":" -f2` + branch=`echo "$commitish" | cut -d":" -f1` + else + commit="$commitish" + branch="$commitish" + fi + + echo "Checking out $commitish" + git branch -D $branch &> /dev/null + if ! git checkout -f $branch &> /dev/null; then + echo "ERROR: Git checkout failed" + exit 1 + fi + + # Check that the specified branch really contains the commit + commit_hash=`git rev-parse --revs-only $commit --` + if [ -z "$commit_hash" -o "`git merge-base $branch $commit`" != "$commit_hash" ]; then + echo "ERROR: branch $branch does not contain commit $commit" + exit 1 + fi + git reset --hard $commit > /dev/null +fi + +# Setup build environment +if [ -z "$base_dir" ]; then + base_dir="$git_topdir/build-perf-test" +fi +echo "Using working dir $base_dir" + +timestamp=`date "+%Y%m%d%H%M%S"` +git_rev=$(git rev-parse --short HEAD) || exit 1 +build_dir="$base_dir/build-$git_rev-$timestamp" +results_dir="$base_dir/results-$git_rev-$timestamp" +globalres_log="$base_dir/globalres.log" +machine="qemux86" + +mkdir -p "$base_dir" +source ./oe-init-build-env $build_dir >/dev/null || exit 1 + +# Additional config +auto_conf="$build_dir/conf/auto.conf" +echo "MACHINE = \"$machine\"" > "$auto_conf" +echo 'BB_NUMBER_THREADS = "8"' >> "$auto_conf" +echo 'PARALLEL_MAKE = "-j 8"' >> "$auto_conf" +echo "DL_DIR = \"$base_dir/downloads\"" >> "$auto_conf" +# Disabling network sanity check slightly reduces the variance of timing results +echo 'CONNECTIVITY_CHECK_URIS = ""' >> "$auto_conf" +# Possibility to define extra settings +if [ -f "$base_dir/auto.conf.extra" ]; then + cat "$base_dir/auto.conf.extra" >> "$auto_conf" +fi + +# Run actual test script +oe-build-perf-test --out-dir "$results_dir" \ + --globalres-file "$globalres_log" \ + "${oe_build_perf_test_extra_opts[@]}" \ + --lock-file "$base_dir/oe-build-perf.lock" + +case $? in + 1) echo "ERROR: oe-build-perf-test script failed!" + exit 1 + ;; + 2) echo "NOTE: some tests failed!" + ;; +esac + +# Commit results to git +if [ -n "$results_repo" ]; then + echo -e "\nArchiving results in $results_repo" + oe-git-archive \ + --git-dir "$results_repo" \ + --branch-name "{hostname}/{branch}/{machine}" \ + --tag-name "{hostname}/{branch}/{machine}/{commit_count}-g{commit}/{tag_number}" \ + --exclude "buildstats.json" \ + --notes "buildstats/{branch_name}" "$results_dir/buildstats.json" \ + "${oe_git_archive_extra_opts[@]}" \ + "$results_dir" + + # Send email report + if [ -n "$email_to" ]; then + echo -e "\nEmailing test report" + os_name=`get_os_release_var PRETTY_NAME` + oe-build-perf-report -r "$results_repo" > report.txt + oe-build-perf-report -r "$results_repo" --html > report.html + "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text report.txt --html report.html "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}" + fi +fi + + +echo -ne "\n\n-----------------\n" +echo "Global results file:" +echo -ne "\n" + +cat "$globalres_log" + +if [ -n "$archive_dir" ]; then + echo -ne "\n\n-----------------\n" + echo "Archiving results in $archive_dir" + mkdir -p "$archive_dir" + results_basename=`basename "$results_dir"` + results_dirname=`dirname "$results_dir"` + tar -czf "$archive_dir/`uname -n`-${results_basename}.tar.gz" -C "$results_dirname" "$results_basename" +fi + +rm -rf "$build_dir" +rm -rf "$results_dir" + +echo "DONE" diff --git a/scripts/contrib/build-perf-test.sh b/scripts/contrib/build-perf-test.sh index be3b648046..7d99228c73 100755 --- a/scripts/contrib/build-perf-test.sh +++ b/scripts/contrib/build-perf-test.sh @@ -128,7 +128,7 @@ rev=$(git rev-parse --short HEAD) || exit 1 OUTDIR="$clonedir/build-perf-test/results-$rev-`date "+%Y%m%d%H%M%S"`" BUILDDIR="$OUTDIR/build" resultsfile="$OUTDIR/results.log" -bboutput="$OUTDIR/bitbake.log" +cmdoutput="$OUTDIR/commands.log" myoutput="$OUTDIR/output.log" globalres="$clonedir/build-perf-test/globalres.log" @@ -180,14 +180,13 @@ time_count=0 declare -a SIZES size_count=0 -bbtime () { - local arg="$@" - log " Timing: bitbake ${arg}" +time_cmd () { + log " Timing: $*" if [ $verbose -eq 0 ]; then - /usr/bin/time -v -o $resultsfile bitbake ${arg} >> $bboutput + /usr/bin/time -v -o $resultsfile "$@" >> $cmdoutput else - /usr/bin/time -v -o $resultsfile bitbake ${arg} + /usr/bin/time -v -o $resultsfile "$@" fi ret=$? if [ $ret -eq 0 ]; then @@ -206,12 +205,16 @@ bbtime () { log "More stats can be found in ${resultsfile}.${i}" } +bbtime () { + time_cmd bitbake "$@" +} + #we don't time bitbake here bbnotime () { local arg="$@" log " Running: bitbake ${arg}" if [ $verbose -eq 0 ]; then - bitbake ${arg} >> $bboutput + bitbake ${arg} >> $cmdoutput else bitbake ${arg} fi @@ -335,7 +338,7 @@ test2 () { # # Start with # i) "rm -rf tmp/cache; time bitbake -p" -# ii) "rm -rf tmp/cache/default-eglibc/; time bitbake -p" +# ii) "rm -rf tmp/cache/default-glibc/; time bitbake -p" # iii) "time bitbake -p" @@ -344,12 +347,39 @@ test3 () { log " Removing tmp/cache && cache" rm -rf tmp/cache cache bbtime -p - log " Removing tmp/cache/default-eglibc/" - rm -rf tmp/cache/default-eglibc/ + log " Removing tmp/cache/default-glibc/" + rm -rf tmp/cache/default-glibc/ bbtime -p bbtime -p } +# +# Test 4 - eSDK +# Measure: eSDK size and installation time +test4 () { + log "Running Test 4: eSDK size and installation time" + bbnotime $IMAGE -c do_populate_sdk_ext + + esdk_installer=(tmp/deploy/sdk/*-toolchain-ext-*.sh) + + if [ ${#esdk_installer[*]} -eq 1 ]; then + s=$((`stat -c %s "$esdk_installer"` / 1024)) + SIZES[(( size_count++ ))]="$s" + log "Download SIZE of eSDK is: $s kB" + + do_sync + time_cmd "$esdk_installer" -y -d "tmp/esdk-deploy" + + s=$((`du -sb "tmp/esdk-deploy" | cut -f1` / 1024)) + SIZES[(( size_count++ ))]="$s" + log "Install SIZE of eSDK is: $s kB" + else + log "ERROR: other than one sdk found (${esdk_installer[*]}), reporting size and time as 0." + SIZES[(( size_count++ ))]="0" + TIMES[(( time_count++ ))]="0" + fi + +} # RUN! @@ -359,6 +389,7 @@ test1_p2 test1_p3 test2 test3 +test4 # if we got til here write to global results write_results diff --git a/scripts/contrib/ddimage b/scripts/contrib/ddimage index a503f11d0d..ab929957a5 100755 --- a/scripts/contrib/ddimage +++ b/scripts/contrib/ddimage @@ -100,5 +100,9 @@ if [ "$RESPONSE" != "y" ]; then fi echo "Writing image..." -dd if="$IMAGE" of="$DEVICE" bs="$BLOCKSIZE" +if which pv >/dev/null 2>&1; then + pv "$IMAGE" | dd of="$DEVICE" bs="$BLOCKSIZE" +else + dd if="$IMAGE" of="$DEVICE" bs="$BLOCKSIZE" +fi sync diff --git a/scripts/contrib/devtool-stress.py b/scripts/contrib/devtool-stress.py new file mode 100755 index 0000000000..d555c51a65 --- /dev/null +++ b/scripts/contrib/devtool-stress.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 + +# devtool stress tester +# +# Written by: Paul Eggleton <paul.eggleton@linux.intel.com> +# +# Copyright 2015 Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +import sys +import os +import os.path +import subprocess +import re +import argparse +import logging +import tempfile +import shutil +import signal +import fnmatch + +scripts_lib_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'lib')) +sys.path.insert(0, scripts_lib_path) +import scriptutils +import argparse_oe +logger = scriptutils.logger_create('devtool-stress') + +def select_recipes(args): + import bb.tinfoil + tinfoil = bb.tinfoil.Tinfoil() + tinfoil.prepare(False) + + pkg_pn = tinfoil.cooker.recipecaches[''].pkg_pn + (latest_versions, preferred_versions) = bb.providers.findProviders(tinfoil.config_data, tinfoil.cooker.recipecaches[''], pkg_pn) + + skip_classes = args.skip_classes.split(',') + + recipelist = [] + for pn in sorted(pkg_pn): + pref = preferred_versions[pn] + inherits = [os.path.splitext(os.path.basename(f))[0] for f in tinfoil.cooker.recipecaches[''].inherits[pref[1]]] + for cls in skip_classes: + if cls in inherits: + break + else: + recipelist.append(pn) + + tinfoil.shutdown() + + resume_from = args.resume_from + if resume_from: + if not resume_from in recipelist: + print('%s is not a testable recipe' % resume_from) + return 1 + if args.only: + only = args.only.split(',') + for onlyitem in only: + for pn in recipelist: + if fnmatch.fnmatch(pn, onlyitem): + break + else: + print('%s does not match any testable recipe' % onlyitem) + return 1 + else: + only = None + if args.skip: + skip = args.skip.split(',') + else: + skip = [] + + recipes = [] + for pn in recipelist: + if resume_from: + if pn == resume_from: + resume_from = None + else: + continue + + if args.only: + for item in only: + if fnmatch.fnmatch(pn, item): + break + else: + continue + + skipit = False + for item in skip: + if fnmatch.fnmatch(pn, item): + skipit = True + if skipit: + continue + + recipes.append(pn) + + return recipes + + +def stress_extract(args): + import bb.process + + recipes = select_recipes(args) + + failures = 0 + tmpdir = tempfile.mkdtemp() + os.setpgrp() + try: + for pn in recipes: + sys.stdout.write('Testing %s ' % (pn + ' ').ljust(40, '.')) + sys.stdout.flush() + failed = False + skipped = None + + srctree = os.path.join(tmpdir, pn) + try: + bb.process.run('devtool extract %s %s' % (pn, srctree)) + except bb.process.ExecutionError as exc: + if exc.exitcode == 4: + skipped = 'incompatible' + else: + failed = True + with open('stress_%s_extract.log' % pn, 'w') as f: + f.write(str(exc)) + + if os.path.exists(srctree): + shutil.rmtree(srctree) + + if failed: + print('failed') + failures += 1 + elif skipped: + print('skipped (%s)' % skipped) + else: + print('ok') + except KeyboardInterrupt: + # We want any child processes killed. This is crude, but effective. + os.killpg(0, signal.SIGTERM) + + if failures: + return 1 + else: + return 0 + + +def stress_modify(args): + import bb.process + + recipes = select_recipes(args) + + failures = 0 + tmpdir = tempfile.mkdtemp() + os.setpgrp() + try: + for pn in recipes: + sys.stdout.write('Testing %s ' % (pn + ' ').ljust(40, '.')) + sys.stdout.flush() + failed = False + reset = True + skipped = None + + srctree = os.path.join(tmpdir, pn) + try: + bb.process.run('devtool modify -x %s %s' % (pn, srctree)) + except bb.process.ExecutionError as exc: + if exc.exitcode == 4: + skipped = 'incompatible' + else: + with open('stress_%s_modify.log' % pn, 'w') as f: + f.write(str(exc)) + failed = 'modify' + reset = False + + if not skipped: + if not failed: + try: + bb.process.run('bitbake -c install %s' % pn) + except bb.process.CmdError as exc: + with open('stress_%s_install.log' % pn, 'w') as f: + f.write(str(exc)) + failed = 'build' + if reset: + try: + bb.process.run('devtool reset %s' % pn) + except bb.process.CmdError as exc: + print('devtool reset failed: %s' % str(exc)) + break + + if os.path.exists(srctree): + shutil.rmtree(srctree) + + if failed: + print('failed (%s)' % failed) + failures += 1 + elif skipped: + print('skipped (%s)' % skipped) + else: + print('ok') + except KeyboardInterrupt: + # We want any child processes killed. This is crude, but effective. + os.killpg(0, signal.SIGTERM) + + if failures: + return 1 + else: + return 0 + + +def main(): + parser = argparse_oe.ArgumentParser(description="devtool stress tester", + epilog="Use %(prog)s <subcommand> --help to get help on a specific command") + parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') + parser.add_argument('-r', '--resume-from', help='Resume from specified recipe', metavar='PN') + parser.add_argument('-o', '--only', help='Only test specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST') + parser.add_argument('-s', '--skip', help='Skip specified recipes (comma-separated without spaces, wildcards allowed)', metavar='PNLIST', default='gcc-source-*,kernel-devsrc,package-index,perf,meta-world-pkgdata,glibc-locale,glibc-mtrace,glibc-scripts,os-release') + parser.add_argument('-c', '--skip-classes', help='Skip recipes inheriting specified classes (comma-separated) - default %(default)s', metavar='CLASSLIST', default='native,nativesdk,cross,cross-canadian,image,populate_sdk,meta,packagegroup') + subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>') + subparsers.required = True + + parser_modify = subparsers.add_parser('modify', + help='Run "devtool modify" followed by a build with bitbake on matching recipes', + description='Runs "devtool modify" followed by a build with bitbake on matching recipes') + parser_modify.set_defaults(func=stress_modify) + + parser_extract = subparsers.add_parser('extract', + help='Run "devtool extract" on matching recipes', + description='Runs "devtool extract" on matching recipes') + parser_extract.set_defaults(func=stress_extract) + + args = parser.parse_args() + + if args.debug: + logger.setLevel(logging.DEBUG) + + import scriptpath + bitbakepath = scriptpath.add_bitbake_lib_path() + if not bitbakepath: + logger.error("Unable to find bitbake by searching parent directory of this script or PATH") + return 1 + logger.debug('Found bitbake path: %s' % bitbakepath) + + ret = args.func(args) + +if __name__ == "__main__": + main() diff --git a/scripts/contrib/graph-tool b/scripts/contrib/graph-tool index 6dc7d337f8..1df5b8c345 100755 --- a/scripts/contrib/graph-tool +++ b/scripts/contrib/graph-tool @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Simple graph query utility # useful for getting answers from .dot files produced by bitbake -g @@ -30,8 +30,7 @@ def get_path_networkx(dotfile, fromnode, tonode): print('ERROR: Please install the networkx python module') sys.exit(1) - graph = networkx.DiGraph(networkx.read_dot(dotfile)) - + graph = networkx.DiGraph(networkx.nx_pydot.read_dot(dotfile)) def node_missing(node): import difflib close_matches = difflib.get_close_matches(node, graph.nodes(), cutoff=0.7) @@ -53,11 +52,11 @@ def find_paths(args, usage): fromnode = args[1] tonode = args[2] - paths = list(get_path_networkx(args[0], fromnode, tonode)) - if paths: - for path in paths: - print ' -> '.join(path) - else: + + path = None + for path in get_path_networkx(args[0], fromnode, tonode): + print(" -> ".join(map(str, path))) + if not path: print("ERROR: no path from %s to %s in graph" % (fromnode, tonode)) sys.exit(1) diff --git a/scripts/contrib/list-packageconfig-flags.py b/scripts/contrib/list-packageconfig-flags.py index 598b5c3fc6..7ce718624a 100755 --- a/scripts/contrib/list-packageconfig-flags.py +++ b/scripts/contrib/list-packageconfig-flags.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,7 +37,6 @@ if not bitbakepath: sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") sys.exit(1) -import bb.cache import bb.cooker import bb.providers import bb.tinfoil @@ -45,7 +44,7 @@ import bb.tinfoil def get_fnlist(bbhandler, pkg_pn, preferred): ''' Get all recipe file names ''' if preferred: - (latest_versions, preferred_versions) = bb.providers.findProviders(bbhandler.config_data, bbhandler.cooker.recipecache, pkg_pn) + (latest_versions, preferred_versions) = bb.providers.findProviders(bbhandler.config_data, bbhandler.cooker.recipecaches[''], pkg_pn) fn_list = [] for pn in sorted(pkg_pn): @@ -58,14 +57,13 @@ def get_fnlist(bbhandler, pkg_pn, preferred): def get_recipesdata(bbhandler, preferred): ''' Get data of all available recipes which have PACKAGECONFIG flags ''' - pkg_pn = bbhandler.cooker.recipecache.pkg_pn + pkg_pn = bbhandler.cooker.recipecaches[''].pkg_pn data_dict = {} for fn in get_fnlist(bbhandler, pkg_pn, preferred): - data = bb.cache.Cache.loadDataFull(fn, bbhandler.cooker.collection.get_file_appends(fn), bbhandler.config_data) + data = bbhandler.parse_recipe_file(fn) flags = data.getVarFlags("PACKAGECONFIG") flags.pop('doc', None) - flags.pop('defaultval', None) if flags: data_dict[fn] = data @@ -78,8 +76,7 @@ def collect_pkgs(data_dict): for fn in data_dict: pkgconfigflags = data_dict[fn].getVarFlags("PACKAGECONFIG") pkgconfigflags.pop('doc', None) - pkgconfigflags.pop('defaultval', None) - pkgname = data_dict[fn].getVar("P", True) + pkgname = data_dict[fn].getVar("P") pkg_dict[pkgname] = sorted(pkgconfigflags.keys()) return pkg_dict @@ -88,7 +85,7 @@ def collect_flags(pkg_dict): ''' Collect available PACKAGECONFIG flags and all affected pkgs ''' # flag_dict = {'flag': ['pkg1', 'pkg2',...]} flag_dict = {} - for pkgname, flaglist in pkg_dict.iteritems(): + for pkgname, flaglist in pkg_dict.items(): for flag in flaglist: if flag in flag_dict: flag_dict[flag].append(pkgname) @@ -106,8 +103,8 @@ def display_pkgs(pkg_dict): pkgname_len += 1 header = '%-*s%s' % (pkgname_len, str("RECIPE NAME"), str("PACKAGECONFIG FLAGS")) - print header - print str("").ljust(len(header), '=') + print(header) + print(str("").ljust(len(header), '=')) for pkgname in sorted(pkg_dict): print('%-*s%s' % (pkgname_len, pkgname, ' '.join(pkg_dict[pkgname]))) @@ -117,28 +114,28 @@ def display_flags(flag_dict): flag_len = len("PACKAGECONFIG FLAG") + 5 header = '%-*s%s' % (flag_len, str("PACKAGECONFIG FLAG"), str("RECIPE NAMES")) - print header - print str("").ljust(len(header), '=') + print(header) + print(str("").ljust(len(header), '=')) for flag in sorted(flag_dict): print('%-*s%s' % (flag_len, flag, ' '.join(sorted(flag_dict[flag])))) def display_all(data_dict): ''' Display all pkgs and PACKAGECONFIG information ''' - print str("").ljust(50, '=') + print(str("").ljust(50, '=')) for fn in data_dict: - print('%s' % data_dict[fn].getVar("P", True)) - print fn - packageconfig = data_dict[fn].getVar("PACKAGECONFIG", True) or '' + print('%s' % data_dict[fn].getVar("P")) + print(fn) + packageconfig = data_dict[fn].getVar("PACKAGECONFIG") or '' if packageconfig.strip() == '': packageconfig = 'None' print('PACKAGECONFIG %s' % packageconfig) - for flag,flag_val in data_dict[fn].getVarFlags("PACKAGECONFIG").iteritems(): - if flag in ["defaultval", "doc"]: + for flag,flag_val in data_dict[fn].getVarFlags("PACKAGECONFIG").items(): + if flag == "doc": continue print('PACKAGECONFIG[%s] %s' % (flag, flag_val)) - print '' + print('') def main(): pkg_dict = {} @@ -162,20 +159,20 @@ def main(): options, args = parser.parse_args(sys.argv) - bbhandler = bb.tinfoil.Tinfoil() - bbhandler.prepare() - print("Gathering recipe data...") - data_dict = get_recipesdata(bbhandler, options.preferred) - - if options.listtype == 'flags': - pkg_dict = collect_pkgs(data_dict) - flag_dict = collect_flags(pkg_dict) - display_flags(flag_dict) - elif options.listtype == 'recipes': - pkg_dict = collect_pkgs(data_dict) - display_pkgs(pkg_dict) - elif options.listtype == 'all': - display_all(data_dict) + with bb.tinfoil.Tinfoil() as bbhandler: + bbhandler.prepare() + print("Gathering recipe data...") + data_dict = get_recipesdata(bbhandler, options.preferred) + + if options.listtype == 'flags': + pkg_dict = collect_pkgs(data_dict) + flag_dict = collect_flags(pkg_dict) + display_flags(flag_dict) + elif options.listtype == 'recipes': + pkg_dict = collect_pkgs(data_dict) + display_pkgs(pkg_dict) + elif options.listtype == 'all': + display_all(data_dict) if __name__ == "__main__": main() diff --git a/scripts/contrib/mkefidisk.sh b/scripts/contrib/mkefidisk.sh index b96b7d4f7d..800733f0af 100755 --- a/scripts/contrib/mkefidisk.sh +++ b/scripts/contrib/mkefidisk.sh @@ -20,6 +20,11 @@ LANG=C +echo +echo "WARNING: This script is deprecated and will be removed soon." +echo "Please consider using wic EFI images instead." +echo + # Set to 1 to enable additional output DEBUG=0 OUT="/dev/null" @@ -136,6 +141,9 @@ unmount_device() { } unmount() { + if [ "$1" = "" ] ; then + return 0 + fi grep -q $1 /proc/mounts if [ $? -eq 0 ]; then debug "Unmounting $1" @@ -149,8 +157,55 @@ unmount() { # Parse and validate arguments # if [ $# -lt 3 ] || [ $# -gt 4 ]; then - usage - exit 1 + if [ $# -eq 1 ]; then + AVAILABLE_DISK=`lsblk | grep "disk" | cut -f 1 -d " "` + X=0 + for disk in `echo $AVAILABLE_DISK`; do + mounted=`lsblk /dev/$disk | awk {'print $7'} | sed "s/MOUNTPOINT//"` + if [ -z "$mounted" ]; then + UNMOUNTED_AVAILABLES="$UNMOUNTED_AVAILABLES /dev/$disk" + info "$X - /dev/$disk" + X=`expr $X + 1` + fi + done + if [ $X -eq 0 ]; then + die "No unmounted device found." + fi + read -p "Choose unmounted device number: " DISK_NUMBER + X=0 + for line in `echo $UNMOUNTED_AVAILABLES`; do + if [ $DISK_NUMBER -eq $X ]; then + DISK_TO_BE_FLASHED=$line + break + else + X=`expr $X + 1` + fi + done + if [ -z "$DISK_TO_BE_FLASHED" ]; then + die "Option \"$DISK_NUMBER\" is invalid. Choose a valid option" + else + if [ -z `echo $DISK_TO_BE_FLASHED | grep "mmc"` ]; then + TARGET_TO_BE_BOOT="/dev/sda" + else + TARGET_TO_BE_BOOT="/dev/mmcblk0" + fi + fi + echo "" + echo "Choose a name of the device that will be boot from" + echo -n "Recommended name is: " + info "$TARGET_TO_BE_BOOT" + read -p "Is target device okay? [y/N]: " RESPONSE + if [ "$RESPONSE" != "y" ]; then + read -p "Choose target device name: " TARGET_TO_BE_BOOT + fi + echo "" + if [ -z "$TARGET_TO_BE_BOOT" ]; then + die "Error: choose a valid target name" + fi + else + usage + exit 1 + fi fi if [ "$1" = "-v" ]; then @@ -159,9 +214,15 @@ if [ "$1" = "-v" ]; then shift fi -DEVICE=$1 -HDDIMG=$2 -TARGET_DEVICE=$3 +if [ -z "$AVAILABLE_DISK" ]; then + DEVICE=$1 + HDDIMG=$2 + TARGET_DEVICE=$3 +else + DEVICE=$DISK_TO_BE_FLASHED + HDDIMG=$1 + TARGET_DEVICE=$TARGET_TO_BE_BOOT +fi LINK=$(readlink $DEVICE) if [ $? -eq 0 ]; then @@ -170,7 +231,11 @@ fi if [ ! -w "$DEVICE" ]; then usage - die "Device $DEVICE does not exist or is not writable" + if [ ! -e "${DEVICE}" ] ; then + die "Device $DEVICE cannot be found" + else + die "Device $DEVICE is not writable (need to run under sudo?)" + fi fi if [ ! -e "$HDDIMG" ]; then @@ -218,11 +283,11 @@ mkdir $BOOTFS_MNT || die "Failed to create $BOOTFS_MNT" # # Partition $DEVICE # -DEVICE_SIZE=$(parted $DEVICE unit mb print | grep ^Disk | cut -d" " -f 3 | sed -e "s/MB//") +DEVICE_SIZE=$(parted -s $DEVICE unit mb print | grep ^Disk | cut -d" " -f 3 | sed -e "s/MB//") # If the device size is not reported there may not be a valid label if [ "$DEVICE_SIZE" = "" ] ; then - parted $DEVICE mklabel msdos || die "Failed to create MSDOS partition table" - DEVICE_SIZE=$(parted $DEVICE unit mb print | grep ^Disk | cut -d" " -f 3 | sed -e "s/MB//") + parted -s $DEVICE mklabel msdos || die "Failed to create MSDOS partition table" + DEVICE_SIZE=$(parted -s $DEVICE unit mb print | grep ^Disk | cut -d" " -f 3 | sed -e "s/MB//") fi SWAP_SIZE=$((DEVICE_SIZE*SWAP_RATIO/100)) ROOTFS_SIZE=$((DEVICE_SIZE-BOOT_SIZE-SWAP_SIZE)) @@ -262,22 +327,22 @@ debug "Deleting partition table on $DEVICE" dd if=/dev/zero of=$DEVICE bs=512 count=2 >$OUT 2>&1 || die "Failed to zero beginning of $DEVICE" debug "Creating new partition table (MSDOS) on $DEVICE" -parted $DEVICE mklabel msdos >$OUT 2>&1 || die "Failed to create MSDOS partition table" +parted -s $DEVICE mklabel msdos >$OUT 2>&1 || die "Failed to create MSDOS partition table" debug "Creating boot partition on $BOOTFS" -parted $DEVICE mkpart primary 0% $BOOT_SIZE >$OUT 2>&1 || die "Failed to create BOOT partition" +parted -s $DEVICE mkpart primary 0% $BOOT_SIZE >$OUT 2>&1 || die "Failed to create BOOT partition" debug "Enabling boot flag on $BOOTFS" -parted $DEVICE set 1 boot on >$OUT 2>&1 || die "Failed to enable boot flag" +parted -s $DEVICE set 1 boot on >$OUT 2>&1 || die "Failed to enable boot flag" debug "Creating ROOTFS partition on $ROOTFS" -parted $DEVICE mkpart primary $ROOTFS_START $ROOTFS_END >$OUT 2>&1 || die "Failed to create ROOTFS partition" +parted -s $DEVICE mkpart primary $ROOTFS_START $ROOTFS_END >$OUT 2>&1 || die "Failed to create ROOTFS partition" debug "Creating swap partition on $SWAP" -parted $DEVICE mkpart primary $SWAP_START 100% >$OUT 2>&1 || die "Failed to create SWAP partition" +parted -s $DEVICE mkpart primary $SWAP_START 100% >$OUT 2>&1 || die "Failed to create SWAP partition" if [ $DEBUG -eq 1 ]; then - parted $DEVICE print + parted -s $DEVICE print fi @@ -309,8 +374,8 @@ mkswap $SWAP >$OUT 2>&1 || die "Failed to prepare swap" # Installing to $DEVICE # debug "Mounting images and device in preparation for installation" -mount -o loop $HDDIMG $HDDIMG_MNT >$OUT 2>&1 || error "Failed to mount $HDDIMG" -mount -o loop $HDDIMG_MNT/rootfs.img $HDDIMG_ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount rootfs.img" +mount -o ro,loop $HDDIMG $HDDIMG_MNT >$OUT 2>&1 || error "Failed to mount $HDDIMG" +mount -o ro,loop $HDDIMG_MNT/rootfs.img $HDDIMG_ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount rootfs.img" mount $ROOTFS $ROOTFS_MNT >$OUT 2>&1 || error "Failed to mount $ROOTFS on $ROOTFS_MNT" mount $BOOTFS $BOOTFS_MNT >$OUT 2>&1 || error "Failed to mount $BOOTFS on $BOOTFS_MNT" @@ -319,7 +384,7 @@ EFIDIR="$BOOTFS_MNT/EFI/BOOT" cp $HDDIMG_MNT/vmlinuz $BOOTFS_MNT >$OUT 2>&1 || error "Failed to copy vmlinuz" # Copy the efi loader and configs (booti*.efi and grub.cfg if it exists) cp -r $HDDIMG_MNT/EFI $BOOTFS_MNT >$OUT 2>&1 || error "Failed to copy EFI dir" -# Silently ignore a missing gummiboot loader dir (we might just be a GRUB image) +# Silently ignore a missing systemd-boot loader dir (we might just be a GRUB image) cp -r $HDDIMG_MNT/loader $BOOTFS_MNT >$OUT 2>&1 # Update the boot loaders configurations for an installed image @@ -342,28 +407,28 @@ if [ -e "$GRUB_CFG" ]; then sed -i "s/ LABEL=[^ ]*/ /" $GRUB_CFG sed -i "s@ root=[^ ]*@ @" $GRUB_CFG - sed -i "s@vmlinuz @vmlinuz root=$TARGET_ROOTFS ro rootwait quiet @" $GRUB_CFG + sed -i "s@vmlinuz @vmlinuz root=$TARGET_ROOTFS ro rootwait console=ttyS0 console=tty0 @" $GRUB_CFG fi -# Look for a gummiboot installation -GUMMI_ENTRIES="$BOOTFS_MNT/loader/entries" -GUMMI_CFG="$GUMMI_ENTRIES/boot.conf" -if [ -d "$GUMMI_ENTRIES" ]; then - info "Configuring Gummiboot" +# Look for a systemd-boot installation +SYSTEMD_BOOT_ENTRIES="$BOOTFS_MNT/loader/entries" +SYSTEMD_BOOT_CFG="$SYSTEMD_BOOT_ENTRIES/boot.conf" +if [ -d "$SYSTEMD_BOOT_ENTRIES" ]; then + info "Configuring SystemD-boot" # remove the install target if it exists - rm $GUMMI_ENTRIES/install.conf >$OUT 2>&1 + rm $SYSTEMD_BOOT_ENTRIES/install.conf >$OUT 2>&1 - if [ ! -e "$GUMMI_CFG" ]; then - echo "ERROR: $GUMMI_CFG not found" + if [ ! -e "$SYSTEMD_BOOT_CFG" ]; then + echo "ERROR: $SYSTEMD_BOOT_CFG not found" fi - sed -i "/initrd /d" $GUMMI_CFG - sed -i "s@ root=[^ ]*@ @" $GUMMI_CFG - sed -i "s@options *LABEL=boot @options LABEL=Boot root=$TARGET_ROOTFS ro rootwait quiet @" $GUMMI_CFG + sed -i "/initrd /d" $SYSTEMD_BOOT_CFG + sed -i "s@ root=[^ ]*@ @" $SYSTEMD_BOOT_CFG + sed -i "s@options *LABEL=boot @options LABEL=Boot root=$TARGET_ROOTFS ro rootwait console=ttyS0 console=tty0 @" $SYSTEMD_BOOT_CFG fi # Ensure we have at least one EFI bootloader configured -if [ ! -e $GRUB_CFG ] && [ ! -e $GUMMI_CFG ]; then +if [ ! -e $GRUB_CFG ] && [ ! -e $SYSTEMD_BOOT_CFG ]; then die "No EFI bootloader configuration found" fi @@ -378,6 +443,9 @@ if [ -d $ROOTFS_MNT/etc/udev/ ] ; then echo "$TARGET_DEVICE" >> $ROOTFS_MNT/etc/udev/mount.blacklist fi +# Add startup.nsh script for automated boot +echo "fs0:\EFI\BOOT\bootx64.efi" > $BOOTFS_MNT/startup.nsh + # Call cleanup to unmount devices and images and remove the TMPDIR cleanup diff --git a/scripts/contrib/oe-build-perf-report-email.py b/scripts/contrib/oe-build-perf-report-email.py new file mode 100755 index 0000000000..261ca514e5 --- /dev/null +++ b/scripts/contrib/oe-build-perf-report-email.py @@ -0,0 +1,269 @@ +#!/usr/bin/python3 +# +# Send build performance test report emails +# +# Copyright (c) 2017, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +import argparse +import base64 +import logging +import os +import pwd +import re +import shutil +import smtplib +import socket +import subprocess +import sys +import tempfile +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + + +# Setup logging +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +log = logging.getLogger('oe-build-perf-report') + + +# Find js scaper script +SCRAPE_JS = os.path.join(os.path.dirname(__file__), '..', 'lib', 'build_perf', + 'scrape-html-report.js') +if not os.path.isfile(SCRAPE_JS): + log.error("Unableto find oe-build-perf-report-scrape.js") + sys.exit(1) + + +class ReportError(Exception): + """Local errors""" + pass + + +def check_utils(): + """Check that all needed utils are installed in the system""" + missing = [] + for cmd in ('phantomjs', 'optipng'): + if not shutil.which(cmd): + missing.append(cmd) + if missing: + log.error("The following tools are missing: %s", ' '.join(missing)) + sys.exit(1) + + +def parse_args(argv): + """Parse command line arguments""" + description = """Email build perf test report""" + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description=description) + + parser.add_argument('--debug', '-d', action='store_true', + help="Verbose logging") + parser.add_argument('--quiet', '-q', action='store_true', + help="Only print errors") + parser.add_argument('--to', action='append', + help="Recipients of the email") + parser.add_argument('--subject', default="Yocto build perf test report", + help="Email subject") + parser.add_argument('--outdir', '-o', + help="Store files in OUTDIR. Can be used to preserve " + "the email parts") + parser.add_argument('--text', + help="Plain text message") + parser.add_argument('--html', + help="HTML peport generated by oe-build-perf-report") + parser.add_argument('--phantomjs-args', action='append', + help="Extra command line arguments passed to PhantomJS") + + args = parser.parse_args(argv) + + if not args.html and not args.text: + parser.error("Please specify --html and/or --text") + + return args + + +def decode_png(infile, outfile): + """Parse/decode/optimize png data from a html element""" + with open(infile) as f: + raw_data = f.read() + + # Grab raw base64 data + b64_data = re.sub('^.*href="data:image/png;base64,', '', raw_data, 1) + b64_data = re.sub('">.+$', '', b64_data, 1) + + # Replace file with proper decoded png + with open(outfile, 'wb') as f: + f.write(base64.b64decode(b64_data)) + + subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT) + + +def encode_png(pngfile): + """Encode png into a <img> html element""" + with open(pngfile, 'rb') as f: + data = f.read() + + b64_data = base64.b64encode(data) + return '<img src="data:image/png;base64,' + b64_data.decode('utf-8') + '">\n' + + +def mangle_html_report(infile, outfile, pngs): + """Mangle html file into a email compatible format""" + paste = True + png_dir = os.path.dirname(outfile) + with open(infile) as f_in: + with open(outfile, 'w') as f_out: + for line in f_in.readlines(): + stripped = line.strip() + # Strip out scripts + if stripped == '<!--START-OF-SCRIPTS-->': + paste = False + elif stripped == '<!--END-OF-SCRIPTS-->': + paste = True + elif paste: + if re.match('^.+href="data:image/png;base64', stripped): + # Strip out encoded pngs (as they're huge in size) + continue + elif 'www.gstatic.com' in stripped: + # HACK: drop references to external static pages + continue + + # Replace charts with <img> elements + match = re.match('<div id="(?P<id>\w+)"', stripped) + if match and match.group('id') in pngs: + #f_out.write('<img src="{}">\n'.format(match.group('id') + '.png')) + png_file = os.path.join(png_dir, match.group('id') + '.png') + f_out.write(encode_png(png_file)) + else: + f_out.write(line) + + +def scrape_html_report(report, outdir, phantomjs_extra_args=None): + """Scrape html report into a format sendable by email""" + tmpdir = tempfile.mkdtemp(dir='.') + log.debug("Using tmpdir %s for phantomjs output", tmpdir) + + if not os.path.isdir(outdir): + os.mkdir(outdir) + if os.path.splitext(report)[1] not in ('.html', '.htm'): + raise ReportError("Invalid file extension for report, needs to be " + "'.html' or '.htm'") + + try: + log.info("Scraping HTML report with PhangomJS") + extra_args = phantomjs_extra_args if phantomjs_extra_args else [] + subprocess.check_output(['phantomjs', '--debug=true'] + extra_args + + [SCRAPE_JS, report, tmpdir], + stderr=subprocess.STDOUT) + + pngs = [] + attachments = [] + for fname in os.listdir(tmpdir): + base, ext = os.path.splitext(fname) + if ext == '.png': + log.debug("Decoding %s", fname) + decode_png(os.path.join(tmpdir, fname), + os.path.join(outdir, fname)) + pngs.append(base) + attachments.append(fname) + elif ext in ('.html', '.htm'): + report_file = fname + else: + log.warning("Unknown file extension: '%s'", ext) + #shutil.move(os.path.join(tmpdir, fname), outdir) + + log.debug("Mangling html report file %s", report_file) + mangle_html_report(os.path.join(tmpdir, report_file), + os.path.join(outdir, report_file), pngs) + return report_file, attachments + finally: + shutil.rmtree(tmpdir) + +def send_email(text_fn, html_fn, subject, recipients): + """Send email""" + # Generate email message + text_msg = html_msg = None + if text_fn: + with open(text_fn) as f: + text_msg = MIMEText("Yocto build performance test report.\n" + + f.read(), 'plain') + if html_fn: + with open(html_fn) as f: + html_msg = MIMEText(f.read(), 'html') + + if text_msg and html_msg: + msg = MIMEMultipart('alternative') + msg.attach(text_msg) + msg.attach(html_msg) + elif text_msg: + msg = text_msg + elif html_msg: + msg = html_msg + else: + raise ReportError("Neither plain text nor html body specified") + + pw_data = pwd.getpwuid(os.getuid()) + full_name = pw_data.pw_gecos.split(',')[0] + email = os.environ.get('EMAIL', + '{}@{}'.format(pw_data.pw_name, socket.getfqdn())) + msg['From'] = "{} <{}>".format(full_name, email) + msg['To'] = ', '.join(recipients) + msg['Subject'] = subject + + # Send email + with smtplib.SMTP('localhost') as smtp: + smtp.send_message(msg) + + +def main(argv=None): + """Script entry point""" + args = parse_args(argv) + if args.quiet: + log.setLevel(logging.ERROR) + if args.debug: + log.setLevel(logging.DEBUG) + + check_utils() + + if args.outdir: + outdir = args.outdir + if not os.path.exists(outdir): + os.mkdir(outdir) + else: + outdir = tempfile.mkdtemp(dir='.') + + try: + log.debug("Storing email parts in %s", outdir) + html_report = None + if args.html: + scrape_html_report(args.html, outdir, args.phantomjs_args) + html_report = os.path.join(outdir, os.path.basename(args.html)) + + if args.to: + log.info("Sending email to %s", ', '.join(args.to)) + send_email(args.text, html_report, args.subject, args.to) + except subprocess.CalledProcessError as err: + log.error("%s, with output:\n%s", str(err), err.output.decode()) + return 1 + except ReportError as err: + log.error(err) + return 1 + finally: + if not args.outdir: + log.debug("Wiping %s", outdir) + shutil.rmtree(outdir) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/contrib/python/generate-manifest-2.7.py b/scripts/contrib/python/generate-manifest-2.7.py index 65486d8ec4..8c3655d395 100755 --- a/scripts/contrib/python/generate-manifest-2.7.py +++ b/scripts/contrib/python/generate-manifest-2.7.py @@ -9,10 +9,14 @@ # * Updated to no longer generate special -dbg package, instead use the # single system -dbg # * Update version with ".1" to indicate this change +# +# February 26, 2017 -- Ming Liu <peter.x.liu@external.atlascopco.com> +# * Updated to support generating manifest for native python import os import sys import time +import argparse VERSION = "2.7.2" @@ -21,16 +25,16 @@ __version__ = "20110222.2" class MakefileMaker: - def __init__( self, outfile ): + def __init__( self, outfile, isNative ): """initialize""" self.packages = {} self.targetPrefix = "${libdir}/python%s/" % VERSION[:3] + self.isNative = isNative self.output = outfile self.out( """ # WARNING: This file is AUTO GENERATED: Manual edits will be lost next time I regenerate the file. -# Generator: '%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> -# Visit the Python for Embedded Systems Site => http://www.Vanille.de/projects/python.spy -""" % ( sys.argv[0], __version__ ) ) +# Generator: '%s%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> +""" % ( sys.argv[0], ' --native' if isNative else '', __version__ ) ) # # helper functions @@ -66,6 +70,20 @@ class MakefileMaker: global VERSION # + # generate rprovides line for native + # + + if self.isNative: + rprovideLine = 'RPROVIDES+="' + for name in sorted(self.packages): + rprovideLine += "%s-native " % name.replace( '${PN}', 'python' ) + rprovideLine += '"' + + self.out( rprovideLine ) + self.out( "" ) + return + + # # generate provides line # @@ -97,7 +115,7 @@ class MakefileMaker: # generate package variables # - for name, data in sorted(self.packages.iteritems()): + for name, data in sorted(self.packages.items()): desc, deps, files = data # @@ -130,7 +148,7 @@ class MakefileMaker: self.out( 'SUMMARY_${PN}-modules="All Python modules"' ) line = 'RDEPENDS_${PN}-modules="' - for name, data in sorted(self.packages.iteritems()): + for name, data in sorted(self.packages.items()): if name not in ['${PN}-dev', '${PN}-distutils-staticdev']: line += "%s " % name @@ -147,31 +165,35 @@ class MakefileMaker: self.doEpilog() if __name__ == "__main__": + parser = argparse.ArgumentParser( description='generate python manifest' ) + parser.add_argument( '-n', '--native', help='generate manifest for native python', action='store_true' ) + parser.add_argument( 'outfile', metavar='OUTPUT_FILE', nargs='?', default='', help='Output file (defaults to stdout)' ) + args = parser.parse_args() - if len( sys.argv ) > 1: + if args.outfile: try: - os.unlink(sys.argv[1]) + os.unlink( args.outfile ) except Exception: sys.exc_clear() - outfile = file( sys.argv[1], "w" ) + outfile = open( args.outfile, "w" ) else: outfile = sys.stdout - m = MakefileMaker( outfile ) + m = MakefileMaker( outfile, args.native ) # Add packages here. Only specify dlopen-style library dependencies here, no ldd-style dependencies! # Parameters: revision, name, description, dependencies, filenames # m.addPackage( "${PN}-core", "Python interpreter and core modules", "${PN}-lang ${PN}-re", - "__future__.* _abcoll.* abc.* copy.* copy_reg.* ConfigParser.* " + + "__future__.* _abcoll.* abc.* ast.* copy.* copy_reg.* ConfigParser.* " + "genericpath.* getopt.* linecache.* new.* " + "os.* posixpath.* struct.* " + "warnings.* site.* stat.* " + "UserDict.* UserList.* UserString.* " + "lib-dynload/binascii.so lib-dynload/_struct.so lib-dynload/time.so " + - "lib-dynload/xreadlines.so types.* platform.* ${bindir}/python* " + - "_weakrefset.* sysconfig.* config/Makefile " + + "lib-dynload/xreadlines.so types.* platform.* ${bindir}/python* " + + "_weakrefset.* sysconfig.* _sysconfigdata.* " + "${includedir}/python${PYTHON_MAJMIN}/pyconfig*.h " + "${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py ") @@ -185,7 +207,8 @@ if __name__ == "__main__": "${base_libdir}/*.a " + "${base_libdir}/*.o " + "${datadir}/aclocal " + - "${datadir}/pkgconfig " ) + "${datadir}/pkgconfig " + + "config/Makefile ") m.addPackage( "${PN}-2to3", "Python automated Python 2 to 3 code translator", "${PN}-core", "${bindir}/2to3 lib2to3" ) # package @@ -244,16 +267,12 @@ if __name__ == "__main__": m.addPackage( "${PN}-distutils-staticdev", "Python distribution utilities (static libraries)", "${PN}-distutils", "config/lib*.a" ) # package - m.addPackage( "${PN}-distutils", "Python Distribution Utilities", "${PN}-core", + m.addPackage( "${PN}-distutils", "Python Distribution Utilities", "${PN}-core ${PN}-email", "config distutils" ) # package m.addPackage( "${PN}-doctest", "Python framework for running examples in docstrings", "${PN}-core ${PN}-lang ${PN}-io ${PN}-re ${PN}-unittest ${PN}-debugger ${PN}-difflib", "doctest.*" ) - # FIXME consider adding to some higher level package - m.addPackage( "${PN}-elementtree", "Python elementree", "${PN}-core", - "lib-dynload/_elementtree.so" ) - m.addPackage( "${PN}-email", "Python email support", "${PN}-core ${PN}-io ${PN}-re ${PN}-mime ${PN}-audio ${PN}-image ${PN}-netclient", "imaplib.* email" ) # package @@ -275,11 +294,11 @@ if __name__ == "__main__": m.addPackage( "${PN}-image", "Python graphical image handling", "${PN}-core", "colorsys.* imghdr.* lib-dynload/imageop.so lib-dynload/rgbimg.so" ) - m.addPackage( "${PN}-io", "Python low-level I/O", "${PN}-core ${PN}-math ${PN}-textutils", + m.addPackage( "${PN}-io", "Python low-level I/O", "${PN}-core ${PN}-math ${PN}-textutils ${PN}-netclient ${PN}-contextlib", "lib-dynload/_socket.so lib-dynload/_io.so lib-dynload/_ssl.so lib-dynload/select.so lib-dynload/termios.so lib-dynload/cStringIO.so " + "pipes.* socket.* ssl.* tempfile.* StringIO.* io.* _pyio.*" ) - m.addPackage( "${PN}-json", "Python JSON support", "${PN}-core ${PN}-math ${PN}-re", + m.addPackage( "${PN}-json", "Python JSON support", "${PN}-core ${PN}-math ${PN}-re ${PN}-codecs", "json lib-dynload/_json.so" ) # package m.addPackage( "${PN}-lang", "Python low-level language support", "${PN}-core", @@ -310,11 +329,11 @@ if __name__ == "__main__": "*Cookie*.* " + "base64.* cookielib.* ftplib.* gopherlib.* hmac.* httplib.* mimetypes.* nntplib.* poplib.* smtplib.* telnetlib.* urllib.* urllib2.* urlparse.* uuid.* rfc822.* mimetools.*" ) - m.addPackage( "${PN}-netserver", "Python Internet Protocol servers", "${PN}-core ${PN}-netclient", + m.addPackage( "${PN}-netserver", "Python Internet Protocol servers", "${PN}-core ${PN}-netclient ${PN}-shell ${PN}-threading", "cgi.* *HTTPServer.* SocketServer.*" ) m.addPackage( "${PN}-numbers", "Python number APIs", "${PN}-core ${PN}-lang ${PN}-re", - "decimal.* numbers.*" ) + "decimal.* fractions.* numbers.*" ) m.addPackage( "${PN}-pickle", "Python serialisation/persistence support", "${PN}-core ${PN}-codecs ${PN}-io ${PN}-re", "pickle.* shelve.* lib-dynload/cPickle.so pickletools.*" ) @@ -322,6 +341,9 @@ if __name__ == "__main__": m.addPackage( "${PN}-pkgutil", "Python package extension utility support", "${PN}-core", "pkgutil.*") + m.addPackage( "${PN}-plistlib", "Generate and parse Mac OS X .plist files", "${PN}-core ${PN}-datetime ${PN}-io", + "plistlib.*") + m.addPackage( "${PN}-pprint", "Python pretty-print support", "${PN}-core ${PN}-io", "pprint.*" ) @@ -361,7 +383,7 @@ if __name__ == "__main__": m.addPackage( "${PN}-terminal", "Python terminal controlling support", "${PN}-core ${PN}-io", "pty.* tty.*" ) - m.addPackage( "${PN}-tests", "Python tests", "${PN}-core", + m.addPackage( "${PN}-tests", "Python tests", "${PN}-core ${PN}-modules", "test" ) # package m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang", @@ -376,8 +398,8 @@ if __name__ == "__main__": m.addPackage( "${PN}-unixadmin", "Python Unix administration support", "${PN}-core", "lib-dynload/nis.so lib-dynload/grp.so lib-dynload/pwd.so getpass.*" ) - m.addPackage( "${PN}-xml", "Python basic XML support", "${PN}-core ${PN}-elementtree ${PN}-re", - "lib-dynload/pyexpat.so xml xmllib.*" ) # package + m.addPackage( "${PN}-xml", "Python basic XML support", "${PN}-core ${PN}-re", + "lib-dynload/_elementtree.so lib-dynload/pyexpat.so xml xmllib.*" ) # package m.addPackage( "${PN}-xmlrpc", "Python XML-RPC support", "${PN}-core ${PN}-xml ${PN}-netserver ${PN}-lang", "xmlrpclib.* SimpleXMLRPCServer.* DocXMLRPCServer.*" ) @@ -388,4 +410,11 @@ if __name__ == "__main__": m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", "mailbox.*" ) + m.addPackage( "${PN}-argparse", "Python command line argument parser", "${PN}-core ${PN}-codecs ${PN}-textutils", + "argparse.*" ) + + m.addPackage( "${PN}-contextlib", "Python utilities for with-statement" + + "contexts.", "${PN}-core", + "${libdir}/python${PYTHON_MAJMIN}/contextlib.*" ) + m.make() diff --git a/scripts/contrib/python/generate-manifest-3.3.py b/scripts/contrib/python/generate-manifest-3.5.py index 288def293d..075860c418 100755 --- a/scripts/contrib/python/generate-manifest-3.3.py +++ b/scripts/contrib/python/generate-manifest-3.5.py @@ -13,27 +13,31 @@ # 2014 Khem Raj <raj.khem@gmail.com> # Added python3 support # +# February 26, 2017 -- Ming Liu <peter.x.liu@external.atlascopco.com> +# * Updated to support generating manifest for native python3 + import os import sys import time +import argparse -VERSION = "3.3.3" +VERSION = "3.5.0" __author__ = "Michael 'Mickey' Lauer <mlauer@vanille-media.de>" __version__ = "20140131" class MakefileMaker: - def __init__( self, outfile ): + def __init__( self, outfile, isNative ): """initialize""" self.packages = {} self.targetPrefix = "${libdir}/python%s/" % VERSION[:3] + self.isNative = isNative self.output = outfile self.out( """ # WARNING: This file is AUTO GENERATED: Manual edits will be lost next time I regenerate the file. -# Generator: '%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> -# Visit the Python for Embedded Systems Site => http://www.Vanille.de/projects/python.spy -""" % ( sys.argv[0], __version__ ) ) +# Generator: '%s%s' Version %s (C) 2002-2010 Michael 'Mickey' Lauer <mlauer@vanille-media.de> +""" % ( sys.argv[0], ' --native' if isNative else '', __version__ ) ) # # helper functions @@ -59,16 +63,40 @@ class MakefileMaker: for filename in filenames: if filename[0] != "$": fullFilenames.append( "%s%s" % ( self.targetPrefix, filename ) ) + fullFilenames.append( "%s%s" % ( self.targetPrefix, + self.pycachePath( filename ) ) ) else: fullFilenames.append( filename ) self.packages[name] = description, dependencies, fullFilenames + def pycachePath( self, filename ): + dirname = os.path.dirname( filename ) + basename = os.path.basename( filename ) + if '.' in basename: + return os.path.join( dirname, '__pycache__', basename ) + else: + return os.path.join( dirname, basename, '__pycache__' ) + def doBody( self ): """generate body of Makefile""" global VERSION # + # generate rprovides line for native + # + + if self.isNative: + rprovideLine = 'RPROVIDES+="' + for name in sorted(self.packages): + rprovideLine += "%s-native " % name.replace( '${PN}', 'python3' ) + rprovideLine += '"' + + self.out( rprovideLine ) + self.out( "" ) + return + + # # generate provides line # @@ -100,7 +128,7 @@ class MakefileMaker: # generate package variables # - for name, data in sorted(self.packages.iteritems()): + for name, data in sorted(self.packages.items()): desc, deps, files = data # @@ -133,7 +161,7 @@ class MakefileMaker: self.out( 'SUMMARY_${PN}-modules="All Python modules"' ) line = 'RDEPENDS_${PN}-modules="' - for name, data in sorted(self.packages.iteritems()): + for name, data in sorted(self.packages.items()): if name not in ['${PN}-dev', '${PN}-distutils-staticdev']: line += "%s " % name @@ -150,33 +178,39 @@ class MakefileMaker: self.doEpilog() if __name__ == "__main__": + parser = argparse.ArgumentParser( description='generate python3 manifest' ) + parser.add_argument( '-n', '--native', help='generate manifest for native python3', action='store_true' ) + parser.add_argument( 'outfile', metavar='OUTPUT_FILE', nargs='?', default='', help='Output file (defaults to stdout)' ) + args = parser.parse_args() - if len( sys.argv ) > 1: + if args.outfile: try: - os.unlink(sys.argv[1]) + os.unlink( args.outfile ) except Exception: sys.exc_clear() - outfile = file( sys.argv[1], "w" ) + outfile = open( args.outfile, "w" ) else: outfile = sys.stdout - m = MakefileMaker( outfile ) + m = MakefileMaker( outfile, args.native ) # Add packages here. Only specify dlopen-style library dependencies here, no ldd-style dependencies! # Parameters: revision, name, description, dependencies, filenames # - m.addPackage( "${PN}-core", "Python interpreter and core modules", "${PN}-lang ${PN}-re", - "__future__.* _abcoll.* abc.* copy.* copy_reg.* ConfigParser.* " + + m.addPackage( "${PN}-core", "Python interpreter and core modules", "${PN}-lang ${PN}-re ${PN}-reprlib ${PN}-codecs ${PN}-io ${PN}-math", + "__future__.* _abcoll.* abc.* ast.* copy.* copyreg.* configparser.* " + "genericpath.* getopt.* linecache.* new.* " + "os.* posixpath.* struct.* " + "warnings.* site.* stat.* " + "UserDict.* UserList.* UserString.* " + "lib-dynload/binascii.*.so lib-dynload/_struct.*.so lib-dynload/time.*.so " + "lib-dynload/xreadlines.*.so types.* platform.* ${bindir}/python* " + - "_weakrefset.* sysconfig.* config/Makefile " + - "${includedir}/python${PYTHON_MAJMIN}/pyconfig*.h " + + "_weakrefset.* sysconfig.* _sysconfigdata.* " + + "${includedir}/python${PYTHON_BINABI}/pyconfig*.h " + "${libdir}/python${PYTHON_MAJMIN}/collections " + + "${libdir}/python${PYTHON_MAJMIN}/_collections_abc.* " + + "${libdir}/python${PYTHON_MAJMIN}/_sitebuiltins.* " + "${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py ") m.addPackage( "${PN}-dev", "Python development package", "${PN}-core", @@ -189,10 +223,11 @@ if __name__ == "__main__": "${base_libdir}/*.a " + "${base_libdir}/*.o " + "${datadir}/aclocal " + - "${datadir}/pkgconfig " ) + "${datadir}/pkgconfig " + + "config/Makefile ") m.addPackage( "${PN}-2to3", "Python automated Python 2 to 3 code translator", "${PN}-core", - "${bindir}/2to3 lib2to3" ) # package + "lib2to3" ) # package m.addPackage( "${PN}-idle", "Python Integrated Development Environment", "${PN}-core ${PN}-tkinter", "${bindir}/idle idlelib" ) # package @@ -206,14 +241,20 @@ if __name__ == "__main__": m.addPackage( "${PN}-audio", "Python Audio Handling", "${PN}-core", "wave.* chunk.* sndhdr.* lib-dynload/ossaudiodev.*.so lib-dynload/audioop.*.so audiodev.* sunaudio.* sunau.* toaiff.*" ) + m.addPackage( "${PN}-argparse", "Python command line argument parser", "${PN}-core ${PN}-codecs ${PN}-textutils", + "argparse.*" ) + + m.addPackage( "${PN}-asyncio", "Python Asynchronous I/O, event loop, coroutines and tasks", "${PN}-core", + "asyncio" ) + m.addPackage( "${PN}-codecs", "Python codecs, encodings & i18n support", "${PN}-core ${PN}-lang", "codecs.* encodings gettext.* locale.* lib-dynload/_locale.*.so lib-dynload/_codecs* lib-dynload/_multibytecodec.*.so lib-dynload/unicodedata.*.so stringprep.* xdrlib.*" ) m.addPackage( "${PN}-compile", "Python bytecode compilation support", "${PN}-core", "py_compile.* compileall.*" ) - m.addPackage( "${PN}-compression", "Python high-level compression support", "${PN}-core ${PN}-codecs", - "gzip.* zipfile.* tarfile.* lib-dynload/bz2.*.so" ) + m.addPackage( "${PN}-compression", "Python high-level compression support", "${PN}-core ${PN}-codecs ${PN}-importlib ${PN}-threading ${PN}-shell", + "gzip.* zipfile.* tarfile.* lib-dynload/bz2.*.so lib-dynload/zlib.*.so" ) m.addPackage( "${PN}-crypt", "Python basic cryptographic and hashing support", "${PN}-core", "hashlib.* md5.* sha.* lib-dynload/crypt.*.so lib-dynload/_hashlib.*.so lib-dynload/_sha256.*.so lib-dynload/_sha512.*.so" ) @@ -224,16 +265,16 @@ if __name__ == "__main__": m.addPackage( "${PN}-curses", "Python curses support", "${PN}-core", "curses lib-dynload/_curses.*.so lib-dynload/_curses_panel.*.so" ) # directory + low level module - m.addPackage( "${PN}-ctypes", "Python C types support", "${PN}-core", + m.addPackage( "${PN}-ctypes", "Python C types support", "${PN}-core ${PN}-subprocess", "ctypes lib-dynload/_ctypes.*.so lib-dynload/_ctypes_test.*.so" ) # directory + low level module m.addPackage( "${PN}-datetime", "Python calendar and time support", "${PN}-core ${PN}-codecs", - "_strptime.* calendar.* lib-dynload/datetime.*.so" ) + "_strptime.* calendar.* datetime.* lib-dynload/_datetime.*.so" ) m.addPackage( "${PN}-db", "Python file-based database support", "${PN}-core", "anydbm.* dumbdbm.* whichdb.* dbm lib-dynload/_dbm.*.so" ) - m.addPackage( "${PN}-debugger", "Python debugger", "${PN}-core ${PN}-io ${PN}-lang ${PN}-re ${PN}-stringold ${PN}-shell ${PN}-pprint", + m.addPackage( "${PN}-debugger", "Python debugger", "${PN}-core ${PN}-io ${PN}-lang ${PN}-re ${PN}-stringold ${PN}-shell ${PN}-pprint ${PN}-importlib ${PN}-pkgutil", "bdb.* pdb.*" ) m.addPackage( "${PN}-difflib", "Python helpers for computing deltas between objects", "${PN}-lang ${PN}-re", @@ -242,27 +283,26 @@ if __name__ == "__main__": m.addPackage( "${PN}-distutils-staticdev", "Python distribution utilities (static libraries)", "${PN}-distutils", "config/lib*.a" ) # package - m.addPackage( "${PN}-distutils", "Python Distribution Utilities", "${PN}-core", + m.addPackage( "${PN}-distutils", "Python Distribution Utilities", "${PN}-core ${PN}-email", "config distutils" ) # package m.addPackage( "${PN}-doctest", "Python framework for running examples in docstrings", "${PN}-core ${PN}-lang ${PN}-io ${PN}-re ${PN}-unittest ${PN}-debugger ${PN}-difflib", "doctest.*" ) - # FIXME consider adding to some higher level package - m.addPackage( "${PN}-elementtree", "Python elementree", "${PN}-core", - "lib-dynload/_elementtree.*.so" ) - m.addPackage( "${PN}-email", "Python email support", "${PN}-core ${PN}-io ${PN}-re ${PN}-mime ${PN}-audio ${PN}-image ${PN}-netclient", "imaplib.* email" ) # package + m.addPackage( "${PN}-enum", "Python support for enumerations", "${PN}-core", + "enum.*" ) + m.addPackage( "${PN}-fcntl", "Python's fcntl interface", "${PN}-core", "lib-dynload/fcntl.*.so" ) m.addPackage( "${PN}-html", "Python HTML processing support", "${PN}-core", - "formatter.* htmlentitydefs.* htmllib.* markupbase.* sgmllib.* HTMLParser.* " ) + "formatter.* htmlentitydefs.* html htmllib.* markupbase.* sgmllib.* HTMLParser.* " ) - m.addPackage( "${PN}-importlib", "Python import implementation library", "${PN}-core", - "importlib" ) + m.addPackage( "${PN}-importlib", "Python import implementation library", "${PN}-core ${PN}-lang", + "importlib imp.*" ) m.addPackage( "${PN}-gdbm", "Python GNU database support", "${PN}-core", "lib-dynload/_gdbm.*.so" ) @@ -272,15 +312,15 @@ if __name__ == "__main__": m.addPackage( "${PN}-io", "Python low-level I/O", "${PN}-core ${PN}-math", "lib-dynload/_socket.*.so lib-dynload/_io.*.so lib-dynload/_ssl.*.so lib-dynload/select.*.so lib-dynload/termios.*.so lib-dynload/cStringIO.*.so " + - "pipes.* socket.* ssl.* tempfile.* StringIO.* io.* _pyio.*" ) + "ipaddress.* pipes.* socket.* ssl.* tempfile.* StringIO.* io.* _pyio.*" ) m.addPackage( "${PN}-json", "Python JSON support", "${PN}-core ${PN}-math ${PN}-re", "json lib-dynload/_json.*.so" ) # package - m.addPackage( "${PN}-lang", "Python low-level language support", "${PN}-core", + m.addPackage( "${PN}-lang", "Python low-level language support", "${PN}-core ${PN}-importlib", "lib-dynload/_bisect.*.so lib-dynload/_collections.*.so lib-dynload/_heapq.*.so lib-dynload/_weakref.*.so lib-dynload/_functools.*.so " + "lib-dynload/array.*.so lib-dynload/itertools.*.so lib-dynload/operator.*.so lib-dynload/parser.*.so " + - "atexit.* bisect.* code.* codeop.* collections.* dis.* functools.* heapq.* inspect.* keyword.* opcode.* symbol.* repr.* token.* " + + "atexit.* bisect.* code.* codeop.* collections.* _collections_abc.* contextlib.* dis.* functools.* heapq.* inspect.* keyword.* opcode.* operator.* symbol.* repr.* token.* " + "tokenize.* traceback.* weakref.*" ) m.addPackage( "${PN}-logging", "Python logging support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-stringold", @@ -289,7 +329,7 @@ if __name__ == "__main__": m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", "mailbox.*" ) - m.addPackage( "${PN}-math", "Python math support", "${PN}-core", + m.addPackage( "${PN}-math", "Python math support", "${PN}-core ${PN}-crypt", "lib-dynload/cmath.*.so lib-dynload/math.*.so lib-dynload/_random.*.so random.* sets.*" ) m.addPackage( "${PN}-mime", "Python MIME handling APIs", "${PN}-core ${PN}-io", @@ -301,18 +341,18 @@ if __name__ == "__main__": m.addPackage( "${PN}-multiprocessing", "Python multiprocessing support", "${PN}-core ${PN}-io ${PN}-lang ${PN}-pickle ${PN}-threading ${PN}-ctypes ${PN}-mmap", "lib-dynload/_multiprocessing.*.so multiprocessing" ) # package - m.addPackage( "${PN}-netclient", "Python Internet Protocol clients", "${PN}-core ${PN}-crypt ${PN}-datetime ${PN}-io ${PN}-lang ${PN}-logging ${PN}-mime", + m.addPackage( "${PN}-netclient", "Python Internet Protocol clients", "${PN}-argparse ${PN}-core ${PN}-crypt ${PN}-datetime ${PN}-io ${PN}-lang ${PN}-logging ${PN}-mime ${PN}-html", "*Cookie*.* " + - "base64.* cookielib.* ftplib.* gopherlib.* hmac.* httplib.* mimetypes.* nntplib.* poplib.* smtplib.* telnetlib.* urllib.* urllib2.* urlparse.* uuid.* rfc822.* mimetools.*" ) + "base64.* cookielib.* ftplib.* gopherlib.* hmac.* http* httplib.* mimetypes.* nntplib.* poplib.* smtplib.* telnetlib.* urllib uuid.* rfc822.* mimetools.*" ) - m.addPackage( "${PN}-netserver", "Python Internet Protocol servers", "${PN}-core ${PN}-netclient", - "cgi.* *HTTPServer.* SocketServer.*" ) + m.addPackage( "${PN}-netserver", "Python Internet Protocol servers", "${PN}-core ${PN}-netclient ${PN}-shell ${PN}-threading", + "cgi.* socketserver.* *HTTPServer.* SocketServer.*" ) m.addPackage( "${PN}-numbers", "Python number APIs", "${PN}-core ${PN}-lang ${PN}-re", - "decimal.* numbers.*" ) + "decimal.* fractions.* numbers.*" ) m.addPackage( "${PN}-pickle", "Python serialisation/persistence support", "${PN}-core ${PN}-codecs ${PN}-io ${PN}-re", - "pickle.* shelve.* lib-dynload/cPickle.*.so pickletools.*" ) + "_compat_pickle.* pickle.* shelve.* lib-dynload/cPickle.*.so pickletools.*" ) m.addPackage( "${PN}-pkgutil", "Python package extension utility support", "${PN}-core", "pkgutil.*") @@ -329,17 +369,23 @@ if __name__ == "__main__": m.addPackage( "${PN}-readline", "Python readline support", "${PN}-core", "lib-dynload/readline.*.so rlcompleter.*" ) + m.addPackage( "${PN}-reprlib", "Python alternate repr() implementation", "${PN}-core", + "reprlib.py" ) + m.addPackage( "${PN}-resource", "Python resource control interface", "${PN}-core", "lib-dynload/resource.*.so" ) - m.addPackage( "${PN}-shell", "Python shell-like functionality", "${PN}-core ${PN}-re", + m.addPackage( "${PN}-selectors", "Python High-level I/O multiplexing", "${PN}-core", + "selectors.*" ) + + m.addPackage( "${PN}-shell", "Python shell-like functionality", "${PN}-core ${PN}-re ${PN}-compression", "cmd.* commands.* dircache.* fnmatch.* glob.* popen2.* shlex.* shutil.*" ) - m.addPackage( "${PN}-robotparser", "Python robots.txt parser", "${PN}-core ${PN}-netclient", - "urllib/robotparser.*") + m.addPackage( "${PN}-signal", "Python set handlers for asynchronous events support", "${PN}-core ${PN}-enum", + "signal.*" ) - m.addPackage( "${PN}-subprocess", "Python subprocess support", "${PN}-core ${PN}-io ${PN}-re ${PN}-fcntl ${PN}-pickle", - "subprocess.*" ) + m.addPackage( "${PN}-subprocess", "Python subprocess support", "${PN}-core ${PN}-io ${PN}-re ${PN}-fcntl ${PN}-pickle ${PN}-threading ${PN}-signal ${PN}-selectors", + "subprocess.* lib-dynload/_posixsubprocess.*.so" ) m.addPackage( "${PN}-sqlite3", "Python Sqlite3 database support", "${PN}-core ${PN}-datetime ${PN}-lang ${PN}-crypt ${PN}-io ${PN}-threading", "lib-dynload/_sqlite3.*.so sqlite3/dbapi2.* sqlite3/__init__.* sqlite3/dump.*" ) @@ -360,21 +406,24 @@ if __name__ == "__main__": "test" ) # package m.addPackage( "${PN}-threading", "Python threading & synchronization support", "${PN}-core ${PN}-lang", - "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* Queue.*" ) + "_threading_local.* dummy_thread.* dummy_threading.* mutex.* threading.* queue.*" ) m.addPackage( "${PN}-tkinter", "Python Tcl/Tk bindings", "${PN}-core", "lib-dynload/_tkinter.*.so lib-tk tkinter" ) # package + m.addPackage( "${PN}-typing", "Python typing support", "${PN}-core", + "typing.*" ) + m.addPackage( "${PN}-unittest", "Python unit testing framework", "${PN}-core ${PN}-stringold ${PN}-lang ${PN}-io ${PN}-difflib ${PN}-pprint ${PN}-shell", "unittest/" ) m.addPackage( "${PN}-unixadmin", "Python Unix administration support", "${PN}-core", "lib-dynload/nis.*.so lib-dynload/grp.*.so lib-dynload/pwd.*.so getpass.*" ) - m.addPackage( "${PN}-xml", "Python basic XML support", "${PN}-core ${PN}-elementtree ${PN}-re", - "lib-dynload/pyexpat.*.so xml xmllib.*" ) # package + m.addPackage( "${PN}-xml", "Python basic XML support", "${PN}-core ${PN}-re", + "lib-dynload/_elementtree.*.so lib-dynload/pyexpat.*.so xml xmllib.*" ) # package - m.addPackage( "${PN}-xmlrpc", "Python XML-RPC support", "${PN}-core ${PN}-xml ${PN}-netserver ${PN}-lang", + m.addPackage( "${PN}-xmlrpc", "Python XML-RPC support", "${PN}-core ${PN}-xml ${PN}-netserver ${PN}-lang ${PN}-pydoc", "xmlrpclib.* SimpleXMLRPCServer.* DocXMLRPCServer.* xmlrpc" ) m.addPackage( "${PN}-mailbox", "Python mailbox format support", "${PN}-core ${PN}-mime", diff --git a/scripts/contrib/uncovered b/scripts/contrib/uncovered new file mode 100755 index 0000000000..a8399ad170 --- /dev/null +++ b/scripts/contrib/uncovered @@ -0,0 +1,39 @@ +#!/bin/bash -eur +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Find python modules uncovered by oe-seltest +# +# Copyright (c) 2016, Intel Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Author: Ed Bartosh <ed.bartosh@linux.intel.com> +# + +if [ ! "$#" -eq 1 -o -t 0 ] ; then + echo 'Usage: coverage report | ./scripts/contrib/uncovered <dir>' 1>&2 + exit 1 +fi + +path=$(readlink -ev $1) + +if [ ! -d "$path" ] ; then + echo "directory $1 doesn't exist" 1>&2 + exit 1 +fi + +diff -u <(grep "$path" | grep -v '0%$' | cut -f1 -d: | sort) \ + <(find $path | xargs file | grep 'Python script' | cut -f1 -d:| sort) | \ + grep "^+$path" | cut -c2- diff --git a/scripts/contrib/verify-homepage.py b/scripts/contrib/verify-homepage.py index 86cc82bca3..76f1749cfa 100755 --- a/scripts/contrib/verify-homepage.py +++ b/scripts/contrib/verify-homepage.py @@ -1,63 +1,62 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -# This script is used for verify HOMEPAGE. +# This script can be used to verify HOMEPAGE values for all recipes in +# the current configuration. # The result is influenced by network environment, since the timeout of connect url is 5 seconds as default. import sys import os import subprocess -import urllib2 +import urllib.request -def search_bitbakepath(): - bitbakepath = "" - # Search path to bitbake lib dir in order to load bb modules - if os.path.exists(os.path.join(os.path.dirname(sys.argv[0]), '../../bitbake/lib/bb')): - bitbakepath = os.path.join(os.path.dirname(sys.argv[0]), '../../bitbake/lib') - bitbakepath = os.path.abspath(bitbakepath) - else: - # Look for bitbake/bin dir in PATH - for pth in os.environ['PATH'].split(':'): - if os.path.exists(os.path.join(pth, '../lib/bb')): - bitbakepath = os.path.abspath(os.path.join(pth, '../lib')) - break - if not bitbakepath: - sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") - sys.exit(1) - return bitbakepath - -# For importing the following modules -sys.path.insert(0, search_bitbakepath()) +# Allow importing scripts/lib modules +scripts_path = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + '/..') +lib_path = scripts_path + '/lib' +sys.path = sys.path + [lib_path] +import scriptpath +import scriptutils + +# Allow importing bitbake modules +bitbakepath = scriptpath.add_bitbake_lib_path() + import bb.tinfoil +logger = scriptutils.logger_create('verify_homepage') + def wgetHomepage(pn, homepage): result = subprocess.call('wget ' + '-q -T 5 -t 1 --spider ' + homepage, shell = True) if result: - bb.warn("Failed to verify HOMEPAGE (%s) of %s" % (homepage, pn)) + logger.warn("%s: failed to verify HOMEPAGE: %s " % (pn, homepage)) return 1 else: return 0 def verifyHomepage(bbhandler): - pkg_pn = bbhandler.cooker.recipecache.pkg_pn + pkg_pn = bbhandler.cooker.recipecaches[''].pkg_pn pnlist = sorted(pkg_pn) count = 0 + checked = [] for pn in pnlist: - fn = pkg_pn[pn].pop() - data = bb.cache.Cache.loadDataFull(fn, bbhandler.cooker.collection.get_file_appends(fn), bbhandler.config_data) - homepage = data.getVar("HOMEPAGE") - if homepage: - try: - urllib2.urlopen(homepage, timeout=5) - except Exception: - count = count + wgetHomepage(pn, homepage) + for fn in pkg_pn[pn]: + # There's no point checking multiple BBCLASSEXTENDed variants of the same recipe + realfn, _, _ = bb.cache.virtualfn2realfn(fn) + if realfn in checked: + continue + data = bbhandler.parse_recipe_file(realfn) + homepage = data.getVar("HOMEPAGE") + if homepage: + try: + urllib.request.urlopen(homepage, timeout=5) + except Exception: + count = count + wgetHomepage(os.path.basename(realfn), homepage) + checked.append(realfn) return count if __name__=='__main__': - failcount = 0 - bbhandler = bb.tinfoil.Tinfoil() - bbhandler.prepare() - print "Start to verify HOMEPAGE:" - failcount = verifyHomepage(bbhandler) - print "finish to verify HOMEPAGE." - print "Summary: %s failed" % failcount + with bb.tinfoil.Tinfoil() as bbhandler: + bbhandler.prepare() + logger.info("Start verifying HOMEPAGE:") + failcount = verifyHomepage(bbhandler) + logger.info("Finished verifying HOMEPAGE.") + logger.info("Summary: %s failed" % failcount) |